From 68cb903594cd03dd708ef70c85c10807a6deefb5 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 18 Jul 2024 13:17:28 +0100
Subject: [PATCH 001/486] Revert d43ec97de081755990264049eba09cb7c83cb321
 "[X86] combineConcatVectorOps - IsConcatFree - peek through bitcasts to find
 inplace subvectors."

I've been given reports of this causing infinite loops downstream - I'm going to revert for now while I investigate.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 17 ++++++++---------
 llvm/test/CodeGen/X86/vselect-avx.ll    | 17 ++++++++++++++---
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 881e06e5f78b4..b07662b67e3e7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56163,19 +56163,18 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
     };
     auto IsConcatFree = [](MVT VT, ArrayRef<SDValue> SubOps, unsigned Op) {
       bool AllConstants = true;
-      bool AllSubs = true;
-      unsigned VecSize = VT.getSizeInBits();
+      bool AllSubVectors = true;
       for (unsigned I = 0, E = SubOps.size(); I != E; ++I) {
-        SDValue BC = peekThroughBitcasts(SubOps[I].getOperand(Op));
-        unsigned SubSize = BC.getValueSizeInBits();
-        unsigned EltSize = BC.getScalarValueSizeInBits();
+        SDValue Sub = SubOps[I].getOperand(Op);
+        unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
+        SDValue BC = peekThroughBitcasts(Sub);
         AllConstants &= ISD::isBuildVectorOfConstantSDNodes(BC.getNode()) ||
                         ISD::isBuildVectorOfConstantFPSDNodes(BC.getNode());
-        AllSubs &= BC.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
-                   BC.getOperand(0).getValueSizeInBits() == VecSize &&
-                   (BC.getConstantOperandVal(1) * EltSize) == (I * SubSize);
+        AllSubVectors &= Sub.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+                         Sub.getOperand(0).getValueType() == VT &&
+                         Sub.getConstantOperandAPInt(1) == (I * NumSubElts);
       }
-      return AllConstants || AllSubs;
+      return AllConstants || AllSubVectors;
     };
 
     switch (Op0.getOpcode()) {
diff --git a/llvm/test/CodeGen/X86/vselect-avx.ll b/llvm/test/CodeGen/X86/vselect-avx.ll
index 364390a4a60e5..bd26948766a56 100644
--- a/llvm/test/CodeGen/X86/vselect-avx.ll
+++ b/llvm/test/CodeGen/X86/vselect-avx.ll
@@ -259,7 +259,7 @@ define void @blendv_split(ptr %p, <8 x i32> %cond, <8 x i32> %a, <8 x i32> %x, <
   ret void
 }
 
-; Concatenate 128-bit pblendvb back together on AVX2+ targets (hidden by SSE __m128i bitcasts)
+; TODO: Concatenate 128-bit pblendvb back together on AVX2+ targets (hidden by SSE __m128i bitcasts)
 define <4 x i64> @vselect_concat_split_v16i8(<4 x i64> %a, <4 x i64> %b, <4 x i64>  %c, <4 x i64> %d) {
 ; AVX1-LABEL: vselect_concat_split_v16i8:
 ; AVX1:       ## %bb.0:
@@ -277,13 +277,24 @@ define <4 x i64> @vselect_concat_split_v16i8(<4 x i64> %a, <4 x i64> %b, <4 x i6
 ; AVX2-LABEL: vselect_concat_split_v16i8:
 ; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm3
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: vselect_concat_split_v16i8:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm2
-; AVX512-NEXT:    vpternlogq $216, %ymm2, %ymm1, %ymm0
+; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 killed $ymm1 def $ymm1
+; AVX512-NEXT:    vpternlogq $226, %xmm0, %xmm2, %xmm1
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT:    vpternlogq $226, %xmm0, %xmm3, %xmm4
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %a.bc = bitcast <4 x i64> %a to <32 x i8>
   %b.bc = bitcast <4 x i64> %b to <32 x i8>

From 8badfccefeaff2c05ef71a8d2fd6d803a1b4e129 Mon Sep 17 00:00:00 2001
From: jameshu15869 <55058507+jameshu15869@users.noreply.github.com>
Date: Thu, 18 Jul 2024 07:18:23 -0500
Subject: [PATCH 002/486] [libc] Add Multithreaded GPU Benchmarks (#98964)

This PR runs benchmarks on a 32 threads (A single warp on NVPTX) by
default, adding the option for single threaded benchmarks. We can
specify that a benchmark should be run on a single thread using the
`SINGLE_THREADED_BENCHMARK()` macro.

I chose to use a flag here so that other options could be added in the
future.
---
 libc/benchmarks/gpu/CMakeLists.txt            |  1 +
 libc/benchmarks/gpu/LibcGpuBenchmark.cpp      |  9 +++++++--
 libc/benchmarks/gpu/LibcGpuBenchmark.h        | 19 ++++++++++++++++---
 libc/benchmarks/gpu/src/ctype/CMakeLists.txt  |  2 ++
 .../gpu/src/ctype/isalnum_benchmark.cpp       |  4 ++++
 5 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/libc/benchmarks/gpu/CMakeLists.txt b/libc/benchmarks/gpu/CMakeLists.txt
index 14ba9f3f64b48..29e27724e1ab3 100644
--- a/libc/benchmarks/gpu/CMakeLists.txt
+++ b/libc/benchmarks/gpu/CMakeLists.txt
@@ -10,6 +10,7 @@ function(add_benchmark benchmark_name)
     "LINK_LIBRARIES" # Multi-value arguments
     ${ARGN}
   )
+  
   if(NOT libc.src.time.clock IN_LIST TARGET_LLVMLIBC_ENTRYPOINTS)
     message(FATAL_ERROR "target does not support clock")
   endif()
diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
index 23fff3e8180f7..c926d8efd7db2 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
@@ -114,8 +114,13 @@ void Benchmark::run_benchmarks() {
       all_results.reset();
 
     gpu::sync_threads();
-    auto current_result = b->run();
-    all_results.update(current_result);
+    if (!b->flags ||
+        ((b->flags & BenchmarkFlags::SINGLE_THREADED) && id == 0) ||
+        ((b->flags & BenchmarkFlags::SINGLE_WAVE) &&
+         id < gpu::get_lane_size())) {
+      auto current_result = b->run();
+      all_results.update(current_result);
+    }
     gpu::sync_threads();
 
     if (id == 0)
diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.h b/libc/benchmarks/gpu/LibcGpuBenchmark.h
index 1f813f8655de6..29d7ba8b0a132 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.h
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.h
@@ -74,16 +74,19 @@ struct BenchmarkResult {
   clock_t total_time = 0;
 };
 
+enum BenchmarkFlags { SINGLE_THREADED = 0x1, SINGLE_WAVE = 0x2 };
+
 BenchmarkResult benchmark(const BenchmarkOptions &options,
                           cpp::function<uint64_t(void)> wrapper_func);
 
 class Benchmark {
   const cpp::function<uint64_t(void)> func;
   const cpp::string_view name;
+  const uint8_t flags;
 
 public:
-  Benchmark(cpp::function<uint64_t(void)> func, char const *name)
-      : func(func), name(name) {
+  Benchmark(cpp::function<uint64_t(void)> func, char const *name, uint8_t flags)
+      : func(func), name(name), flags(flags) {
     add_benchmark(this);
   }
 
@@ -104,6 +107,16 @@ class Benchmark {
 
 #define BENCHMARK(SuiteName, TestName, Func)                                   \
   LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance(     \
-      Func, #SuiteName "." #TestName)
+      Func, #SuiteName "." #TestName, 0)
+
+#define SINGLE_THREADED_BENCHMARK(SuiteName, TestName, Func)                   \
+  LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance(     \
+      Func, #SuiteName "." #TestName,                                          \
+      LIBC_NAMESPACE::benchmarks::BenchmarkFlags::SINGLE_THREADED)
+
+#define SINGLE_WAVE_BENCHMARK(SuiteName, TestName, Func)                       \
+  LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance(     \
+      Func, #SuiteName "." #TestName,                                          \
+      LIBC_NAMESPACE::benchmarks::BenchmarkFlags::SINGLE_WAVE)
 
 #endif
diff --git a/libc/benchmarks/gpu/src/ctype/CMakeLists.txt b/libc/benchmarks/gpu/src/ctype/CMakeLists.txt
index 79f01425770da..f277624dbb901 100644
--- a/libc/benchmarks/gpu/src/ctype/CMakeLists.txt
+++ b/libc/benchmarks/gpu/src/ctype/CMakeLists.txt
@@ -8,6 +8,8 @@ add_benchmark(
     isalnum_benchmark.cpp
   DEPENDS
     libc.src.ctype.isalnum
+  LOADER_ARGS
+    --threads 64
 )
 
 add_benchmark(
diff --git a/libc/benchmarks/gpu/src/ctype/isalnum_benchmark.cpp b/libc/benchmarks/gpu/src/ctype/isalnum_benchmark.cpp
index 6f8d247902f76..ffa5a99860bfc 100644
--- a/libc/benchmarks/gpu/src/ctype/isalnum_benchmark.cpp
+++ b/libc/benchmarks/gpu/src/ctype/isalnum_benchmark.cpp
@@ -7,6 +7,10 @@ uint64_t BM_IsAlnum() {
   return LIBC_NAMESPACE::latency(LIBC_NAMESPACE::isalnum, x);
 }
 BENCHMARK(LlvmLibcIsAlNumGpuBenchmark, IsAlnum, BM_IsAlnum);
+SINGLE_THREADED_BENCHMARK(LlvmLibcIsAlNumGpuBenchmark, IsAlnumSingleThread,
+                          BM_IsAlnum);
+SINGLE_WAVE_BENCHMARK(LlvmLibcIsAlNumGpuBenchmark, IsAlnumSingleWave,
+                      BM_IsAlnum);
 
 uint64_t BM_IsAlnumCapital() {
   char x = 'A';

From bf02f41726a48e5eb1dbe7c188f9e36ec6a29ac2 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Thu, 18 Jul 2024 08:23:41 -0400
Subject: [PATCH 003/486] Fix a regression with alignas on structure members in
 C (#98642)

This was a 19.x regression and thus has no release note.

Fixes #95032
---
 clang/lib/Sema/ParsedAttr.cpp   |  2 +-
 clang/lib/Sema/SemaDeclAttr.cpp | 56 ++++++++++++++++++++-------------
 clang/test/C/C2y/n3254.c        | 28 ++++++++---------
 clang/test/Sema/alignas.c       | 19 +++++++++--
 4 files changed, 66 insertions(+), 39 deletions(-)

diff --git a/clang/lib/Sema/ParsedAttr.cpp b/clang/lib/Sema/ParsedAttr.cpp
index 6abc90336c994..2109494aa5889 100644
--- a/clang/lib/Sema/ParsedAttr.cpp
+++ b/clang/lib/Sema/ParsedAttr.cpp
@@ -225,7 +225,7 @@ bool ParsedAttr::slidesFromDeclToDeclSpecLegacyBehavior() const {
     // atributes.
     return false;
 
-  assert(isStandardAttributeSyntax());
+  assert(isStandardAttributeSyntax() || isAlignas());
 
   // We have historically allowed some type attributes with standard attribute
   // syntax to slide to the decl-specifier-seq, so we have to keep supporting
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 20f46c003a464..41295bfb3b94f 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -6435,8 +6435,14 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
     return;
 
   // Ignore C++11 attributes on declarator chunks: they appertain to the type
-  // instead.
-  if (AL.isCXX11Attribute() && !Options.IncludeCXX11Attributes)
+  // instead. Note, isCXX11Attribute() will look at whether the attribute is
+  // [[]] or alignas, while isC23Attribute() will only look at [[]]. This is
+  // important for ensuring that alignas in C23 is properly handled on a
+  // structure member declaration because it is a type-specifier-qualifier in
+  // C but still applies to the declaration rather than the type.
+  if ((S.getLangOpts().CPlusPlus ? AL.isCXX11Attribute()
+                                 : AL.isC23Attribute()) &&
+      !Options.IncludeCXX11Attributes)
     return;
 
   // Unknown attributes are automatically warned on. Target-specific attributes
@@ -7500,29 +7506,37 @@ void Sema::ProcessDeclAttributes(Scope *S, Decl *D, const Declarator &PD) {
   // Ordering of attributes can be important, so we take care to process
   // attributes in the order in which they appeared in the source code.
 
+  auto ProcessAttributesWithSliding =
+      [&](const ParsedAttributesView &Src,
+          const ProcessDeclAttributeOptions &Options) {
+        ParsedAttributesView NonSlidingAttrs;
+        for (ParsedAttr &AL : Src) {
+          // FIXME: this sliding is specific to standard attributes and should
+          // eventually be deprecated and removed as those are not intended to
+          // slide to anything.
+          if ((AL.isStandardAttributeSyntax() || AL.isAlignas()) &&
+              AL.slidesFromDeclToDeclSpecLegacyBehavior()) {
+            // Skip processing the attribute, but do check if it appertains to
+            // the declaration. This is needed for the `MatrixType` attribute,
+            // which, despite being a type attribute, defines a `SubjectList`
+            // that only allows it to be used on typedef declarations.
+            AL.diagnoseAppertainsTo(*this, D);
+          } else {
+            NonSlidingAttrs.addAtEnd(&AL);
+          }
+        }
+        ProcessDeclAttributeList(S, D, NonSlidingAttrs, Options);
+      };
+
   // First, process attributes that appeared on the declaration itself (but
   // only if they don't have the legacy behavior of "sliding" to the DeclSepc).
-  ParsedAttributesView NonSlidingAttrs;
-  for (ParsedAttr &AL : PD.getDeclarationAttributes()) {
-    if (AL.slidesFromDeclToDeclSpecLegacyBehavior()) {
-      // Skip processing the attribute, but do check if it appertains to the
-      // declaration. This is needed for the `MatrixType` attribute, which,
-      // despite being a type attribute, defines a `SubjectList` that only
-      // allows it to be used on typedef declarations.
-      AL.diagnoseAppertainsTo(*this, D);
-    } else {
-      NonSlidingAttrs.addAtEnd(&AL);
-    }
-  }
-  ProcessDeclAttributeList(S, D, NonSlidingAttrs);
+  ProcessAttributesWithSliding(PD.getDeclarationAttributes(), {});
 
   // Apply decl attributes from the DeclSpec if present.
-  if (!PD.getDeclSpec().getAttributes().empty()) {
-    ProcessDeclAttributeList(S, D, PD.getDeclSpec().getAttributes(),
-                             ProcessDeclAttributeOptions()
-                                 .WithIncludeCXX11Attributes(false)
-                                 .WithIgnoreTypeAttributes(true));
-  }
+  ProcessAttributesWithSliding(PD.getDeclSpec().getAttributes(),
+                               ProcessDeclAttributeOptions()
+                                   .WithIncludeCXX11Attributes(false)
+                                   .WithIgnoreTypeAttributes(true));
 
   // Walk the declarator structure, applying decl attributes that were in a type
   // position to the decl itself.  This handles cases like:
diff --git a/clang/test/C/C2y/n3254.c b/clang/test/C/C2y/n3254.c
index e08659cf377aa..60d068cf9980b 100644
--- a/clang/test/C/C2y/n3254.c
+++ b/clang/test/C/C2y/n3254.c
@@ -21,9 +21,9 @@ struct S {
 // CHECK-LABEL: define dso_local i32 @foo(
 // CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[BUFFER:%.*]] = alloca [12 x i8], align 1
+// CHECK-NEXT:    [[BUFFER:%.*]] = alloca [12 x i8], align 4
 // CHECK-NEXT:    [[S_PTR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[BUFFER]], i8 0, i64 12, i1 false)
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[BUFFER]], i8 0, i64 12, i1 false)
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER]], i64 0, i64 0
 // CHECK-NEXT:    store ptr [[ARRAYDECAY]], ptr [[S_PTR]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_PTR]], align 8
@@ -40,13 +40,13 @@ int foo() {
 // CHECK-LABEL: define dso_local signext i8 @bar(
 // CHECK-SAME: ) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[BUFFER:%.*]] = alloca [12 x i8], align 1
+// CHECK-NEXT:    [[BUFFER:%.*]] = alloca [12 x i8], align 4
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER]], i64 0, i64 0
 // CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[ARRAYDECAY]], i32 0, i32 1
-// CHECK-NEXT:    store i8 97, ptr [[C]], align 1
+// CHECK-NEXT:    store i8 97, ptr [[C]], align 4
 // CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER]], i64 0, i64 0
 // CHECK-NEXT:    [[C2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDECAY1]], i32 0, i32 1
-// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[C2]], align 1
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[C2]], align 4
 // CHECK-NEXT:    ret i8 [[TMP0]]
 //
 char bar() {
@@ -58,13 +58,13 @@ char bar() {
 // CHECK-LABEL: define dso_local float @baz(
 // CHECK-SAME: ) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[BUFFER:%.*]] = alloca [12 x i8], align 1
+// CHECK-NEXT:    [[BUFFER:%.*]] = alloca [12 x i8], align 4
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER]], i64 0, i64 0
 // CHECK-NEXT:    [[F:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[ARRAYDECAY]], i32 0, i32 2
-// CHECK-NEXT:    store float 3.000000e+00, ptr [[F]], align 1
+// CHECK-NEXT:    store float 3.000000e+00, ptr [[F]], align 4
 // CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER]], i64 0, i64 0
 // CHECK-NEXT:    [[F2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDECAY1]], i32 0, i32 2
-// CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F2]], align 1
+// CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F2]], align 4
 // CHECK-NEXT:    ret float [[TMP0]]
 //
 float baz() {
@@ -80,9 +80,9 @@ struct T {
 // CHECK-LABEL: define dso_local signext i8 @quux(
 // CHECK-SAME: ) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[T:%.*]] = alloca [[STRUCT_T:%.*]], align 1
+// CHECK-NEXT:    [[T:%.*]] = alloca [[STRUCT_T:%.*]], align 4
 // CHECK-NEXT:    [[S_PTR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[T]], i8 0, i64 12, i1 false)
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[T]], i8 0, i64 12, i1 false)
 // CHECK-NEXT:    [[BUFFER:%.*]] = getelementptr inbounds [[STRUCT_T]], ptr [[T]], i32 0, i32 0
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER]], i64 0, i64 0
 // CHECK-NEXT:    store ptr [[ARRAYDECAY]], ptr [[S_PTR]], align 8
@@ -100,10 +100,10 @@ char quux() {
 // CHECK-LABEL: define dso_local float @quibble(
 // CHECK-SAME: ) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[BUFFER:%.*]] = alloca [12 x i8], align 1
+// CHECK-NEXT:    [[BUFFER:%.*]] = alloca [12 x i8], align 4
 // CHECK-NEXT:    [[T_PTR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[S_PTR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[BUFFER]], i8 0, i64 12, i1 false)
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[BUFFER]], i8 0, i64 12, i1 false)
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER]], i64 0, i64 0
 // CHECK-NEXT:    store ptr [[ARRAYDECAY]], ptr [[T_PTR]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_PTR]], align 8
@@ -125,13 +125,13 @@ float quibble() {
 // CHECK-LABEL: define dso_local i32 @quorble(
 // CHECK-SAME: ) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[BUFFER:%.*]] = alloca [12 x i8], align 1
+// CHECK-NEXT:    [[BUFFER:%.*]] = alloca [12 x i8], align 4
 // CHECK-NEXT:    [[S_PTR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER]], i64 0, i64 0
 // CHECK-NEXT:    [[BUFFER1:%.*]] = getelementptr inbounds [[STRUCT_T:%.*]], ptr [[ARRAYDECAY]], i32 0, i32 0
 // CHECK-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER1]], i64 0, i64 0
 // CHECK-NEXT:    [[X:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[ARRAYDECAY2]], i32 0, i32 0
-// CHECK-NEXT:    store i32 12, ptr [[X]], align 1
+// CHECK-NEXT:    store i32 12, ptr [[X]], align 4
 // CHECK-NEXT:    [[ARRAYDECAY3:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER]], i64 0, i64 0
 // CHECK-NEXT:    [[BUFFER4:%.*]] = getelementptr inbounds [[STRUCT_T]], ptr [[ARRAYDECAY3]], i32 0, i32 0
 // CHECK-NEXT:    [[ARRAYDECAY5:%.*]] = getelementptr inbounds [12 x i8], ptr [[BUFFER4]], i64 0, i64 0
diff --git a/clang/test/Sema/alignas.c b/clang/test/Sema/alignas.c
index 020eff6a141c0..391553bc540ec 100644
--- a/clang/test/Sema/alignas.c
+++ b/clang/test/Sema/alignas.c
@@ -1,5 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c11 -Dalignof=__alignof %s
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c11 -Dalignof=_Alignof -DUSING_C11_SYNTAX %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c23 -DUSING_C11_SYNTAX %s
 
 _Alignas(3) int align_illegal; //expected-error {{requested alignment is not a power of 2}}
 _Alignas(int) char align_big;
@@ -18,12 +19,24 @@ void f(_Alignas(1) char c) { // expected-error {{'_Alignas' attribute cannot be
 }
 
 #ifdef USING_C11_SYNTAX
-// expected-warning@+4{{'_Alignof' applied to an expression is a GNU extension}}
-// expected-warning@+4{{'_Alignof' applied to an expression is a GNU extension}}
-// expected-warning@+4{{'_Alignof' applied to an expression is a GNU extension}}
+// expected-warning-re@+4{{'{{(_A|a)}}lignof' applied to an expression is a GNU extension}}
+// expected-warning-re@+4{{'{{(_A|a)}}lignof' applied to an expression is a GNU extension}}
+// expected-warning-re@+4{{'{{(_A|a)}}lignof' applied to an expression is a GNU extension}}
 #endif
 _Static_assert(alignof(align_big) == alignof(int), "k's alignment is wrong");
 _Static_assert(alignof(align_small) == 1, "j's alignment is wrong");
 _Static_assert(alignof(align_multiple) == 8, "l's alignment is wrong");
 _Static_assert(alignof(struct align_member) == 8, "quuux's alignment is wrong");
 _Static_assert(sizeof(struct align_member) == 8, "quuux's size is wrong");
+
+struct GH95032_1 {
+  _Alignas(16) char bytes[16];
+};
+_Static_assert(_Alignof(struct GH95032_1) == 16, "");
+
+#if __STDC_VERSION__ >= 202311L
+struct GH95032_2 {
+  alignas(16) char bytes[16];
+};
+static_assert(alignof(struct GH95032_2) == 16);
+#endif

From c5f402f95d9617882b26d5799f503383b895c2e4 Mon Sep 17 00:00:00 2001
From: Egor Zhdan <e_zhdan@apple.com>
Date: Thu, 18 Jul 2024 13:27:24 +0100
Subject: [PATCH 004/486] [APINotes] Reduce memory footprint for Obj-C/C++
 contexts

We were storing extraneous data for certain Objective-C/C++ entities.
Specifically, for declarations that can be nested in another context
(such as functions) we were storing the kind of the parent context in
addition to its ID. The ID is always sufficient.

This removes the logically incorrect usages of `ContextTableKey` that
don't actually describe a context, but rather describe a single
declaration. This introduces `SingleDeclTableKey` to store that kind of
entities in a more compact and reasonable way.
---
 clang/lib/APINotes/APINotesFormat.h   | 57 +++++++++++++++++++++++----
 clang/lib/APINotes/APINotesReader.cpp | 32 ++++++---------
 clang/lib/APINotes/APINotesWriter.cpp | 46 +++++++++++----------
 3 files changed, 84 insertions(+), 51 deletions(-)

diff --git a/clang/lib/APINotes/APINotesFormat.h b/clang/lib/APINotes/APINotesFormat.h
index e3aa76df8316c..42dfe7a773a97 100644
--- a/clang/lib/APINotes/APINotesFormat.h
+++ b/clang/lib/APINotes/APINotesFormat.h
@@ -24,7 +24,7 @@ const uint16_t VERSION_MAJOR = 0;
 /// API notes file minor version number.
 ///
 /// When the format changes IN ANY WAY, this number should be incremented.
-const uint16_t VERSION_MINOR = 26; // SwiftCopyable
+const uint16_t VERSION_MINOR = 27; // SingleDeclTableKey
 
 const uint8_t kSwiftCopyable = 1;
 const uint8_t kSwiftNonCopyable = 2;
@@ -269,12 +269,6 @@ struct ContextTableKey {
       : parentContextID(parentContextID), contextKind(contextKind),
         contextID(contextID) {}
 
-  ContextTableKey(std::optional<Context> context, IdentifierID nameID)
-      : parentContextID(context ? context->id.Value : (uint32_t)-1),
-        contextKind(context ? static_cast<uint8_t>(context->kind)
-                            : static_cast<uint8_t>(-1)),
-        contextID(nameID) {}
-
   llvm::hash_code hashValue() const {
     return llvm::hash_value(
         std::tuple{parentContextID, contextKind, contextID});
@@ -286,6 +280,32 @@ inline bool operator==(const ContextTableKey &lhs, const ContextTableKey &rhs) {
          lhs.contextKind == rhs.contextKind && lhs.contextID == rhs.contextID;
 }
 
+/// A stored Objective-C or C++ declaration, represented by the ID of its parent
+/// context, and the name of the declaration.
+struct SingleDeclTableKey {
+  uint32_t parentContextID;
+  uint32_t nameID;
+
+  SingleDeclTableKey() : parentContextID(-1), nameID(-1) {}
+
+  SingleDeclTableKey(uint32_t ParentContextID, uint32_t NameID)
+      : parentContextID(ParentContextID), nameID(NameID) {}
+
+  SingleDeclTableKey(std::optional<Context> ParentCtx, IdentifierID NameID)
+      : parentContextID(ParentCtx ? ParentCtx->id.Value
+                                  : static_cast<uint32_t>(-1)),
+        nameID(NameID) {}
+
+  llvm::hash_code hashValue() const {
+    return llvm::hash_value(std::make_pair(parentContextID, nameID));
+  }
+};
+
+inline bool operator==(const SingleDeclTableKey &lhs,
+                       const SingleDeclTableKey &rhs) {
+  return lhs.parentContextID == rhs.parentContextID && lhs.nameID == rhs.nameID;
+}
+
 } // namespace api_notes
 } // namespace clang
 
@@ -341,6 +361,29 @@ template <> struct DenseMapInfo<clang::api_notes::ContextTableKey> {
     return lhs == rhs;
   }
 };
+
+template <> struct DenseMapInfo<clang::api_notes::SingleDeclTableKey> {
+  static inline clang::api_notes::SingleDeclTableKey getEmptyKey() {
+    return clang::api_notes::SingleDeclTableKey();
+  }
+
+  static inline clang::api_notes::SingleDeclTableKey getTombstoneKey() {
+    return clang::api_notes::SingleDeclTableKey{
+        DenseMapInfo<uint32_t>::getTombstoneKey(),
+        DenseMapInfo<uint32_t>::getTombstoneKey()};
+  }
+
+  static unsigned
+  getHashValue(const clang::api_notes::SingleDeclTableKey &value) {
+    return value.hashValue();
+  }
+
+  static bool isEqual(const clang::api_notes::SingleDeclTableKey &lhs,
+                      const clang::api_notes::SingleDeclTableKey &rhs) {
+    return lhs == rhs;
+  }
+};
+
 } // namespace llvm
 
 #endif
diff --git a/clang/lib/APINotes/APINotesReader.cpp b/clang/lib/APINotes/APINotesReader.cpp
index 8454e092b55ac..7600738374840 100644
--- a/clang/lib/APINotes/APINotesReader.cpp
+++ b/clang/lib/APINotes/APINotesReader.cpp
@@ -429,15 +429,13 @@ class ObjCSelectorTableInfo {
 
 /// Used to deserialize the on-disk global variable table.
 class GlobalVariableTableInfo
-    : public VersionedTableInfo<GlobalVariableTableInfo, ContextTableKey,
+    : public VersionedTableInfo<GlobalVariableTableInfo, SingleDeclTableKey,
                                 GlobalVariableInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
     auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
-    auto ContextKind =
-        endian::readNext<uint8_t, llvm::endianness::little>(Data);
     auto NameID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
-    return {CtxID, ContextKind, NameID};
+    return {CtxID, NameID};
   }
 
   hash_value_type ComputeHash(internal_key_type Key) {
@@ -454,15 +452,13 @@ class GlobalVariableTableInfo
 
 /// Used to deserialize the on-disk global function table.
 class GlobalFunctionTableInfo
-    : public VersionedTableInfo<GlobalFunctionTableInfo, ContextTableKey,
+    : public VersionedTableInfo<GlobalFunctionTableInfo, SingleDeclTableKey,
                                 GlobalFunctionInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
     auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
-    auto ContextKind =
-        endian::readNext<uint8_t, llvm::endianness::little>(Data);
     auto NameID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
-    return {CtxID, ContextKind, NameID};
+    return {CtxID, NameID};
   }
 
   hash_value_type ComputeHash(internal_key_type Key) {
@@ -501,15 +497,13 @@ class EnumConstantTableInfo
 
 /// Used to deserialize the on-disk tag table.
 class TagTableInfo
-    : public VersionedTableInfo<TagTableInfo, ContextTableKey, TagInfo> {
+    : public VersionedTableInfo<TagTableInfo, SingleDeclTableKey, TagInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
     auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
-    auto ContextKind =
-        endian::readNext<uint8_t, llvm::endianness::little>(Data);
     auto NameID =
         endian::readNext<IdentifierID, llvm::endianness::little>(Data);
-    return {CtxID, ContextKind, NameID};
+    return {CtxID, NameID};
   }
 
   hash_value_type ComputeHash(internal_key_type Key) {
@@ -563,16 +557,14 @@ class TagTableInfo
 
 /// Used to deserialize the on-disk typedef table.
 class TypedefTableInfo
-    : public VersionedTableInfo<TypedefTableInfo, ContextTableKey,
+    : public VersionedTableInfo<TypedefTableInfo, SingleDeclTableKey,
                                 TypedefInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
     auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
-    auto ContextKind =
-        endian::readNext<uint8_t, llvm::endianness::little>(Data);
     auto nameID =
         endian::readNext<IdentifierID, llvm::endianness::little>(Data);
-    return {CtxID, ContextKind, nameID};
+    return {CtxID, nameID};
   }
 
   hash_value_type ComputeHash(internal_key_type Key) {
@@ -1929,7 +1921,7 @@ auto APINotesReader::lookupGlobalVariable(llvm::StringRef Name,
   if (!NameID)
     return std::nullopt;
 
-  ContextTableKey Key(Ctx, *NameID);
+  SingleDeclTableKey Key(Ctx, *NameID);
 
   auto Known = Implementation->GlobalVariableTable->find(Key);
   if (Known == Implementation->GlobalVariableTable->end())
@@ -1948,7 +1940,7 @@ auto APINotesReader::lookupGlobalFunction(llvm::StringRef Name,
   if (!NameID)
     return std::nullopt;
 
-  ContextTableKey Key(Ctx, *NameID);
+  SingleDeclTableKey Key(Ctx, *NameID);
 
   auto Known = Implementation->GlobalFunctionTable->find(Key);
   if (Known == Implementation->GlobalFunctionTable->end())
@@ -1982,7 +1974,7 @@ auto APINotesReader::lookupTag(llvm::StringRef Name, std::optional<Context> Ctx)
   if (!NameID)
     return std::nullopt;
 
-  ContextTableKey Key(Ctx, *NameID);
+  SingleDeclTableKey Key(Ctx, *NameID);
 
   auto Known = Implementation->TagTable->find(Key);
   if (Known == Implementation->TagTable->end())
@@ -2001,7 +1993,7 @@ auto APINotesReader::lookupTypedef(llvm::StringRef Name,
   if (!NameID)
     return std::nullopt;
 
-  ContextTableKey Key(Ctx, *NameID);
+  SingleDeclTableKey Key(Ctx, *NameID);
 
   auto Known = Implementation->TypedefTable->find(Key);
   if (Known == Implementation->TypedefTable->end())
diff --git a/clang/lib/APINotes/APINotesWriter.cpp b/clang/lib/APINotes/APINotesWriter.cpp
index 4053d515ef426..1090d3f20df21 100644
--- a/clang/lib/APINotes/APINotesWriter.cpp
+++ b/clang/lib/APINotes/APINotesWriter.cpp
@@ -75,17 +75,17 @@ class APINotesWriter::Implementation {
 
   /// Information about global variables.
   ///
-  /// Indexed by the context ID, contextKind, identifier ID.
+  /// Indexed by the context ID, identifier ID.
   llvm::DenseMap<
-      ContextTableKey,
+      SingleDeclTableKey,
       llvm::SmallVector<std::pair<VersionTuple, GlobalVariableInfo>, 1>>
       GlobalVariables;
 
   /// Information about global functions.
   ///
-  /// Indexed by the context ID, contextKind, identifier ID.
+  /// Indexed by the context ID, identifier ID.
   llvm::DenseMap<
-      ContextTableKey,
+      SingleDeclTableKey,
       llvm::SmallVector<std::pair<VersionTuple, GlobalFunctionInfo>, 1>>
       GlobalFunctions;
 
@@ -98,15 +98,15 @@ class APINotesWriter::Implementation {
 
   /// Information about tags.
   ///
-  /// Indexed by the context ID, contextKind, identifier ID.
-  llvm::DenseMap<ContextTableKey,
+  /// Indexed by the context ID, identifier ID.
+  llvm::DenseMap<SingleDeclTableKey,
                  llvm::SmallVector<std::pair<VersionTuple, TagInfo>, 1>>
       Tags;
 
   /// Information about typedefs.
   ///
-  /// Indexed by the context ID, contextKind, identifier ID.
-  llvm::DenseMap<ContextTableKey,
+  /// Indexed by the context ID, identifier ID.
+  llvm::DenseMap<SingleDeclTableKey,
                  llvm::SmallVector<std::pair<VersionTuple, TypedefInfo>, 1>>
       Typedefs;
 
@@ -865,18 +865,17 @@ void APINotesWriter::Implementation::writeObjCSelectorBlock(
 namespace {
 /// Used to serialize the on-disk global variable table.
 class GlobalVariableTableInfo
-    : public VersionedTableInfo<GlobalVariableTableInfo, ContextTableKey,
+    : public VersionedTableInfo<GlobalVariableTableInfo, SingleDeclTableKey,
                                 GlobalVariableInfo> {
 public:
   unsigned getKeyLength(key_type_ref) {
-    return sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint32_t);
+    return sizeof(uint32_t) + sizeof(uint32_t);
   }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
     llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Key.parentContextID);
-    writer.write<uint8_t>(Key.contextKind);
-    writer.write<uint32_t>(Key.contextID);
+    writer.write<uint32_t>(Key.nameID);
   }
 
   hash_value_type ComputeHash(key_type_ref Key) {
@@ -979,18 +978,17 @@ void emitFunctionInfo(raw_ostream &OS, const FunctionInfo &FI) {
 
 /// Used to serialize the on-disk global function table.
 class GlobalFunctionTableInfo
-    : public VersionedTableInfo<GlobalFunctionTableInfo, ContextTableKey,
+    : public VersionedTableInfo<GlobalFunctionTableInfo, SingleDeclTableKey,
                                 GlobalFunctionInfo> {
 public:
   unsigned getKeyLength(key_type_ref) {
-    return sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint32_t);
+    return sizeof(uint32_t) + sizeof(uint32_t);
   }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
     llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Key.parentContextID);
-    writer.write<uint8_t>(Key.contextKind);
-    writer.write<uint32_t>(Key.contextID);
+    writer.write<uint32_t>(Key.nameID);
   }
 
   hash_value_type ComputeHash(key_type_ref Key) {
@@ -1091,20 +1089,20 @@ void APINotesWriter::Implementation::writeEnumConstantBlock(
 namespace {
 template <typename Derived, typename UnversionedDataType>
 class CommonTypeTableInfo
-    : public VersionedTableInfo<Derived, ContextTableKey, UnversionedDataType> {
+    : public VersionedTableInfo<Derived, SingleDeclTableKey,
+                                UnversionedDataType> {
 public:
   using key_type_ref = typename CommonTypeTableInfo::key_type_ref;
   using hash_value_type = typename CommonTypeTableInfo::hash_value_type;
 
   unsigned getKeyLength(key_type_ref) {
-    return sizeof(uint32_t) + sizeof(uint8_t) + sizeof(IdentifierID);
+    return sizeof(uint32_t) + sizeof(IdentifierID);
   }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
     llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Key.parentContextID);
-    writer.write<uint8_t>(Key.contextKind);
-    writer.write<IdentifierID>(Key.contextID);
+    writer.write<IdentifierID>(Key.nameID);
   }
 
   hash_value_type ComputeHash(key_type_ref Key) {
@@ -1351,7 +1349,7 @@ void APINotesWriter::addGlobalVariable(std::optional<Context> Ctx,
                                        const GlobalVariableInfo &Info,
                                        VersionTuple SwiftVersion) {
   IdentifierID VariableID = Implementation->getIdentifier(Name);
-  ContextTableKey Key(Ctx, VariableID);
+  SingleDeclTableKey Key(Ctx, VariableID);
   Implementation->GlobalVariables[Key].push_back({SwiftVersion, Info});
 }
 
@@ -1360,7 +1358,7 @@ void APINotesWriter::addGlobalFunction(std::optional<Context> Ctx,
                                        const GlobalFunctionInfo &Info,
                                        VersionTuple SwiftVersion) {
   IdentifierID NameID = Implementation->getIdentifier(Name);
-  ContextTableKey Key(Ctx, NameID);
+  SingleDeclTableKey Key(Ctx, NameID);
   Implementation->GlobalFunctions[Key].push_back({SwiftVersion, Info});
 }
 
@@ -1374,7 +1372,7 @@ void APINotesWriter::addEnumConstant(llvm::StringRef Name,
 void APINotesWriter::addTag(std::optional<Context> Ctx, llvm::StringRef Name,
                             const TagInfo &Info, VersionTuple SwiftVersion) {
   IdentifierID TagID = Implementation->getIdentifier(Name);
-  ContextTableKey Key(Ctx, TagID);
+  SingleDeclTableKey Key(Ctx, TagID);
   Implementation->Tags[Key].push_back({SwiftVersion, Info});
 }
 
@@ -1382,7 +1380,7 @@ void APINotesWriter::addTypedef(std::optional<Context> Ctx,
                                 llvm::StringRef Name, const TypedefInfo &Info,
                                 VersionTuple SwiftVersion) {
   IdentifierID TypedefID = Implementation->getIdentifier(Name);
-  ContextTableKey Key(Ctx, TypedefID);
+  SingleDeclTableKey Key(Ctx, TypedefID);
   Implementation->Typedefs[Key].push_back({SwiftVersion, Info});
 }
 } // namespace api_notes

From 39bb244a16e59cb8f2080f96e9de599007762635 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen@sifive.com>
Date: Thu, 18 Jul 2024 20:49:53 +0800
Subject: [PATCH 005/486] [SLP][REVEC] Make Instruction::Call support vector
 instructions. (#99317)

---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  2 +-
 llvm/test/Transforms/SLPVectorizer/revec.ll   | 20 +++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b994645cece61..d8c3bae06e932 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -13499,7 +13499,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) {
         }
         ScalarArg = CEI->getArgOperand(I);
         if (cast<VectorType>(OpVec->getType())->getElementType() !=
-                ScalarArg->getType() &&
+                ScalarArg->getType()->getScalarType() &&
             It == MinBWs.end()) {
           auto *CastTy =
               getWidenedType(ScalarArg->getType(), VecTy->getNumElements());
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 4b37b100763a9..c2dc6d0ab73b7 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -38,3 +38,23 @@ entry:
   store <4 x i32> %add.i65, ptr %arrayidx42, align 4
   ret void
 }
+
+define void @test2(ptr %in, ptr %out) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i16>, ptr [[IN:%.*]], align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP0]], <16 x i16> [[TMP0]])
+; CHECK-NEXT:    store <16 x i16> [[TMP1]], ptr [[OUT:%.*]], align 2
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = getelementptr i16, ptr %in, i64 8
+  %1 = load <8 x i16>, ptr %in, align 2
+  %2 = load <8 x i16>, ptr %0, align 2
+  %3 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %1, <8 x i16> %1)
+  %4 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %2, <8 x i16> %2)
+  %5 = getelementptr i16, ptr %out, i64 8
+  store <8 x i16> %3, ptr %out, align 2
+  store <8 x i16> %4, ptr %5, align 2
+  ret void
+}

From 27ec379f636ceac655faa290e78735ea98e02cbf Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Thu, 18 Jul 2024 13:53:22 +0100
Subject: [PATCH 006/486] [AMDGPU] Do not select llvm.amdgcn.inverse.ballot
 with wrong wave size (#99470)

This produces a "cannot select" error, instead of failing later with an
illegal vgpr to sgpr copy.
---
 llvm/lib/Target/AMDGPU/SIInstructions.td               |  2 ++
 .../CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i32.ll   | 10 ++++++++--
 .../CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i64.ll   | 10 ++++++++--
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index f2721fbd164bf..2e617e5646c59 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -212,11 +212,13 @@ def EXIT_STRICT_WQM : SPseudoInstSI <(outs SReg_1:$sdst), (ins SReg_1:$src0)> {
 }
 
 let usesCustomInserter = 1 in {
+let WaveSizePredicate = isWave32 in
 def S_INVERSE_BALLOT_U32 : SPseudoInstSI<
   (outs SReg_32:$sdst), (ins SSrc_b32:$mask),
   [(set i1:$sdst, (int_amdgcn_inverse_ballot i32:$mask))]
 >;
 
+let WaveSizePredicate = isWave64 in
 def S_INVERSE_BALLOT_U64 : SPseudoInstSI<
   (outs SReg_64:$sdst), (ins SSrc_b64:$mask),
   [(set i1:$sdst, (int_amdgcn_inverse_ballot i64:$mask))]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i32.ll
index 71ed71cd84bcd..3781faa54e7dc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i32.ll
@@ -1,6 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32,-wavefrontsize64 -global-isel=1 -verify-machineinstrs < %s | FileCheck  -check-prefixes=GFX11,GISEL %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32,-wavefrontsize64 -global-isel=0 -verify-machineinstrs < %s | FileCheck  -check-prefixes=GFX11,SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32 -global-isel=1 -verify-machineinstrs < %s | FileCheck  -check-prefixes=GFX11,GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32 -global-isel=0 -verify-machineinstrs < %s | FileCheck  -check-prefixes=GFX11,SDAG %s
+
+; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -global-isel=1 < %s 2>&1 | FileCheck  -check-prefix=GISEL-ERR %s
+; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -global-isel=0 < %s 2>&1 | FileCheck  -check-prefix=SDAG-ERR %s
+
+; GISEL-ERR: LLVM ERROR: cannot select: {{.*}}  = G_INTRINSIC intrinsic(@llvm.amdgcn.inverse.ballot)
+; SDAG-ERR: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.inverse.ballot
 
 declare i1 @llvm.amdgcn.inverse.ballot(i32)
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i64.ll
index 2e3dc11feed1e..29218a3625216 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i64.ll
@@ -1,6 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=-wavefrontsize32,+wavefrontsize64 -global-isel=1 -verify-machineinstrs < %s | FileCheck -check-prefixes=GISEL %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=-wavefrontsize32,+wavefrontsize64 -global-isel=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize64 -global-isel=1 -verify-machineinstrs < %s | FileCheck -check-prefixes=GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize64 -global-isel=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=SDAG %s
+
+; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -global-isel=1 < %s 2>&1 | FileCheck  -check-prefix=GISEL-ERR %s
+; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -global-isel=0 < %s 2>&1 | FileCheck  -check-prefix=SDAG-ERR %s
+
+; GISEL-ERR: LLVM ERROR: cannot select: {{.*}}  = G_INTRINSIC intrinsic(@llvm.amdgcn.inverse.ballot)
+; SDAG-ERR: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.inverse.ballot
 
 declare i1 @llvm.amdgcn.inverse.ballot.i64(i64)
 

From 7f2bd53b142ec147da9f8bd98775be1d14457ba1 Mon Sep 17 00:00:00 2001
From: Robin Caloudis <robin.caloudis@gmx.de>
Date: Thu, 18 Jul 2024 15:01:09 +0200
Subject: [PATCH 007/486] [libc++] Fix acceptance of
 convertible-to-{float,double,long double} in std::isfinite() (#98841)

Closes https://github.com/llvm/llvm-project/issues/98816.
---
 libcxx/include/__math/traits.h                    | 12 ++++++++++++
 libcxx/test/std/numerics/c.math/isfinite.pass.cpp | 12 ++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/libcxx/include/__math/traits.h b/libcxx/include/__math/traits.h
index a448266797557..27ec52ecef022 100644
--- a/libcxx/include/__math/traits.h
+++ b/libcxx/include/__math/traits.h
@@ -55,6 +55,18 @@ _LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isfin
   return true;
 }
 
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isfinite(float __x) _NOEXCEPT {
+  return __builtin_isfinite(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isfinite(double __x) _NOEXCEPT {
+  return __builtin_isfinite(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isfinite(long double __x) _NOEXCEPT {
+  return __builtin_isfinite(__x);
+}
+
 // isinf
 
 template <class _A1, __enable_if_t<is_arithmetic<_A1>::value && numeric_limits<_A1>::has_infinity, int> = 0>
diff --git a/libcxx/test/std/numerics/c.math/isfinite.pass.cpp b/libcxx/test/std/numerics/c.math/isfinite.pass.cpp
index 6bbc3aaac6d13..3d5be61634334 100644
--- a/libcxx/test/std/numerics/c.math/isfinite.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/isfinite.pass.cpp
@@ -62,9 +62,21 @@ struct TestInt {
   }
 };
 
+template <typename T>
+struct ConvertibleTo {
+  operator T() const { return T(); }
+};
+
 int main(int, char**) {
   types::for_each(types::floating_point_types(), TestFloat());
   types::for_each(types::integral_types(), TestInt());
 
+  // Make sure we can call `std::isfinite` with convertible types
+  {
+    assert(std::isfinite(ConvertibleTo<float>()));
+    assert(std::isfinite(ConvertibleTo<double>()));
+    assert(std::isfinite(ConvertibleTo<long double>()));
+  }
+
   return 0;
 }

From 6a141610f1fc3a53b5b1fd86fa996a90f5c1b849 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 18 Jul 2024 14:02:13 +0100
Subject: [PATCH 008/486] [X86] Add getGFNICtrlMask helper for the constant
 creation and bitcasting. NFC.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b07662b67e3e7..56d08e7f76908 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -29096,6 +29096,16 @@ uint64_t getGFNICtrlImm(unsigned Opcode, unsigned Amt = 0) {
   llvm_unreachable("Unsupported GFNI opcode");
 }
 
+// Generate a GFNI gf2p8affine bitmask for vXi8 bitreverse/shift/rotate.
+SDValue getGFNICtrlMask(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, MVT VT,
+                        unsigned Amt = 0) {
+  assert(VT.getVectorElementType() == MVT::i8 &&
+         (VT.getSizeInBits() % 64) == 0 && "Illegal GFNI control type");
+  uint64_t Imm = getGFNICtrlImm(Opcode, Amt);
+  MVT MaskVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
+  return DAG.getBitcast(VT, DAG.getConstant(Imm, DL, MaskVT));
+}
+
 // Return true if the required (according to Opcode) shift-imm form is natively
 // supported by the Subtarget
 static bool supportedVectorShiftWithImm(EVT VT, const X86Subtarget &Subtarget,
@@ -29284,9 +29294,7 @@ static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG,
       return SDValue();
 
     if (Subtarget.hasGFNI()) {
-      uint64_t ShiftMask = getGFNICtrlImm(Op.getOpcode(), ShiftAmt);
-      MVT MaskVT = MVT::getVectorVT(MVT::i64, NumElts / 8);
-      SDValue Mask = DAG.getBitcast(VT, DAG.getConstant(ShiftMask, dl, MaskVT));
+      SDValue Mask = getGFNICtrlMask(Op.getOpcode(), DAG, dl, VT, ShiftAmt);
       return DAG.getNode(X86ISD::GF2P8AFFINEQB, dl, VT, R, Mask,
                          DAG.getTargetConstant(0, dl, MVT::i8));
     }
@@ -30191,9 +30199,7 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
   if (IsCstSplat && Subtarget.hasGFNI() && VT.getScalarType() == MVT::i8 &&
       DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
     uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
-    uint64_t RotMask = getGFNICtrlImm(Opcode, RotAmt);
-    MVT MaskVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
-    SDValue Mask = DAG.getBitcast(VT, DAG.getConstant(RotMask, DL, MaskVT));
+    SDValue Mask = getGFNICtrlMask(Opcode, DAG, DL, VT, RotAmt);
     return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, R, Mask,
                        DAG.getTargetConstant(0, DL, MVT::i8));
   }
@@ -31528,10 +31534,7 @@ static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
 
   // If we have GFNI, we can use GF2P8AFFINEQB to reverse the bits.
   if (Subtarget.hasGFNI()) {
-    MVT MatrixVT = MVT::getVectorVT(MVT::i64, NumElts / 8);
-    SDValue Matrix =
-        DAG.getConstant(getGFNICtrlImm(ISD::BITREVERSE), DL, MatrixVT);
-    Matrix = DAG.getBitcast(VT, Matrix);
+    SDValue Matrix = getGFNICtrlMask(ISD::BITREVERSE, DAG, DL, VT);
     return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix,
                        DAG.getTargetConstant(0, DL, MVT::i8));
   }

From fde51e24b593087a15494203ef68a77bb4f12115 Mon Sep 17 00:00:00 2001
From: Janet Cobb <jason.e.cobb@gmail.com>
Date: Thu, 18 Jul 2024 09:04:15 -0400
Subject: [PATCH 009/486] [libc++][test] Raise a useful error when no
 -std=c++NN flag is found to work (#99423)

Recently ran into an issue with symptoms very similar to
https://github.com/llvm/llvm-project/issues/56816 while attempting to
build and test libc++ on NixOS. The error message is cryptic (just
`StopIteration`), which was very annoying to track down. The error at
least saying "hey your compiler's bad" would have saved me quite a bit
of time figuring out the issue.
---
 libcxx/utils/libcxx/test/params.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index 3aadc54cf92dc..13c7297fd7304 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -88,6 +88,17 @@ def getStdFlag(cfg, std):
     return None
 
 
+def getDefaultStdValue(cfg):
+    viable = [s for s in reversed(_allStandards) if getStdFlag(cfg, s)]
+
+    if not viable:
+        raise RuntimeError(
+            "Unable to successfully detect the presence of any -std=c++NN flag. This likely indicates an issue with your compiler."
+        )
+
+    return viable[0]
+
+
 def getSpeedOptimizationFlag(cfg):
     if _isClang(cfg) or _isAppleClang(cfg) or _isGCC(cfg):
         return "-O3"
@@ -170,9 +181,7 @@ def getSuitableClangTidy(cfg):
         choices=_allStandards,
         type=str,
         help="The version of the standard to compile the test suite with.",
-        default=lambda cfg: next(
-            s for s in reversed(_allStandards) if getStdFlag(cfg, s)
-        ),
+        default=lambda cfg: getDefaultStdValue(cfg),
         actions=lambda std: [
             AddFeature(std),
             AddSubstitution("%{cxx_std}", re.sub(r"\+", "x", std)),

From 3eb666e292baf87c969be733de858b0cb7ead13f Mon Sep 17 00:00:00 2001
From: Krasimir Georgiev <krasimir@google.com>
Date: Thu, 18 Jul 2024 13:04:59 +0000
Subject: [PATCH 010/486] update bazel for
 a6d2da8b9d7be19816dd4c76b02016c19618c1be

---
 .../llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel     | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
index a6f9d4f2fdac2..4575c40bc76c5 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
@@ -104,16 +104,19 @@ libc_support_library(
     name = "qsort_test_helper",
     hdrs = ["SortingTest.h"],
     deps = [
+        "//libc:__support_macros_config",
         "//libc:qsort_util",
         "//libc/test/UnitTest:LibcUnitTest",
     ],
 )
+
 libc_test(
     name = "qsort_test",
     srcs = ["qsort_test.cpp"],
     libc_function_deps = ["//libc:qsort"],
     deps = [":qsort_test_helper"],
 )
+
 libc_test(
     name = "quick_sort_test",
     srcs = ["quick_sort_test.cpp"],
@@ -122,6 +125,7 @@ libc_test(
         "//libc:qsort_util",
     ],
 )
+
 libc_test(
     name = "heap_sort_test",
     srcs = ["heap_sort_test.cpp"],

From fc65a9603bf16ed1fe98fbee6933bca9e2083384 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Thu, 18 Jul 2024 07:11:00 +0200
Subject: [PATCH 011/486] [clang][Interp] Run record destructors when
 deallocating dynamic memory

---
 clang/lib/AST/Interp/Interp.cpp      | 75 ++++++++++++++++++++++++++++
 clang/lib/AST/Interp/Interp.h        |  6 ++-
 clang/test/AST/Interp/new-delete.cpp | 73 +++++++++++++++++++++++++++
 3 files changed, 153 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index fb63228f8aea8..2be9b5360d055 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -819,6 +819,81 @@ bool CheckNonNullArgs(InterpState &S, CodePtr OpPC, const Function *F,
   return true;
 }
 
+// FIXME: This is similar to code we already have in Compiler.cpp.
+// I think it makes sense to instead add the field and base destruction stuff
+// to the destructor Function itself. Then destroying a record would really
+// _just_ be calling its destructor. That would also help with the diagnostic
+// difference when the destructor or a field/base fails.
+static bool runRecordDestructor(InterpState &S, CodePtr OpPC,
+                                const Pointer &BasePtr,
+                                const Descriptor *Desc) {
+  assert(Desc->isRecord());
+  const Record *R = Desc->ElemRecord;
+  assert(R);
+
+  // Fields.
+  for (const Record::Field &Field : llvm::reverse(R->fields())) {
+    const Descriptor *D = Field.Desc;
+    if (D->isRecord()) {
+      if (!runRecordDestructor(S, OpPC, BasePtr.atField(Field.Offset), D))
+        return false;
+    } else if (D->isCompositeArray()) {
+      const Descriptor *ElemDesc = Desc->ElemDesc;
+      assert(ElemDesc->isRecord());
+      for (unsigned I = 0; I != Desc->getNumElems(); ++I) {
+        if (!runRecordDestructor(S, OpPC, BasePtr.atIndex(I).narrow(),
+                                 ElemDesc))
+          return false;
+      }
+    }
+  }
+
+  // Destructor of this record.
+  if (const CXXDestructorDecl *Dtor = R->getDestructor();
+      Dtor && !Dtor->isTrivial()) {
+    const Function *DtorFunc = S.getContext().getOrCreateFunction(Dtor);
+    if (!DtorFunc)
+      return false;
+
+    S.Stk.push<Pointer>(BasePtr);
+    if (!Call(S, OpPC, DtorFunc, 0))
+      return false;
+  }
+
+  // Bases.
+  for (const Record::Base &Base : llvm::reverse(R->bases())) {
+    if (!runRecordDestructor(S, OpPC, BasePtr.atField(Base.Offset), Base.Desc))
+      return false;
+  }
+
+  return true;
+}
+
+bool RunDestructors(InterpState &S, CodePtr OpPC, const Block *B) {
+  assert(B);
+  const Descriptor *Desc = B->getDescriptor();
+
+  if (Desc->isPrimitive() || Desc->isPrimitiveArray())
+    return true;
+
+  assert(Desc->isRecord() || Desc->isCompositeArray());
+
+  if (Desc->isCompositeArray()) {
+    const Descriptor *ElemDesc = Desc->ElemDesc;
+    assert(ElemDesc->isRecord());
+
+    Pointer RP(const_cast<Block *>(B));
+    for (unsigned I = 0; I != Desc->getNumElems(); ++I) {
+      if (!runRecordDestructor(S, OpPC, RP.atIndex(I).narrow(), ElemDesc))
+        return false;
+    }
+    return true;
+  }
+
+  assert(Desc->isRecord());
+  return runRecordDestructor(S, OpPC, Pointer(const_cast<Block *>(B)), Desc);
+}
+
 bool Interpret(InterpState &S, APValue &Result) {
   // The current stack frame when we started Interpret().
   // This is being used by the ops to determine wheter
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index b4f8c03280c85..17b3157cb40a9 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -2872,8 +2872,8 @@ inline bool AllocCN(InterpState &S, CodePtr OpPC, const Descriptor *ElementDesc,
   return true;
 }
 
+bool RunDestructors(InterpState &S, CodePtr OpPC, const Block *B);
 static inline bool Free(InterpState &S, CodePtr OpPC, bool DeleteIsArrayForm) {
-
   if (!CheckDynamicMemoryAllocation(S, OpPC))
     return false;
 
@@ -2904,6 +2904,10 @@ static inline bool Free(InterpState &S, CodePtr OpPC, bool DeleteIsArrayForm) {
   assert(Source);
   assert(BlockToDelete);
 
+  // Invoke destructors before deallocating the memory.
+  if (!RunDestructors(S, OpPC, BlockToDelete))
+    return false;
+
   DynamicAllocator &Allocator = S.getAllocator();
   bool WasArrayAlloc = Allocator.isArrayAllocation(Source);
   const Descriptor *BlockDesc = BlockToDelete->getDescriptor();
diff --git a/clang/test/AST/Interp/new-delete.cpp b/clang/test/AST/Interp/new-delete.cpp
index 04ce3ae5f6637..cb46426c0e3be 100644
--- a/clang/test/AST/Interp/new-delete.cpp
+++ b/clang/test/AST/Interp/new-delete.cpp
@@ -476,7 +476,80 @@ constexpr Sp ss[] = {Sp{new int{154}}}; // both-error {{must be initialized by a
                                         // both-note {{pointer to heap-allocated object}} \
                                         // both-note {{allocation performed here}}
 
+namespace DeleteRunsDtors {
+  struct InnerFoo {
+    int *mem;
+    constexpr ~InnerFoo() {
+      delete mem;
+    }
+  };
+
+  struct Foo {
+    int *a;
+    InnerFoo IF;
+
+    constexpr Foo() {
+      a = new int(13);
+      IF.mem = new int(100);
+    }
+    constexpr ~Foo() { delete a; }
+  };
+
+  constexpr int abc() {
+    Foo *F = new Foo();
+    int n = *F->a;
+    delete F;
+
+    return n;
+  }
+  static_assert(abc() == 13);
+
+  constexpr int abc2() {
+    Foo *f = new Foo[3];
+
+    delete[] f;
+
+    return 1;
+  }
+  static_assert(abc2() == 1);
+}
+
+/// FIXME: There is a slight difference in diagnostics here, because we don't
+/// create a new frame when we delete record fields or bases at all.
+namespace FaultyDtorCalledByDelete {
+  struct InnerFoo {
+    int *mem;
+    constexpr ~InnerFoo() {
+      if (mem) {
+        (void)(1/0); // both-warning {{division by zero is undefined}} \
+                     // both-note {{division by zero}}
+      }
+      delete mem;
+    }
+  };
+
+  struct Foo {
+    int *a;
+    InnerFoo IF;
 
+    constexpr Foo() {
+      a = new int(13);
+      IF.mem = new int(100);
+    }
+    constexpr ~Foo() { delete a; }
+  };
+
+  constexpr int abc() {
+    Foo *F = new Foo();
+    int n = *F->a;
+    delete F; // both-note {{in call to}} \
+              // ref-note {{in call to}}
+
+    return n;
+  }
+  static_assert(abc() == 13); // both-error {{not an integral constant expression}} \
+                              // both-note {{in call to 'abc()'}}
+}
 
 
 #else

From 4dfa75c663e53be1d548b340e562dd5c4e87fe65 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 18 Jul 2024 15:17:22 +0200
Subject: [PATCH 012/486] [libc++] Merge is_scoped_enum.h into is_enum.h
 (#99458)

---
 libcxx/include/CMakeLists.txt                 |  1 -
 libcxx/include/__type_traits/is_enum.h        | 10 ++++++
 libcxx/include/__type_traits/is_scoped_enum.h | 33 -------------------
 libcxx/include/module.modulemap               |  1 -
 libcxx/include/type_traits                    |  4 ---
 5 files changed, 10 insertions(+), 39 deletions(-)
 delete mode 100644 libcxx/include/__type_traits/is_scoped_enum.h

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index cd64fe91449c2..26bad4f656a07 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -793,7 +793,6 @@ set(files
   __type_traits/is_referenceable.h
   __type_traits/is_same.h
   __type_traits/is_scalar.h
-  __type_traits/is_scoped_enum.h
   __type_traits/is_signed.h
   __type_traits/is_signed_integer.h
   __type_traits/is_specialization.h
diff --git a/libcxx/include/__type_traits/is_enum.h b/libcxx/include/__type_traits/is_enum.h
index 77ca3ea108742..2fab6db2c8d50 100644
--- a/libcxx/include/__type_traits/is_enum.h
+++ b/libcxx/include/__type_traits/is_enum.h
@@ -26,6 +26,16 @@ template <class _Tp>
 inline constexpr bool is_enum_v = __is_enum(_Tp);
 #endif
 
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_scoped_enum : bool_constant<__is_scoped_enum(_Tp)> {};
+
+template <class _Tp>
+inline constexpr bool is_scoped_enum_v = __is_scoped_enum(_Tp);
+
+#endif // _LIBCPP_STD_VER >= 23
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___TYPE_TRAITS_IS_ENUM_H
diff --git a/libcxx/include/__type_traits/is_scoped_enum.h b/libcxx/include/__type_traits/is_scoped_enum.h
deleted file mode 100644
index cb3e25cf57331..0000000000000
--- a/libcxx/include/__type_traits/is_scoped_enum.h
+++ /dev/null
@@ -1,33 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP___TYPE_TRAITS_IS_SCOPED_ENUM_H
-#define _LIBCPP___TYPE_TRAITS_IS_SCOPED_ENUM_H
-
-#include <__config>
-#include <__type_traits/integral_constant.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-#if _LIBCPP_STD_VER >= 23
-
-template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_scoped_enum : bool_constant<__is_scoped_enum(_Tp)> {};
-
-template <class _Tp>
-inline constexpr bool is_scoped_enum_v = __is_scoped_enum(_Tp);
-
-#endif // _LIBCPP_STD_VER >= 23
-
-_LIBCPP_END_NAMESPACE_STD
-
-#endif // _LIBCPP___TYPE_TRAITS_IS_SCOPED_ENUM_H
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 7608aef3f3a43..5ed284d80f35e 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -2009,7 +2009,6 @@ module std_private_type_traits_is_scalar                                 [system
   header "__type_traits/is_scalar.h"
   export std_private_type_traits_is_null_pointer
 }
-module std_private_type_traits_is_scoped_enum                            [system] { header "__type_traits/is_scoped_enum.h" }
 module std_private_type_traits_is_signed                                 [system] { header "__type_traits/is_signed.h" }
 module std_private_type_traits_is_signed_integer                         [system] { header "__type_traits/is_signed_integer.h" }
 module std_private_type_traits_is_specialization                         [system] { header "__type_traits/is_specialization.h" }
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index ffa137338b6a2..7f231cd09df51 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -514,10 +514,6 @@ namespace std
 #  include <__type_traits/unwrap_ref.h>
 #endif
 
-#if _LIBCPP_STD_VER >= 23
-#  include <__type_traits/is_scoped_enum.h>
-#endif
-
 #include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)

From 561246e90282a72b5b0c437cbbdae171526aad8f Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 18 Jul 2024 12:21:31 +0200
Subject: [PATCH 013/486] [libc++][NFC] Remove wrong #endif comment

---
 libcxx/include/__type_traits/remove_cv.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/include/__type_traits/remove_cv.h b/libcxx/include/__type_traits/remove_cv.h
index 2c4e9e419a1be..50e9f3e8aa78d 100644
--- a/libcxx/include/__type_traits/remove_cv.h
+++ b/libcxx/include/__type_traits/remove_cv.h
@@ -28,7 +28,7 @@ using __remove_cv_t = typename remove_cv<_Tp>::type;
 #else
 template <class _Tp>
 using __remove_cv_t = __remove_cv(_Tp);
-#endif // __has_builtin(__remove_cv)
+#endif
 
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>

From 15495b8cd4051d05c1b88c919e7c509a8ea4056a Mon Sep 17 00:00:00 2001
From: Johannes Reifferscheid <jreiffers@google.com>
Date: Thu, 18 Jul 2024 15:20:57 +0200
Subject: [PATCH 014/486] [mlir] Fix unused-variable warning w/o assertions.
 (#99489)

---
 mlir/test/CAPI/rewrite.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mlir/test/CAPI/rewrite.c b/mlir/test/CAPI/rewrite.c
index a8b593eabb781..b33d225767046 100644
--- a/mlir/test/CAPI/rewrite.c
+++ b/mlir/test/CAPI/rewrite.c
@@ -68,6 +68,8 @@ void testInsertionPoint(MlirContext ctx) {
   // Get insertion blocks
   MlirBlock block1 = mlirRewriterBaseGetBlock(rewriter);
   MlirBlock block2 = mlirRewriterBaseGetInsertionBlock(rewriter);
+  (void)block1;
+  (void)block2;
   assert(body.ptr == block1.ptr);
   assert(body.ptr == block2.ptr);
 

From d00b35534d068510025d22e5bd9c4fdac45757fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Thu, 18 Jul 2024 09:01:42 +0200
Subject: [PATCH 015/486] [clang][Interp] Fix CheckCallable for
 undefined-and-not-constexpr fns

---
 clang/lib/AST/Interp/Interp.cpp | 86 +++++++++++++++++----------------
 clang/lib/AST/Interp/Interp.h   |  6 +--
 clang/test/AST/Interp/cxx2a.cpp | 15 ++++++
 3 files changed, 63 insertions(+), 44 deletions(-)
 create mode 100644 clang/test/AST/Interp/cxx2a.cpp

diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index 2be9b5360d055..be47f72e65a29 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -579,57 +579,61 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) {
     return false;
   }
 
-  if (!F->isConstexpr() || !F->hasBody()) {
-    const SourceLocation &Loc = S.Current->getLocation(OpPC);
-    if (S.getLangOpts().CPlusPlus11) {
-      const FunctionDecl *DiagDecl = F->getDecl();
+  if (F->isConstexpr() && F->hasBody() && F->getDecl()->isConstexpr())
+    return true;
 
-      // Invalid decls have been diagnosed before.
-      if (DiagDecl->isInvalidDecl())
-        return false;
+  // Implicitly constexpr.
+  if (F->isLambdaStaticInvoker())
+    return true;
 
-      // If this function is not constexpr because it is an inherited
-      // non-constexpr constructor, diagnose that directly.
-      const auto *CD = dyn_cast<CXXConstructorDecl>(DiagDecl);
-      if (CD && CD->isInheritingConstructor()) {
-        const auto *Inherited = CD->getInheritedConstructor().getConstructor();
-        if (!Inherited->isConstexpr())
-          DiagDecl = CD = Inherited;
-      }
+  const SourceLocation &Loc = S.Current->getLocation(OpPC);
+  if (S.getLangOpts().CPlusPlus11) {
+    const FunctionDecl *DiagDecl = F->getDecl();
+
+    // Invalid decls have been diagnosed before.
+    if (DiagDecl->isInvalidDecl())
+      return false;
+
+    // If this function is not constexpr because it is an inherited
+    // non-constexpr constructor, diagnose that directly.
+    const auto *CD = dyn_cast<CXXConstructorDecl>(DiagDecl);
+    if (CD && CD->isInheritingConstructor()) {
+      const auto *Inherited = CD->getInheritedConstructor().getConstructor();
+      if (!Inherited->isConstexpr())
+        DiagDecl = CD = Inherited;
+    }
 
-      // FIXME: If DiagDecl is an implicitly-declared special member function
-      // or an inheriting constructor, we should be much more explicit about why
-      // it's not constexpr.
-      if (CD && CD->isInheritingConstructor()) {
-        S.FFDiag(Loc, diag::note_constexpr_invalid_inhctor, 1)
+    // FIXME: If DiagDecl is an implicitly-declared special member function
+    // or an inheriting constructor, we should be much more explicit about why
+    // it's not constexpr.
+    if (CD && CD->isInheritingConstructor()) {
+      S.FFDiag(Loc, diag::note_constexpr_invalid_inhctor, 1)
           << CD->getInheritedConstructor().getConstructor()->getParent();
-        S.Note(DiagDecl->getLocation(), diag::note_declared_at);
-      } else {
-        // Don't emit anything if the function isn't defined and we're checking
-        // for a constant expression. It might be defined at the point we're
-        // actually calling it.
-        bool IsExtern = DiagDecl->getStorageClass() == SC_Extern;
-        if (!DiagDecl->isDefined() && !IsExtern &&
-            S.checkingPotentialConstantExpression())
-          return false;
+      S.Note(DiagDecl->getLocation(), diag::note_declared_at);
+    } else {
+      // Don't emit anything if the function isn't defined and we're checking
+      // for a constant expression. It might be defined at the point we're
+      // actually calling it.
+      bool IsExtern = DiagDecl->getStorageClass() == SC_Extern;
+      if (!DiagDecl->isDefined() && !IsExtern && DiagDecl->isConstexpr() &&
+          S.checkingPotentialConstantExpression())
+        return false;
 
-        // If the declaration is defined, declared 'constexpr' _and_ has a body,
-        // the below diagnostic doesn't add anything useful.
-        if (DiagDecl->isDefined() && DiagDecl->isConstexpr() &&
-            DiagDecl->hasBody())
-          return false;
+      // If the declaration is defined, declared 'constexpr' _and_ has a body,
+      // the below diagnostic doesn't add anything useful.
+      if (DiagDecl->isDefined() && DiagDecl->isConstexpr() &&
+          DiagDecl->hasBody())
+        return false;
 
-        S.FFDiag(Loc, diag::note_constexpr_invalid_function, 1)
+      S.FFDiag(Loc, diag::note_constexpr_invalid_function, 1)
           << DiagDecl->isConstexpr() << (bool)CD << DiagDecl;
-        S.Note(DiagDecl->getLocation(), diag::note_declared_at);
-      }
-    } else {
-      S.FFDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
+      S.Note(DiagDecl->getLocation(), diag::note_declared_at);
     }
-    return false;
+  } else {
+    S.FFDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
   }
 
-  return true;
+  return false;
 }
 
 bool CheckCallDepth(InterpState &S, CodePtr OpPC) {
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index 17b3157cb40a9..2e159012f5ffd 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -2531,14 +2531,14 @@ inline bool Call(InterpState &S, CodePtr OpPC, const Function *Func,
       if (!CheckInvoke(S, OpPC, ThisPtr))
         return false;
     }
-
-    if (S.checkingPotentialConstantExpression())
-      return false;
   }
 
   if (!CheckCallable(S, OpPC, Func))
     return false;
 
+  if (Func->hasThisPointer() && S.checkingPotentialConstantExpression())
+    return false;
+
   if (!CheckCallDepth(S, OpPC))
     return false;
 
diff --git a/clang/test/AST/Interp/cxx2a.cpp b/clang/test/AST/Interp/cxx2a.cpp
new file mode 100644
index 0000000000000..27d1aa1a27f75
--- /dev/null
+++ b/clang/test/AST/Interp/cxx2a.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -std=c++2a -fsyntax-only -fcxx-exceptions -verify=ref,both %s
+// RUN: %clang_cc1 -std=c++2a -fsyntax-only -fcxx-exceptions -verify=expected,both %s -fexperimental-new-constant-interpreter
+
+template <unsigned N>
+struct S {
+  S() requires (N==1) = default;
+  S() requires (N==2) {} // both-note {{declared here}}
+  consteval S() requires (N==3) = default;
+};
+
+consteval int aConstevalFunction() { // both-error {{consteval function never produces a constant expression}}
+  S<2> s4; // both-note {{non-constexpr constructor 'S' cannot be used in a constant expression}}
+  return 0;
+}
+/// We're NOT calling the above function. The diagnostics should appear anyway.

From d9cb65ff483a2f79d0d3f0239796abe829372e52 Mon Sep 17 00:00:00 2001
From: Romaric Jodin <rjodin@chromium.org>
Date: Thu, 18 Jul 2024 15:28:58 +0200
Subject: [PATCH 016/486] libclc: fix convert with half (#99481)

Fix following update of libclc introducing more fp16 support:
https://github.com/llvm/llvm-project/commit/7e6a73959ae97b1f9476a90290a492ba90cb950d
---
 libclc/generic/include/clc/convert.h | 1 +
 libclc/generic/lib/gen_convert.py    | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/libclc/generic/include/clc/convert.h b/libclc/generic/include/clc/convert.h
index db7bb0402491e..8219df47ad2c6 100644
--- a/libclc/generic/include/clc/convert.h
+++ b/libclc/generic/include/clc/convert.h
@@ -62,6 +62,7 @@
 #define _CLC_VECTOR_CONVERT_TO(SUFFIX)                                         \
   _CLC_VECTOR_CONVERT_TO1(SUFFIX)                                              \
   _CLC_VECTOR_CONVERT_FROM(half, SUFFIX)
+#else
 #define _CLC_VECTOR_CONVERT_TO(SUFFIX) \
   _CLC_VECTOR_CONVERT_TO1(SUFFIX)
 #endif
diff --git a/libclc/generic/lib/gen_convert.py b/libclc/generic/lib/gen_convert.py
index bd36faa4e9197..41bd8cebf88b6 100644
--- a/libclc/generic/lib/gen_convert.py
+++ b/libclc/generic/lib/gen_convert.py
@@ -142,7 +142,10 @@ def conditional_guard(src, dst):
         float64_count = float64_count + 1
     elif dst in float16_types:
         float16_count = float16_count + 1
-    if float64_count > 0:
+    if float64_count > 0 and float16_count > 0:
+        print("#if defined(cl_khr_fp16) && defined(cl_khr_fp64)")
+        return True
+    elif float64_count > 0:
         # In embedded profile, if cl_khr_fp64 is supported cles_khr_int64 has to be
         print("#ifdef cl_khr_fp64")
         return True

From ad7aeb0ff58ebd29f68adb85c64e8010639e2a76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Thu, 18 Jul 2024 15:29:45 +0200
Subject: [PATCH 017/486] Revert "[clang][Interp] Fix CheckCallable for
 undefined-and-not-constexpr fns"

This reverts commit d00b35534d068510025d22e5bd9c4fdac45757fb.

This breaks the ms-constexpr test:
https://lab.llvm.org/buildbot/#/builders/144/builds/2605
---
 clang/lib/AST/Interp/Interp.cpp | 86 ++++++++++++++++-----------------
 clang/lib/AST/Interp/Interp.h   |  6 +--
 clang/test/AST/Interp/cxx2a.cpp | 15 ------
 3 files changed, 44 insertions(+), 63 deletions(-)
 delete mode 100644 clang/test/AST/Interp/cxx2a.cpp

diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index be47f72e65a29..2be9b5360d055 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -579,61 +579,57 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) {
     return false;
   }
 
-  if (F->isConstexpr() && F->hasBody() && F->getDecl()->isConstexpr())
-    return true;
-
-  // Implicitly constexpr.
-  if (F->isLambdaStaticInvoker())
-    return true;
-
-  const SourceLocation &Loc = S.Current->getLocation(OpPC);
-  if (S.getLangOpts().CPlusPlus11) {
-    const FunctionDecl *DiagDecl = F->getDecl();
+  if (!F->isConstexpr() || !F->hasBody()) {
+    const SourceLocation &Loc = S.Current->getLocation(OpPC);
+    if (S.getLangOpts().CPlusPlus11) {
+      const FunctionDecl *DiagDecl = F->getDecl();
 
-    // Invalid decls have been diagnosed before.
-    if (DiagDecl->isInvalidDecl())
-      return false;
+      // Invalid decls have been diagnosed before.
+      if (DiagDecl->isInvalidDecl())
+        return false;
 
-    // If this function is not constexpr because it is an inherited
-    // non-constexpr constructor, diagnose that directly.
-    const auto *CD = dyn_cast<CXXConstructorDecl>(DiagDecl);
-    if (CD && CD->isInheritingConstructor()) {
-      const auto *Inherited = CD->getInheritedConstructor().getConstructor();
-      if (!Inherited->isConstexpr())
-        DiagDecl = CD = Inherited;
-    }
+      // If this function is not constexpr because it is an inherited
+      // non-constexpr constructor, diagnose that directly.
+      const auto *CD = dyn_cast<CXXConstructorDecl>(DiagDecl);
+      if (CD && CD->isInheritingConstructor()) {
+        const auto *Inherited = CD->getInheritedConstructor().getConstructor();
+        if (!Inherited->isConstexpr())
+          DiagDecl = CD = Inherited;
+      }
 
-    // FIXME: If DiagDecl is an implicitly-declared special member function
-    // or an inheriting constructor, we should be much more explicit about why
-    // it's not constexpr.
-    if (CD && CD->isInheritingConstructor()) {
-      S.FFDiag(Loc, diag::note_constexpr_invalid_inhctor, 1)
+      // FIXME: If DiagDecl is an implicitly-declared special member function
+      // or an inheriting constructor, we should be much more explicit about why
+      // it's not constexpr.
+      if (CD && CD->isInheritingConstructor()) {
+        S.FFDiag(Loc, diag::note_constexpr_invalid_inhctor, 1)
           << CD->getInheritedConstructor().getConstructor()->getParent();
-      S.Note(DiagDecl->getLocation(), diag::note_declared_at);
-    } else {
-      // Don't emit anything if the function isn't defined and we're checking
-      // for a constant expression. It might be defined at the point we're
-      // actually calling it.
-      bool IsExtern = DiagDecl->getStorageClass() == SC_Extern;
-      if (!DiagDecl->isDefined() && !IsExtern && DiagDecl->isConstexpr() &&
-          S.checkingPotentialConstantExpression())
-        return false;
+        S.Note(DiagDecl->getLocation(), diag::note_declared_at);
+      } else {
+        // Don't emit anything if the function isn't defined and we're checking
+        // for a constant expression. It might be defined at the point we're
+        // actually calling it.
+        bool IsExtern = DiagDecl->getStorageClass() == SC_Extern;
+        if (!DiagDecl->isDefined() && !IsExtern &&
+            S.checkingPotentialConstantExpression())
+          return false;
 
-      // If the declaration is defined, declared 'constexpr' _and_ has a body,
-      // the below diagnostic doesn't add anything useful.
-      if (DiagDecl->isDefined() && DiagDecl->isConstexpr() &&
-          DiagDecl->hasBody())
-        return false;
+        // If the declaration is defined, declared 'constexpr' _and_ has a body,
+        // the below diagnostic doesn't add anything useful.
+        if (DiagDecl->isDefined() && DiagDecl->isConstexpr() &&
+            DiagDecl->hasBody())
+          return false;
 
-      S.FFDiag(Loc, diag::note_constexpr_invalid_function, 1)
+        S.FFDiag(Loc, diag::note_constexpr_invalid_function, 1)
           << DiagDecl->isConstexpr() << (bool)CD << DiagDecl;
-      S.Note(DiagDecl->getLocation(), diag::note_declared_at);
+        S.Note(DiagDecl->getLocation(), diag::note_declared_at);
+      }
+    } else {
+      S.FFDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
     }
-  } else {
-    S.FFDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
+    return false;
   }
 
-  return false;
+  return true;
 }
 
 bool CheckCallDepth(InterpState &S, CodePtr OpPC) {
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index 2e159012f5ffd..17b3157cb40a9 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -2531,14 +2531,14 @@ inline bool Call(InterpState &S, CodePtr OpPC, const Function *Func,
       if (!CheckInvoke(S, OpPC, ThisPtr))
         return false;
     }
+
+    if (S.checkingPotentialConstantExpression())
+      return false;
   }
 
   if (!CheckCallable(S, OpPC, Func))
     return false;
 
-  if (Func->hasThisPointer() && S.checkingPotentialConstantExpression())
-    return false;
-
   if (!CheckCallDepth(S, OpPC))
     return false;
 
diff --git a/clang/test/AST/Interp/cxx2a.cpp b/clang/test/AST/Interp/cxx2a.cpp
deleted file mode 100644
index 27d1aa1a27f75..0000000000000
--- a/clang/test/AST/Interp/cxx2a.cpp
+++ /dev/null
@@ -1,15 +0,0 @@
-// RUN: %clang_cc1 -std=c++2a -fsyntax-only -fcxx-exceptions -verify=ref,both %s
-// RUN: %clang_cc1 -std=c++2a -fsyntax-only -fcxx-exceptions -verify=expected,both %s -fexperimental-new-constant-interpreter
-
-template <unsigned N>
-struct S {
-  S() requires (N==1) = default;
-  S() requires (N==2) {} // both-note {{declared here}}
-  consteval S() requires (N==3) = default;
-};
-
-consteval int aConstevalFunction() { // both-error {{consteval function never produces a constant expression}}
-  S<2> s4; // both-note {{non-constexpr constructor 'S' cannot be used in a constant expression}}
-  return 0;
-}
-/// We're NOT calling the above function. The diagnostics should appear anyway.

From a778909168746e266ad52b817a758328cdd28311 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Thu, 18 Jul 2024 13:31:47 +0000
Subject: [PATCH 018/486] [gn build] Port 4dfa75c663e5

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index b976e9745fbef..0349e636f8009 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -863,7 +863,6 @@ if (current_toolchain == default_toolchain) {
       "__type_traits/is_referenceable.h",
       "__type_traits/is_same.h",
       "__type_traits/is_scalar.h",
-      "__type_traits/is_scoped_enum.h",
       "__type_traits/is_signed.h",
       "__type_traits/is_signed_integer.h",
       "__type_traits/is_specialization.h",

From 684a61506a3ddc943b8baef1d14c96bbf82e6c04 Mon Sep 17 00:00:00 2001
From: "A. Jiang" <de34@live.cn>
Date: Thu, 18 Jul 2024 21:58:35 +0800
Subject: [PATCH 019/486] [libc++][chrono] Remove non-standard relational
 operators for `std::chrono::weekday` (#98730)

These operators are absent in https://eel.is/c++draft/time.syn and a note in
https://eel.is/c++draft/time.cal.wd.overview#1 indicates that the absence is
intended.

This patch removes the undocumented extension, while providing a migration path
for vendors by providing the `_LIBCPP_ENABLE_REMOVED_WEEKDAY_RELATIONAL_OPERATORS`
macro. This macro will be honored for the LLVM 19 release and will be removed after
that, at which point allocator will be removed unconditionally.
---
 libcxx/docs/ReleaseNotes/19.rst                        | 10 ++++++++++
 libcxx/docs/ReleaseNotes/20.rst                        |  4 ++++
 libcxx/include/__chrono/weekday.h                      |  3 +++
 .../time.cal.weekday.nonmembers/comparisons.pass.cpp   |  3 +++
 4 files changed, 20 insertions(+)

diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index a28dae52e8579..36cb23dfde6c9 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -143,6 +143,12 @@ Deprecations and Removals
   of randomness, and others. Users that were checking whether including a header would fail (e.g. via a script
   or CMake's ``try_compile`` will experience a change in behavior).
 
+- libc++ no longer supports relational comparison for ``std::chrono::weekday``. The relational comparison operators were
+  provided as an undocumented extension. If you were using relational comparison on ``std::chrono::weekday``, compare
+  the results of ``c_encoding()`` or ``iso_encoding()`` instead. The
+  ``_LIBCPP_ENABLE_REMOVED_WEEKDAY_RELATIONAL_OPERATORS`` macro can be defined to temporarily re-enable this extension.
+  This macro will be honored for one release and ignored starting in LLVM 20.
+
 - The operators in the ``rel_ops`` namespace have been deprecated. The deprecation is part of the paper
   P0768R1 "Library Support for the Spaceship (Comparison) Operator".
 
@@ -157,6 +163,10 @@ LLVM 20
 
 - The C++20 synchronization library will be removed entirely in language modes prior to C++20 in LLVM 20.
 
+- The relational operators for ``std::chrono::weekday`` will be removed entirely, and the
+  ``_LIBCPP_ENABLE_REMOVED_WEEKDAY_RELATIONAL_OPERATORS`` macro that was used to re-enable this extension will be
+  ignored in LLVM 20.
+
 LLVM 21
 ~~~~~~~
 TODO
diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index 79b9788f92eda..fb677b1667ddc 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -55,6 +55,10 @@ Deprecations and Removals
 
 - TODO: The C++20 synchronization library will be removed entirely in language modes prior to C++20 in LLVM 20.
 
+- TODO: The relational operators for ``std::chrono::weekday`` will be removed entirely, and the
+  ``_LIBCPP_ENABLE_REMOVED_WEEKDAY_RELATIONAL_OPERATORS`` macro that was used to re-enable this extension will be
+  ignored in LLVM 20.
+
 
 Upcoming Deprecations and Removals
 ----------------------------------
diff --git a/libcxx/include/__chrono/weekday.h b/libcxx/include/__chrono/weekday.h
index 5a7dedc6e3a16..86c780cc71825 100644
--- a/libcxx/include/__chrono/weekday.h
+++ b/libcxx/include/__chrono/weekday.h
@@ -79,6 +79,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const weekday& __lhs, con
   return __lhs.c_encoding() == __rhs.c_encoding();
 }
 
+// TODO(LLVM 20): Remove the escape hatch
+#  ifdef _LIBCPP_ENABLE_REMOVED_WEEKDAY_RELATIONAL_OPERATORS
 _LIBCPP_HIDE_FROM_ABI inline constexpr bool operator<(const weekday& __lhs, const weekday& __rhs) noexcept {
   return __lhs.c_encoding() < __rhs.c_encoding();
 }
@@ -94,6 +96,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr bool operator<=(const weekday& __lhs, con
 _LIBCPP_HIDE_FROM_ABI inline constexpr bool operator>=(const weekday& __lhs, const weekday& __rhs) noexcept {
   return !(__lhs < __rhs);
 }
+#  endif // _LIBCPP_ENABLE_REMOVED_WEEKDAY_RELATIONAL_OPERATORS
 
 _LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator+(const weekday& __lhs, const days& __rhs) noexcept {
   auto const __mu = static_cast<long long>(__lhs.c_encoding()) + __rhs.count();
diff --git a/libcxx/test/std/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/comparisons.pass.cpp b/libcxx/test/std/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/comparisons.pass.cpp
index 33d8cd9f776d2..0101d205b5555 100644
--- a/libcxx/test/std/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/comparisons.pass.cpp
+++ b/libcxx/test/std/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/comparisons.pass.cpp
@@ -17,10 +17,13 @@
 #include <chrono>
 #include <type_traits>
 #include <cassert>
+#include <concepts>
 
 #include "test_macros.h"
 #include "test_comparisons.h"
 
+static_assert(!std::totally_ordered<std::chrono::weekday>);
+
 int main(int, char**)
 {
     using weekday = std::chrono::weekday;

From 078198f310d55925ccd9e1aa5b6ff4af3b36bbc7 Mon Sep 17 00:00:00 2001
From: Jeremy Morse <jeremy.morse@sony.com>
Date: Thu, 18 Jul 2024 15:04:02 +0100
Subject: [PATCH 020/486] [DebugInfo][InstrRef] Index DebugVariables and some
 DILocations (#99318)

A lot of time in LiveDebugValues is spent computing DenseMap keys for
DebugVariables, and they're made up of three pointers, so are large.
This patch installs an index for them: for the SSA and value-to-location
mapping parts of InstrRefBasedLDV we don't need to access things like
the variable declaration or the inlining site, so just use a uint32_t
identifier for each variable fragment that's tracked. The compile-time
performance improvements are substantial (almost 0.4% on the tracker).

About 80% of this patch is just replacing DebugVariable references with
DebugVariableIDs instead, however there are some larger consequences. We
spend lots of time fetching DILocations when emitting DBG_VALUE
instructions, so index those with the DebugVariables: this means all
DILocations on all new DBG_VALUE instructions will normalise to the
first-seen DILocation for the variable (which should be fine).

We also used to keep an ordering of when each variable was seen first in
a DBG_* instruction, in the AllVarsNumbering collection, so that we can
emit new DBG_* instructions in a stable order. We can hang this off the
DebugVariable index instead, so AllVarsNumbering is deleted.

Finally, rather than ordering by AllVarsNumbering just before DBG_*
instructions are linked into the output MIR, store instructions along
with their DebugVariableID, so that they can be sorted by that instead.
---
 .../LiveDebugValues/InstrRefBasedImpl.cpp     | 201 +++++++++---------
 .../LiveDebugValues/InstrRefBasedImpl.h       | 106 ++++++---
 .../MIR/X86/live-debug-values-fragments.mir   |   4 +-
 llvm/unittests/CodeGen/InstrRefLDVTest.cpp    |   3 +-
 4 files changed, 186 insertions(+), 128 deletions(-)

diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
index 247258a1ff553..b9cf36a07846c 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
@@ -183,6 +183,7 @@ class TransferTracker {
   /// information from it. (XXX make it const?)
   MLocTracker *MTracker;
   MachineFunction &MF;
+  const DebugVariableMap &DVMap;
   bool ShouldEmitDebugEntryValues;
 
   /// Record of all changes in variable locations at a block position. Awkwardly
@@ -191,7 +192,9 @@ class TransferTracker {
   struct Transfer {
     MachineBasicBlock::instr_iterator Pos; /// Position to insert DBG_VALUes
     MachineBasicBlock *MBB; /// non-null if we should insert after.
-    SmallVector<MachineInstr *, 4> Insts; /// Vector of DBG_VALUEs to insert.
+    /// Vector of DBG_VALUEs to insert. Store with their DebugVariableID so that
+    /// they can be sorted into a stable order for emission at a later time.
+    SmallVector<std::pair<DebugVariableID, MachineInstr *>, 4> Insts;
   };
 
   /// Stores the resolved operands (machine locations and constants) and
@@ -227,15 +230,15 @@ class TransferTracker {
   /// Map from LocIdxes to which DebugVariables are based that location.
   /// Mantained while stepping through the block. Not accurate if
   /// VarLocs[Idx] != MTracker->LocIdxToIDNum[Idx].
-  DenseMap<LocIdx, SmallSet<DebugVariable, 4>> ActiveMLocs;
+  DenseMap<LocIdx, SmallSet<DebugVariableID, 4>> ActiveMLocs;
 
   /// Map from DebugVariable to it's current location and qualifying meta
   /// information. To be used in conjunction with ActiveMLocs to construct
   /// enough information for the DBG_VALUEs for a particular LocIdx.
-  DenseMap<DebugVariable, ResolvedDbgValue> ActiveVLocs;
+  DenseMap<DebugVariableID, ResolvedDbgValue> ActiveVLocs;
 
   /// Temporary cache of DBG_VALUEs to be entered into the Transfers collection.
-  SmallVector<MachineInstr *, 4> PendingDbgValues;
+  SmallVector<std::pair<DebugVariableID, MachineInstr *>, 4> PendingDbgValues;
 
   /// Record of a use-before-def: created when a value that's live-in to the
   /// current block isn't available in any machine location, but it will be
@@ -244,12 +247,12 @@ class TransferTracker {
     /// Value of this variable, def'd in block.
     SmallVector<DbgOp> Values;
     /// Identity of this variable.
-    DebugVariable Var;
+    DebugVariableID VarID;
     /// Additional variable properties.
     DbgValueProperties Properties;
-    UseBeforeDef(ArrayRef<DbgOp> Values, const DebugVariable &Var,
+    UseBeforeDef(ArrayRef<DbgOp> Values, DebugVariableID VarID,
                  const DbgValueProperties &Properties)
-        : Values(Values.begin(), Values.end()), Var(Var),
+        : Values(Values.begin(), Values.end()), VarID(VarID),
           Properties(Properties) {}
   };
 
@@ -260,15 +263,16 @@ class TransferTracker {
   /// The set of variables that are in UseBeforeDefs and can become a location
   /// once the relevant value is defined. An element being erased from this
   /// collection prevents the use-before-def materializing.
-  DenseSet<DebugVariable> UseBeforeDefVariables;
+  DenseSet<DebugVariableID> UseBeforeDefVariables;
 
   const TargetRegisterInfo &TRI;
   const BitVector &CalleeSavedRegs;
 
   TransferTracker(const TargetInstrInfo *TII, MLocTracker *MTracker,
-                  MachineFunction &MF, const TargetRegisterInfo &TRI,
+                  MachineFunction &MF, const DebugVariableMap &DVMap,
+                  const TargetRegisterInfo &TRI,
                   const BitVector &CalleeSavedRegs, const TargetPassConfig &TPC)
-      : TII(TII), MTracker(MTracker), MF(MF), TRI(TRI),
+      : TII(TII), MTracker(MTracker), MF(MF), DVMap(DVMap), TRI(TRI),
         CalleeSavedRegs(CalleeSavedRegs) {
     TLI = MF.getSubtarget().getTargetLowering();
     auto &TM = TPC.getTM<TargetMachine>();
@@ -352,7 +356,7 @@ class TransferTracker {
   ///    determine the values used by Value.
   void loadVarInloc(MachineBasicBlock &MBB, DbgOpIDMap &DbgOpStore,
                     const SmallVectorImpl<ValueLocPair> &ValueToLoc,
-                    DebugVariable Var, DbgValue Value) {
+                    DebugVariableID VarID, DbgValue Value) {
     SmallVector<DbgOp> DbgOps;
     SmallVector<ResolvedDbgOp> ResolvedDbgOps;
     bool IsValueValid = true;
@@ -401,7 +405,7 @@ class TransferTracker {
                                       static_cast<unsigned>(Num.getInst()));
           continue;
         }
-        recoverAsEntryValue(Var, Value.Properties, Num);
+        recoverAsEntryValue(VarID, Value.Properties, Num);
         IsValueValid = false;
         break;
       }
@@ -419,8 +423,7 @@ class TransferTracker {
 
     // Add UseBeforeDef entry for the last value to be defined in this block.
     if (LastUseBeforeDef) {
-      addUseBeforeDef(Var, Value.Properties, DbgOps,
-                      LastUseBeforeDef);
+      addUseBeforeDef(VarID, Value.Properties, DbgOps, LastUseBeforeDef);
       return;
     }
 
@@ -428,13 +431,15 @@ class TransferTracker {
     // the transfer.
     for (const ResolvedDbgOp &Op : ResolvedDbgOps)
       if (!Op.IsConst)
-        ActiveMLocs[Op.Loc].insert(Var);
+        ActiveMLocs[Op.Loc].insert(VarID);
     auto NewValue = ResolvedDbgValue{ResolvedDbgOps, Value.Properties};
-    auto Result = ActiveVLocs.insert(std::make_pair(Var, NewValue));
+    auto Result = ActiveVLocs.insert(std::make_pair(VarID, NewValue));
     if (!Result.second)
       Result.first->second = NewValue;
+    auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
     PendingDbgValues.push_back(
-        MTracker->emitLoc(ResolvedDbgOps, Var, Value.Properties));
+        std::make_pair(VarID, &*MTracker->emitLoc(ResolvedDbgOps, Var, DILoc,
+                                                  Value.Properties)));
   }
 
   /// Load object with live-in variable values. \p mlocs contains the live-in
@@ -445,7 +450,7 @@ class TransferTracker {
   /// FIXME: could just examine mloctracker instead of passing in \p mlocs?
   void
   loadInlocs(MachineBasicBlock &MBB, ValueTable &MLocs, DbgOpIDMap &DbgOpStore,
-             const SmallVectorImpl<std::pair<DebugVariable, DbgValue>> &VLocs,
+             const SmallVectorImpl<std::pair<DebugVariableID, DbgValue>> &VLocs,
              unsigned NumLocs) {
     ActiveMLocs.clear();
     ActiveVLocs.clear();
@@ -506,11 +511,11 @@ class TransferTracker {
 
   /// Record that \p Var has value \p ID, a value that becomes available
   /// later in the function.
-  void addUseBeforeDef(const DebugVariable &Var,
+  void addUseBeforeDef(DebugVariableID VarID,
                        const DbgValueProperties &Properties,
                        const SmallVectorImpl<DbgOp> &DbgOps, unsigned Inst) {
-    UseBeforeDefs[Inst].emplace_back(DbgOps, Var, Properties);
-    UseBeforeDefVariables.insert(Var);
+    UseBeforeDefs[Inst].emplace_back(DbgOps, VarID, Properties);
+    UseBeforeDefVariables.insert(VarID);
   }
 
   /// After the instruction at index \p Inst and position \p pos has been
@@ -529,7 +534,7 @@ class TransferTracker {
     // Populate ValueToLoc with illegal default mappings for every value used by
     // any UseBeforeDef variables for this instruction.
     for (auto &Use : MIt->second) {
-      if (!UseBeforeDefVariables.count(Use.Var))
+      if (!UseBeforeDefVariables.count(Use.VarID))
         continue;
 
       for (DbgOp &Op : Use.Values) {
@@ -568,7 +573,7 @@ class TransferTracker {
     // Using the map of values to locations, produce a final set of values for
     // this variable.
     for (auto &Use : MIt->second) {
-      if (!UseBeforeDefVariables.count(Use.Var))
+      if (!UseBeforeDefVariables.count(Use.VarID))
         continue;
 
       SmallVector<ResolvedDbgOp> DbgOps;
@@ -591,8 +596,9 @@ class TransferTracker {
         continue;
 
       // Otherwise, we're good to go.
-      PendingDbgValues.push_back(
-          MTracker->emitLoc(DbgOps, Use.Var, Use.Properties));
+      auto &[Var, DILoc] = DVMap.lookupDVID(Use.VarID);
+      PendingDbgValues.push_back(std::make_pair(
+          Use.VarID, MTracker->emitLoc(DbgOps, Var, DILoc, Use.Properties)));
     }
     flushDbgValues(pos, nullptr);
   }
@@ -642,7 +648,7 @@ class TransferTracker {
     return Reg != SP && Reg != FP;
   }
 
-  bool recoverAsEntryValue(const DebugVariable &Var,
+  bool recoverAsEntryValue(DebugVariableID VarID,
                            const DbgValueProperties &Prop,
                            const ValueIDNum &Num) {
     // Is this variable location a candidate to be an entry value. First,
@@ -663,6 +669,8 @@ class TransferTracker {
       DIExpr = *NonVariadicExpression;
     }
 
+    auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
+
     // Is the variable appropriate for entry values (i.e., is a parameter).
     if (!isEntryValueVariable(Var, DIExpr))
       return false;
@@ -676,9 +684,8 @@ class TransferTracker {
         DIExpression::prepend(DIExpr, DIExpression::EntryValue);
     Register Reg = MTracker->LocIdxToLocID[Num.getLoc()];
     MachineOperand MO = MachineOperand::CreateReg(Reg, false);
-
-    PendingDbgValues.push_back(
-        emitMOLoc(MO, Var, {NewExpr, Prop.Indirect, false}));
+    PendingDbgValues.push_back(std::make_pair(
+        VarID, &*emitMOLoc(MO, Var, {NewExpr, Prop.Indirect, false})));
     return true;
   }
 
@@ -687,19 +694,20 @@ class TransferTracker {
     DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
                       MI.getDebugLoc()->getInlinedAt());
     DbgValueProperties Properties(MI);
+    DebugVariableID VarID = DVMap.getDVID(Var);
 
     // Ignore non-register locations, we don't transfer those.
     if (MI.isUndefDebugValue() ||
         all_of(MI.debug_operands(),
                [](const MachineOperand &MO) { return !MO.isReg(); })) {
-      auto It = ActiveVLocs.find(Var);
+      auto It = ActiveVLocs.find(VarID);
       if (It != ActiveVLocs.end()) {
         for (LocIdx Loc : It->second.loc_indices())
-          ActiveMLocs[Loc].erase(Var);
+          ActiveMLocs[Loc].erase(VarID);
         ActiveVLocs.erase(It);
       }
       // Any use-before-defs no longer apply.
-      UseBeforeDefVariables.erase(Var);
+      UseBeforeDefVariables.erase(VarID);
       return;
     }
 
@@ -725,14 +733,15 @@ class TransferTracker {
                 SmallVectorImpl<ResolvedDbgOp> &NewLocs) {
     DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
                       MI.getDebugLoc()->getInlinedAt());
+    DebugVariableID VarID = DVMap.getDVID(Var);
     // Any use-before-defs no longer apply.
-    UseBeforeDefVariables.erase(Var);
+    UseBeforeDefVariables.erase(VarID);
 
     // Erase any previous location.
-    auto It = ActiveVLocs.find(Var);
+    auto It = ActiveVLocs.find(VarID);
     if (It != ActiveVLocs.end()) {
       for (LocIdx Loc : It->second.loc_indices())
-        ActiveMLocs[Loc].erase(Var);
+        ActiveMLocs[Loc].erase(VarID);
     }
 
     // If there _is_ no new location, all we had to do was erase.
@@ -742,7 +751,7 @@ class TransferTracker {
       return;
     }
 
-    SmallVector<std::pair<LocIdx, DebugVariable>> LostMLocs;
+    SmallVector<std::pair<LocIdx, DebugVariableID>> LostMLocs;
     for (ResolvedDbgOp &Op : NewLocs) {
       if (Op.IsConst)
         continue;
@@ -769,17 +778,17 @@ class TransferTracker {
         for (const auto &LostMLoc : LostMLocs)
           ActiveMLocs[LostMLoc.first].erase(LostMLoc.second);
         LostMLocs.clear();
-        It = ActiveVLocs.find(Var);
+        It = ActiveVLocs.find(VarID);
         ActiveMLocs[NewLoc.asU64()].clear();
         VarLocs[NewLoc.asU64()] = MTracker->readMLoc(NewLoc);
       }
 
-      ActiveMLocs[NewLoc].insert(Var);
+      ActiveMLocs[NewLoc].insert(VarID);
     }
 
     if (It == ActiveVLocs.end()) {
       ActiveVLocs.insert(
-          std::make_pair(Var, ResolvedDbgValue(NewLocs, Properties)));
+          std::make_pair(VarID, ResolvedDbgValue(NewLocs, Properties)));
     } else {
       It->second.Ops.assign(NewLocs);
       It->second.Properties = Properties;
@@ -822,21 +831,21 @@ class TransferTracker {
     // explicitly undef, then stop here.
     if (!NewLoc && !MakeUndef) {
       // Try and recover a few more locations with entry values.
-      for (const auto &Var : ActiveMLocIt->second) {
-        auto &Prop = ActiveVLocs.find(Var)->second.Properties;
-        recoverAsEntryValue(Var, Prop, OldValue);
+      for (DebugVariableID VarID : ActiveMLocIt->second) {
+        auto &Prop = ActiveVLocs.find(VarID)->second.Properties;
+        recoverAsEntryValue(VarID, Prop, OldValue);
       }
       flushDbgValues(Pos, nullptr);
       return;
     }
 
     // Examine all the variables based on this location.
-    DenseSet<DebugVariable> NewMLocs;
+    DenseSet<DebugVariableID> NewMLocs;
     // If no new location has been found, every variable that depends on this
     // MLoc is dead, so end their existing MLoc->Var mappings as well.
-    SmallVector<std::pair<LocIdx, DebugVariable>> LostMLocs;
-    for (const auto &Var : ActiveMLocIt->second) {
-      auto ActiveVLocIt = ActiveVLocs.find(Var);
+    SmallVector<std::pair<LocIdx, DebugVariableID>> LostMLocs;
+    for (DebugVariableID VarID : ActiveMLocIt->second) {
+      auto ActiveVLocIt = ActiveVLocs.find(VarID);
       // Re-state the variable location: if there's no replacement then NewLoc
       // is std::nullopt and a $noreg DBG_VALUE will be created. Otherwise, a
       // DBG_VALUE identifying the alternative location will be emitted.
@@ -855,19 +864,21 @@ class TransferTracker {
         replace_copy(ActiveVLocIt->second.Ops, DbgOps.begin(), OldOp, NewOp);
       }
 
-      PendingDbgValues.push_back(MTracker->emitLoc(DbgOps, Var, Properties));
+      auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
+      PendingDbgValues.push_back(std::make_pair(
+          VarID, &*MTracker->emitLoc(DbgOps, Var, DILoc, Properties)));
 
       // Update machine locations <=> variable locations maps. Defer updating
       // ActiveMLocs to avoid invalidating the ActiveMLocIt iterator.
       if (!NewLoc) {
         for (LocIdx Loc : ActiveVLocIt->second.loc_indices()) {
           if (Loc != MLoc)
-            LostMLocs.emplace_back(Loc, Var);
+            LostMLocs.emplace_back(Loc, VarID);
         }
         ActiveVLocs.erase(ActiveVLocIt);
       } else {
         ActiveVLocIt->second.Ops = DbgOps;
-        NewMLocs.insert(Var);
+        NewMLocs.insert(VarID);
       }
     }
 
@@ -891,8 +902,8 @@ class TransferTracker {
     // Commit ActiveMLoc changes.
     ActiveMLocIt->second.clear();
     if (!NewMLocs.empty())
-      for (auto &Var : NewMLocs)
-        ActiveMLocs[*NewLoc].insert(Var);
+      for (DebugVariableID VarID : NewMLocs)
+        ActiveMLocs[*NewLoc].insert(VarID);
   }
 
   /// Transfer variables based on \p Src to be based on \p Dst. This handles
@@ -915,17 +926,18 @@ class TransferTracker {
     // For each variable based on Src; create a location at Dst.
     ResolvedDbgOp SrcOp(Src);
     ResolvedDbgOp DstOp(Dst);
-    for (const auto &Var : MovingVars) {
-      auto ActiveVLocIt = ActiveVLocs.find(Var);
+    for (DebugVariableID VarID : MovingVars) {
+      auto ActiveVLocIt = ActiveVLocs.find(VarID);
       assert(ActiveVLocIt != ActiveVLocs.end());
 
       // Update all instances of Src in the variable's tracked values to Dst.
       std::replace(ActiveVLocIt->second.Ops.begin(),
                    ActiveVLocIt->second.Ops.end(), SrcOp, DstOp);
 
-      MachineInstr *MI = MTracker->emitLoc(ActiveVLocIt->second.Ops, Var,
+      auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
+      MachineInstr *MI = MTracker->emitLoc(ActiveVLocIt->second.Ops, Var, DILoc,
                                            ActiveVLocIt->second.Properties);
-      PendingDbgValues.push_back(MI);
+      PendingDbgValues.push_back(std::make_pair(VarID, MI));
     }
     ActiveMLocs[Src].clear();
     flushDbgValues(Pos, nullptr);
@@ -1176,11 +1188,9 @@ LLVM_DUMP_METHOD void MLocTracker::dump_mloc_map() {
 
 MachineInstrBuilder
 MLocTracker::emitLoc(const SmallVectorImpl<ResolvedDbgOp> &DbgOps,
-                     const DebugVariable &Var,
+                     const DebugVariable &Var, const DILocation *DILoc,
                      const DbgValueProperties &Properties) {
-  DebugLoc DL = DILocation::get(Var.getVariable()->getContext(), 0, 0,
-                                Var.getVariable()->getScope(),
-                                const_cast<DILocation *>(Var.getInlinedAt()));
+  DebugLoc DL = DebugLoc(DILoc);
 
   const MCInstrDesc &Desc = Properties.IsVariadic
                                 ? TII.get(TargetOpcode::DBG_VALUE_LIST)
@@ -1726,7 +1736,8 @@ bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI,
       LastUseBeforeDef = std::max(LastUseBeforeDef, NewID.getInst());
     }
     if (IsValidUseBeforeDef) {
-      TTracker->addUseBeforeDef(V, {MI.getDebugExpression(), false, true},
+      DebugVariableID VID = DVMap.insertDVID(V, MI.getDebugLoc().get());
+      TTracker->addUseBeforeDef(VID, {MI.getDebugExpression(), false, true},
                                 DbgOps, LastUseBeforeDef);
     }
   }
@@ -1735,9 +1746,11 @@ bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI,
   // This DBG_VALUE is potentially a $noreg / undefined location, if
   // FoundLoc is illegal.
   // (XXX -- could morph the DBG_INSTR_REF in the future).
-  MachineInstr *DbgMI = MTracker->emitLoc(NewLocs, V, Properties);
+  MachineInstr *DbgMI =
+      MTracker->emitLoc(NewLocs, V, MI.getDebugLoc().get(), Properties);
+  DebugVariableID ID = DVMap.getDVID(V);
 
-  TTracker->PendingDbgValues.push_back(DbgMI);
+  TTracker->PendingDbgValues.push_back(std::make_pair(ID, DbgMI));
   TTracker->flushDbgValues(MI.getIterator(), nullptr);
   return true;
 }
@@ -3112,7 +3125,8 @@ void InstrRefBasedLDV::getBlocksForScope(
 }
 
 void InstrRefBasedLDV::buildVLocValueMap(
-    const DILocation *DILoc, const SmallSet<DebugVariable, 4> &VarsWeCareAbout,
+    const DILocation *DILoc,
+    const SmallSet<DebugVariableID, 4> &VarsWeCareAbout,
     SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks, LiveInsT &Output,
     FuncValueTable &MOutLocs, FuncValueTable &MInLocs,
     SmallVectorImpl<VLocTracker> &AllTheVLocs) {
@@ -3188,7 +3202,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
   // between blocks. This keeps the locality of working on one lexical scope at
   // at time, but avoids re-processing variable values because some other
   // variable has been assigned.
-  for (const auto &Var : VarsWeCareAbout) {
+  for (DebugVariableID VarID : VarsWeCareAbout) {
     // Re-initialize live-ins and live-outs, to clear the remains of previous
     // variables live-ins / live-outs.
     for (unsigned int I = 0; I < NumBlocks; ++I) {
@@ -3202,7 +3216,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
     SmallPtrSet<MachineBasicBlock *, 32> DefBlocks;
     for (const MachineBasicBlock *ExpMBB : BlocksToExplore) {
       auto &TransferFunc = AllTheVLocs[ExpMBB->getNumber()].Vars;
-      if (TransferFunc.contains(Var))
+      if (TransferFunc.contains(VarID))
         DefBlocks.insert(const_cast<MachineBasicBlock *>(ExpMBB));
     }
 
@@ -3212,7 +3226,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
     // only one value definition, things are very simple.
     if (DefBlocks.size() == 1) {
       placePHIsForSingleVarDefinition(MutBlocksToExplore, *DefBlocks.begin(),
-                                      AllTheVLocs, Var, Output);
+                                      AllTheVLocs, VarID, Output);
       continue;
     }
 
@@ -3285,7 +3299,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
 
         // Do transfer function.
         auto &VTracker = AllTheVLocs[MBB->getNumber()];
-        auto TransferIt = VTracker.Vars.find(Var);
+        auto TransferIt = VTracker.Vars.find(VarID);
         if (TransferIt != VTracker.Vars.end()) {
           // Erase on empty transfer (DBG_VALUE $noreg).
           if (TransferIt->second.Kind == DbgValue::Undef) {
@@ -3347,9 +3361,11 @@ void InstrRefBasedLDV::buildVLocValueMap(
         continue;
       if (BlockLiveIn->Kind == DbgValue::VPHI)
         BlockLiveIn->Kind = DbgValue::Def;
+      auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
       assert(BlockLiveIn->Properties.DIExpr->getFragmentInfo() ==
-             Var.getFragment() && "Fragment info missing during value prop");
-      Output[MBB->getNumber()].push_back(std::make_pair(Var, *BlockLiveIn));
+                 Var.getFragment() &&
+             "Fragment info missing during value prop");
+      Output[MBB->getNumber()].push_back(std::make_pair(VarID, *BlockLiveIn));
     }
   } // Per-variable loop.
 
@@ -3360,7 +3376,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
 void InstrRefBasedLDV::placePHIsForSingleVarDefinition(
     const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks,
     MachineBasicBlock *AssignMBB, SmallVectorImpl<VLocTracker> &AllTheVLocs,
-    const DebugVariable &Var, LiveInsT &Output) {
+    DebugVariableID VarID, LiveInsT &Output) {
   // If there is a single definition of the variable, then working out it's
   // value everywhere is very simple: it's every block dominated by the
   // definition. At the dominance frontier, the usual algorithm would:
@@ -3373,7 +3389,7 @@ void InstrRefBasedLDV::placePHIsForSingleVarDefinition(
 
   // Pick out the variables value from the block transfer function.
   VLocTracker &VLocs = AllTheVLocs[AssignMBB->getNumber()];
-  auto ValueIt = VLocs.Vars.find(Var);
+  auto ValueIt = VLocs.Vars.find(VarID);
   const DbgValue &Value = ValueIt->second;
 
   // If it's an explicit assignment of "undef", that means there is no location
@@ -3388,7 +3404,7 @@ void InstrRefBasedLDV::placePHIsForSingleVarDefinition(
     if (!DomTree->properlyDominates(AssignMBB, ScopeBlock))
       continue;
 
-    Output[ScopeBlock->getNumber()].push_back({Var, Value});
+    Output[ScopeBlock->getNumber()].push_back({VarID, Value});
   }
 
   // All blocks that aren't dominated have no live-in value, thus no variable
@@ -3515,9 +3531,9 @@ bool InstrRefBasedLDV::depthFirstVLocAndEmit(
     const ScopeToVarsT &ScopeToVars, ScopeToAssignBlocksT &ScopeToAssignBlocks,
     LiveInsT &Output, FuncValueTable &MOutLocs, FuncValueTable &MInLocs,
     SmallVectorImpl<VLocTracker> &AllTheVLocs, MachineFunction &MF,
-    DenseMap<DebugVariable, unsigned> &AllVarsNumbering,
     const TargetPassConfig &TPC) {
-  TTracker = new TransferTracker(TII, MTracker, MF, *TRI, CalleeSavedRegs, TPC);
+  TTracker =
+      new TransferTracker(TII, MTracker, MF, DVMap, *TRI, CalleeSavedRegs, TPC);
   unsigned NumLocs = MTracker->getNumLocs();
   VTracker = nullptr;
 
@@ -3622,31 +3638,24 @@ bool InstrRefBasedLDV::depthFirstVLocAndEmit(
     if (MInLocs.hasTableFor(*MBB))
       EjectBlock(*MBB);
 
-  return emitTransfers(AllVarsNumbering);
+  return emitTransfers();
 }
 
-bool InstrRefBasedLDV::emitTransfers(
-    DenseMap<DebugVariable, unsigned> &AllVarsNumbering) {
+bool InstrRefBasedLDV::emitTransfers() {
   // Go through all the transfers recorded in the TransferTracker -- this is
   // both the live-ins to a block, and any movements of values that happen
   // in the middle.
-  for (const auto &P : TTracker->Transfers) {
+  for (auto &P : TTracker->Transfers) {
     // We have to insert DBG_VALUEs in a consistent order, otherwise they
     // appear in DWARF in different orders. Use the order that they appear
     // when walking through each block / each instruction, stored in
-    // AllVarsNumbering.
-    SmallVector<std::pair<unsigned, MachineInstr *>> Insts;
-    for (MachineInstr *MI : P.Insts) {
-      DebugVariable Var(MI->getDebugVariable(), MI->getDebugExpression(),
-                        MI->getDebugLoc()->getInlinedAt());
-      Insts.emplace_back(AllVarsNumbering.find(Var)->second, MI);
-    }
-    llvm::sort(Insts, llvm::less_first());
+    // DVMap.
+    llvm::sort(P.Insts, llvm::less_first());
 
     // Insert either before or after the designated point...
     if (P.MBB) {
       MachineBasicBlock &MBB = *P.MBB;
-      for (const auto &Pair : Insts)
+      for (const auto &Pair : P.Insts)
         MBB.insert(P.Pos, Pair.second);
     } else {
       // Terminators, like tail calls, can clobber things. Don't try and place
@@ -3655,7 +3664,7 @@ bool InstrRefBasedLDV::emitTransfers(
         continue;
 
       MachineBasicBlock &MBB = *P.Pos->getParent();
-      for (const auto &Pair : Insts)
+      for (const auto &Pair : P.Insts)
         MBB.insertAfterBundle(P.Pos, Pair.second);
     }
   }
@@ -3710,7 +3719,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
   initialSetup(MF);
 
   MLocTransfer.resize(MaxNumBlocks);
-  vlocs.resize(MaxNumBlocks, VLocTracker(OverlapFragments, EmptyExpr));
+  vlocs.resize(MaxNumBlocks, VLocTracker(DVMap, OverlapFragments, EmptyExpr));
   SavedLiveIns.resize(MaxNumBlocks);
 
   produceMLocTransferFunction(MF, MLocTransfer, MaxNumBlocks);
@@ -3766,10 +3775,6 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
     MTracker->reset();
   }
 
-  // Number all variables in the order that they appear, to be used as a stable
-  // insertion order later.
-  DenseMap<DebugVariable, unsigned> AllVarsNumbering;
-
   // Map from one LexicalScope to all the variables in that scope.
   ScopeToVarsT ScopeToVars;
 
@@ -3788,16 +3793,15 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
     auto *VTracker = &vlocs[MBB->getNumber()];
     // Collect each variable with a DBG_VALUE in this block.
     for (auto &idx : VTracker->Vars) {
-      const auto &Var = idx.first;
-      const DILocation *ScopeLoc = VTracker->Scopes[Var];
+      DebugVariableID VarID = idx.first;
+      const DILocation *ScopeLoc = VTracker->Scopes[VarID];
       assert(ScopeLoc != nullptr);
       auto *Scope = LS.findLexicalScope(ScopeLoc);
 
       // No insts in scope -> shouldn't have been recorded.
       assert(Scope != nullptr);
 
-      AllVarsNumbering.insert(std::make_pair(Var, AllVarsNumbering.size()));
-      ScopeToVars[Scope].insert(Var);
+      ScopeToVars[Scope].insert(VarID);
       ScopeToAssignBlocks[Scope].insert(VTracker->MBB);
       ScopeToDILocation[Scope] = ScopeLoc;
       ++VarAssignCount;
@@ -3821,7 +3825,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
     // the "else" block of this condition.
     Changed = depthFirstVLocAndEmit(
         MaxNumBlocks, ScopeToDILocation, ScopeToVars, ScopeToAssignBlocks,
-        SavedLiveIns, MOutLocs, MInLocs, vlocs, MF, AllVarsNumbering, *TPC);
+        SavedLiveIns, MOutLocs, MInLocs, vlocs, MF, *TPC);
   }
 
   delete MTracker;
@@ -3840,6 +3844,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
   SeenFragments.clear();
   SeenDbgPHIs.clear();
   DbgOpStore.clear();
+  DVMap.clear();
 
   return Changed;
 }
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
index 8770983481c2f..8c03e38eee062 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
@@ -35,6 +35,44 @@ class DbgOpIDMap;
 
 using namespace llvm;
 
+using DebugVariableID = unsigned;
+using VarAndLoc = std::pair<DebugVariable, const DILocation *>;
+
+/// Mapping from DebugVariable to/from a unique identifying number. Each
+/// DebugVariable consists of three pointers, and after a small amount of
+/// work to identify overlapping fragments of variables we mostly only use
+/// DebugVariables as identities of variables. It's much more compile-time
+/// efficient to use an ID number instead, which this class provides.
+class DebugVariableMap {
+  DenseMap<DebugVariable, unsigned> VarToIdx;
+  SmallVector<VarAndLoc> IdxToVar;
+
+public:
+  DebugVariableID getDVID(const DebugVariable &Var) const {
+    auto It = VarToIdx.find(Var);
+    assert(It != VarToIdx.end());
+    return It->second;
+  }
+
+  DebugVariableID insertDVID(DebugVariable &Var, const DILocation *Loc) {
+    unsigned Size = VarToIdx.size();
+    auto ItPair = VarToIdx.insert({Var, Size});
+    if (ItPair.second) {
+      IdxToVar.push_back({Var, Loc});
+      return Size;
+    }
+
+    return ItPair.first->second;
+  }
+
+  const VarAndLoc &lookupDVID(DebugVariableID ID) const { return IdxToVar[ID]; }
+
+  void clear() {
+    VarToIdx.clear();
+    IdxToVar.clear();
+  }
+};
+
 /// Handle-class for a particular "location". This value-type uniquely
 /// symbolises a register or stack location, allowing manipulation of locations
 /// without concern for where that location is. Practically, this allows us to
@@ -985,7 +1023,7 @@ class MLocTracker {
   /// information in \pProperties, for variable Var. Don't insert it anywhere,
   /// just return the builder for it.
   MachineInstrBuilder emitLoc(const SmallVectorImpl<ResolvedDbgOp> &DbgOps,
-                              const DebugVariable &Var,
+                              const DebugVariable &Var, const DILocation *DILoc,
                               const DbgValueProperties &Properties);
 };
 
@@ -1003,38 +1041,45 @@ using OverlapMap =
 /// identified.
 class VLocTracker {
 public:
+  /// Ref to function-wide map of DebugVariable <=> ID-numbers.
+  DebugVariableMap &DVMap;
   /// Map DebugVariable to the latest Value it's defined to have.
   /// Needs to be a MapVector because we determine order-in-the-input-MIR from
-  /// the order in this container.
+  /// the order in this container. (FIXME: likely no longer true as the ordering
+  /// is now provided by DebugVariableMap).
   /// We only retain the last DbgValue in each block for each variable, to
   /// determine the blocks live-out variable value. The Vars container forms the
   /// transfer function for this block, as part of the dataflow analysis. The
   /// movement of values between locations inside of a block is handled at a
   /// much later stage, in the TransferTracker class.
-  MapVector<DebugVariable, DbgValue> Vars;
-  SmallDenseMap<DebugVariable, const DILocation *, 8> Scopes;
+  MapVector<DebugVariableID, DbgValue> Vars;
+  SmallDenseMap<DebugVariableID, const DILocation *, 8> Scopes;
   MachineBasicBlock *MBB = nullptr;
   const OverlapMap &OverlappingFragments;
   DbgValueProperties EmptyProperties;
 
 public:
-  VLocTracker(const OverlapMap &O, const DIExpression *EmptyExpr)
-      : OverlappingFragments(O), EmptyProperties(EmptyExpr, false, false) {}
+  VLocTracker(DebugVariableMap &DVMap, const OverlapMap &O,
+              const DIExpression *EmptyExpr)
+      : DVMap(DVMap), OverlappingFragments(O),
+        EmptyProperties(EmptyExpr, false, false) {}
 
   void defVar(const MachineInstr &MI, const DbgValueProperties &Properties,
               const SmallVectorImpl<DbgOpID> &DebugOps) {
     assert(MI.isDebugValueLike());
     DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
                       MI.getDebugLoc()->getInlinedAt());
+    // Either insert or fetch an ID number for this variable.
+    DebugVariableID VarID = DVMap.insertDVID(Var, MI.getDebugLoc().get());
     DbgValue Rec = (DebugOps.size() > 0)
                        ? DbgValue(DebugOps, Properties)
                        : DbgValue(Properties, DbgValue::Undef);
 
     // Attempt insertion; overwrite if it's already mapped.
-    auto Result = Vars.insert(std::make_pair(Var, Rec));
+    auto Result = Vars.insert(std::make_pair(VarID, Rec));
     if (!Result.second)
       Result.first->second = Rec;
-    Scopes[Var] = MI.getDebugLoc().get();
+    Scopes[VarID] = MI.getDebugLoc().get();
 
     considerOverlaps(Var, MI.getDebugLoc().get());
   }
@@ -1056,13 +1101,15 @@ class VLocTracker {
 
       DebugVariable Overlapped(Var.getVariable(), OptFragmentInfo,
                                Var.getInlinedAt());
+      // Produce an ID number for this overlapping fragment of a variable.
+      DebugVariableID OverlappedID = DVMap.insertDVID(Overlapped, Loc);
       DbgValue Rec = DbgValue(EmptyProperties, DbgValue::Undef);
 
       // Attempt insertion; overwrite if it's already mapped.
-      auto Result = Vars.insert(std::make_pair(Overlapped, Rec));
+      auto Result = Vars.insert(std::make_pair(OverlappedID, Rec));
       if (!Result.second)
         Result.first->second = Rec;
-      Scopes[Overlapped] = Loc;
+      Scopes[OverlappedID] = Loc;
     }
   }
 
@@ -1093,7 +1140,7 @@ class InstrRefBasedLDV : public LDVImpl {
   /// variables to their values.
   using LiveIdxT = DenseMap<const MachineBasicBlock *, DbgValue *>;
 
-  using VarAndLoc = std::pair<DebugVariable, DbgValue>;
+  using VarAndLoc = std::pair<DebugVariableID, DbgValue>;
 
   /// Type for a live-in value: the predecessor block, and its value.
   using InValueT = std::pair<MachineBasicBlock *, DbgValue *>;
@@ -1106,7 +1153,8 @@ class InstrRefBasedLDV : public LDVImpl {
   using ScopeToDILocT = DenseMap<const LexicalScope *, const DILocation *>;
 
   /// Mapping from lexical scopes to variables in that scope.
-  using ScopeToVarsT = DenseMap<const LexicalScope *, SmallSet<DebugVariable, 4>>;
+  using ScopeToVarsT =
+      DenseMap<const LexicalScope *, SmallSet<DebugVariableID, 4>>;
 
   /// Mapping from lexical scopes to blocks where variables in that scope are
   /// assigned. Such blocks aren't necessarily "in" the lexical scope, it's
@@ -1200,6 +1248,11 @@ class InstrRefBasedLDV : public LDVImpl {
 
   DbgOpIDMap DbgOpStore;
 
+  /// Mapping between DebugVariables and unique ID numbers. This is a more
+  /// efficient way to represent the identity of a variable, versus a plain
+  /// DebugVariable.
+  DebugVariableMap DVMap;
+
   /// True if we need to examine call instructions for stack clobbers. We
   /// normally assume that they don't clobber SP, but stack probes on Windows
   /// do.
@@ -1330,9 +1383,9 @@ class InstrRefBasedLDV : public LDVImpl {
   /// performance as it doesn't have to find the dominance frontier between
   /// different assignments.
   void placePHIsForSingleVarDefinition(
-          const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks,
-          MachineBasicBlock *MBB, SmallVectorImpl<VLocTracker> &AllTheVLocs,
-          const DebugVariable &Var, LiveInsT &Output);
+      const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks,
+      MachineBasicBlock *MBB, SmallVectorImpl<VLocTracker> &AllTheVLocs,
+      DebugVariableID Var, LiveInsT &Output);
 
   /// Calculate the iterated-dominance-frontier for a set of defs, using the
   /// existing LLVM facilities for this. Works for a single "value" or
@@ -1381,7 +1434,7 @@ class InstrRefBasedLDV : public LDVImpl {
   /// scope, but which do contain DBG_VALUEs, which VarLocBasedImpl tracks
   /// locations through.
   void buildVLocValueMap(const DILocation *DILoc,
-                         const SmallSet<DebugVariable, 4> &VarsWeCareAbout,
+                         const SmallSet<DebugVariableID, 4> &VarsWeCareAbout,
                          SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks,
                          LiveInsT &Output, FuncValueTable &MOutLocs,
                          FuncValueTable &MInLocs,
@@ -1414,10 +1467,8 @@ class InstrRefBasedLDV : public LDVImpl {
       const SmallVectorImpl<const MachineBasicBlock *> &BlockOrders);
 
   /// Take collections of DBG_VALUE instructions stored in TTracker, and
-  /// install them into their output blocks. Preserves a stable order of
-  /// DBG_VALUEs produced (which would otherwise cause nondeterminism) through
-  /// the AllVarsNumbering order.
-  bool emitTransfers(DenseMap<DebugVariable, unsigned> &AllVarsNumbering);
+  /// install them into their output blocks.
+  bool emitTransfers();
 
   /// Boilerplate computation of some initial sets, artifical blocks and
   /// RPOT block ordering.
@@ -1437,13 +1488,14 @@ class InstrRefBasedLDV : public LDVImpl {
   /// block information can be fully computed before exploration finishes,
   /// allowing us to emit it and free data structures earlier than otherwise.
   /// It's also good for locality.
-  bool depthFirstVLocAndEmit(
-      unsigned MaxNumBlocks, const ScopeToDILocT &ScopeToDILocation,
-      const ScopeToVarsT &ScopeToVars, ScopeToAssignBlocksT &ScopeToBlocks,
-      LiveInsT &Output, FuncValueTable &MOutLocs, FuncValueTable &MInLocs,
-      SmallVectorImpl<VLocTracker> &AllTheVLocs, MachineFunction &MF,
-      DenseMap<DebugVariable, unsigned> &AllVarsNumbering,
-      const TargetPassConfig &TPC);
+  bool depthFirstVLocAndEmit(unsigned MaxNumBlocks,
+                             const ScopeToDILocT &ScopeToDILocation,
+                             const ScopeToVarsT &ScopeToVars,
+                             ScopeToAssignBlocksT &ScopeToBlocks,
+                             LiveInsT &Output, FuncValueTable &MOutLocs,
+                             FuncValueTable &MInLocs,
+                             SmallVectorImpl<VLocTracker> &AllTheVLocs,
+                             MachineFunction &MF, const TargetPassConfig &TPC);
 
   bool ExtendRanges(MachineFunction &MF, MachineDominatorTree *DomTree,
                     TargetPassConfig *TPC, unsigned InputBBLimit,
diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir
index b54c748ac9e84..67bfd85dcb379 100644
--- a/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir
+++ b/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir
@@ -17,12 +17,12 @@
 # CHECK-LABEL: bb.3.bb3:
 # CHECK:      DBG_VALUE $ecx, $noreg, !{{[0-9]+}},
 # CHECK-SAME:                 !DIExpression(DW_OP_LLVM_fragment, 0, 32)
-# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}}, 
+# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}},
 # CHECK-SAME:                 !DIExpression({{[^)]+}}, DW_OP_LLVM_fragment, 0, 32)
 # CHECK-SAME:                 $ecx, $r8d
 # CHECK-NEXT: DBG_VALUE $ebx, $noreg, !{{[0-9]+}},
 # CHECK-SAME:                 !DIExpression(DW_OP_LLVM_fragment, 32, 32)
-# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}}, 
+# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}},
 # CHECK-SAME:                 !DIExpression({{[^)]+}}, DW_OP_LLVM_fragment, 32, 32)
 # CHECK-SAME:                 $ebx, $r10d
 # CHECK-NEXT: XOR32rr
diff --git a/llvm/unittests/CodeGen/InstrRefLDVTest.cpp b/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
index 306a97c3149cc..50a8cb97ae061 100644
--- a/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
+++ b/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
@@ -55,6 +55,7 @@ class InstrRefLDVTest : public testing::Test {
   DIBasicType *LongInt;
   DIExpression *EmptyExpr;
   LiveDebugValues::OverlapMap Overlaps;
+  LiveDebugValues::DebugVariableMap DVMap;
 
   DebugLoc OutermostLoc, InBlockLoc, NotNestedBlockLoc, InlinedLoc;
 
@@ -176,7 +177,7 @@ class InstrRefLDVTest : public testing::Test {
 
   void addVTracker() {
     ASSERT_TRUE(LDV);
-    VTracker = std::make_unique<VLocTracker>(Overlaps, EmptyExpr);
+    VTracker = std::make_unique<VLocTracker>(DVMap, Overlaps, EmptyExpr);
     LDV->VTracker = &*VTracker;
   }
 

From 50b657c8f655a86826e94131729b0f13a58acbca Mon Sep 17 00:00:00 2001
From: Jeremy Morse <jeremy.morse@sony.com>
Date: Thu, 18 Jul 2024 15:05:57 +0100
Subject: [PATCH 021/486] Revert "[DebugInfo][InstrRef] Index DebugVariables
 and some DILocations (#99318)"

This reverts commit 078198f310d55925ccd9e1aa5b6ff4af3b36bbc7.

Buildbots unhappy, I must have fluffed it
---
 .../LiveDebugValues/InstrRefBasedImpl.cpp     | 201 +++++++++---------
 .../LiveDebugValues/InstrRefBasedImpl.h       | 106 +++------
 .../MIR/X86/live-debug-values-fragments.mir   |   4 +-
 llvm/unittests/CodeGen/InstrRefLDVTest.cpp    |   3 +-
 4 files changed, 128 insertions(+), 186 deletions(-)

diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
index b9cf36a07846c..247258a1ff553 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
@@ -183,7 +183,6 @@ class TransferTracker {
   /// information from it. (XXX make it const?)
   MLocTracker *MTracker;
   MachineFunction &MF;
-  const DebugVariableMap &DVMap;
   bool ShouldEmitDebugEntryValues;
 
   /// Record of all changes in variable locations at a block position. Awkwardly
@@ -192,9 +191,7 @@ class TransferTracker {
   struct Transfer {
     MachineBasicBlock::instr_iterator Pos; /// Position to insert DBG_VALUes
     MachineBasicBlock *MBB; /// non-null if we should insert after.
-    /// Vector of DBG_VALUEs to insert. Store with their DebugVariableID so that
-    /// they can be sorted into a stable order for emission at a later time.
-    SmallVector<std::pair<DebugVariableID, MachineInstr *>, 4> Insts;
+    SmallVector<MachineInstr *, 4> Insts; /// Vector of DBG_VALUEs to insert.
   };
 
   /// Stores the resolved operands (machine locations and constants) and
@@ -230,15 +227,15 @@ class TransferTracker {
   /// Map from LocIdxes to which DebugVariables are based that location.
   /// Mantained while stepping through the block. Not accurate if
   /// VarLocs[Idx] != MTracker->LocIdxToIDNum[Idx].
-  DenseMap<LocIdx, SmallSet<DebugVariableID, 4>> ActiveMLocs;
+  DenseMap<LocIdx, SmallSet<DebugVariable, 4>> ActiveMLocs;
 
   /// Map from DebugVariable to it's current location and qualifying meta
   /// information. To be used in conjunction with ActiveMLocs to construct
   /// enough information for the DBG_VALUEs for a particular LocIdx.
-  DenseMap<DebugVariableID, ResolvedDbgValue> ActiveVLocs;
+  DenseMap<DebugVariable, ResolvedDbgValue> ActiveVLocs;
 
   /// Temporary cache of DBG_VALUEs to be entered into the Transfers collection.
-  SmallVector<std::pair<DebugVariableID, MachineInstr *>, 4> PendingDbgValues;
+  SmallVector<MachineInstr *, 4> PendingDbgValues;
 
   /// Record of a use-before-def: created when a value that's live-in to the
   /// current block isn't available in any machine location, but it will be
@@ -247,12 +244,12 @@ class TransferTracker {
     /// Value of this variable, def'd in block.
     SmallVector<DbgOp> Values;
     /// Identity of this variable.
-    DebugVariableID VarID;
+    DebugVariable Var;
     /// Additional variable properties.
     DbgValueProperties Properties;
-    UseBeforeDef(ArrayRef<DbgOp> Values, DebugVariableID VarID,
+    UseBeforeDef(ArrayRef<DbgOp> Values, const DebugVariable &Var,
                  const DbgValueProperties &Properties)
-        : Values(Values.begin(), Values.end()), VarID(VarID),
+        : Values(Values.begin(), Values.end()), Var(Var),
           Properties(Properties) {}
   };
 
@@ -263,16 +260,15 @@ class TransferTracker {
   /// The set of variables that are in UseBeforeDefs and can become a location
   /// once the relevant value is defined. An element being erased from this
   /// collection prevents the use-before-def materializing.
-  DenseSet<DebugVariableID> UseBeforeDefVariables;
+  DenseSet<DebugVariable> UseBeforeDefVariables;
 
   const TargetRegisterInfo &TRI;
   const BitVector &CalleeSavedRegs;
 
   TransferTracker(const TargetInstrInfo *TII, MLocTracker *MTracker,
-                  MachineFunction &MF, const DebugVariableMap &DVMap,
-                  const TargetRegisterInfo &TRI,
+                  MachineFunction &MF, const TargetRegisterInfo &TRI,
                   const BitVector &CalleeSavedRegs, const TargetPassConfig &TPC)
-      : TII(TII), MTracker(MTracker), MF(MF), DVMap(DVMap), TRI(TRI),
+      : TII(TII), MTracker(MTracker), MF(MF), TRI(TRI),
         CalleeSavedRegs(CalleeSavedRegs) {
     TLI = MF.getSubtarget().getTargetLowering();
     auto &TM = TPC.getTM<TargetMachine>();
@@ -356,7 +352,7 @@ class TransferTracker {
   ///    determine the values used by Value.
   void loadVarInloc(MachineBasicBlock &MBB, DbgOpIDMap &DbgOpStore,
                     const SmallVectorImpl<ValueLocPair> &ValueToLoc,
-                    DebugVariableID VarID, DbgValue Value) {
+                    DebugVariable Var, DbgValue Value) {
     SmallVector<DbgOp> DbgOps;
     SmallVector<ResolvedDbgOp> ResolvedDbgOps;
     bool IsValueValid = true;
@@ -405,7 +401,7 @@ class TransferTracker {
                                       static_cast<unsigned>(Num.getInst()));
           continue;
         }
-        recoverAsEntryValue(VarID, Value.Properties, Num);
+        recoverAsEntryValue(Var, Value.Properties, Num);
         IsValueValid = false;
         break;
       }
@@ -423,7 +419,8 @@ class TransferTracker {
 
     // Add UseBeforeDef entry for the last value to be defined in this block.
     if (LastUseBeforeDef) {
-      addUseBeforeDef(VarID, Value.Properties, DbgOps, LastUseBeforeDef);
+      addUseBeforeDef(Var, Value.Properties, DbgOps,
+                      LastUseBeforeDef);
       return;
     }
 
@@ -431,15 +428,13 @@ class TransferTracker {
     // the transfer.
     for (const ResolvedDbgOp &Op : ResolvedDbgOps)
       if (!Op.IsConst)
-        ActiveMLocs[Op.Loc].insert(VarID);
+        ActiveMLocs[Op.Loc].insert(Var);
     auto NewValue = ResolvedDbgValue{ResolvedDbgOps, Value.Properties};
-    auto Result = ActiveVLocs.insert(std::make_pair(VarID, NewValue));
+    auto Result = ActiveVLocs.insert(std::make_pair(Var, NewValue));
     if (!Result.second)
       Result.first->second = NewValue;
-    auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
     PendingDbgValues.push_back(
-        std::make_pair(VarID, &*MTracker->emitLoc(ResolvedDbgOps, Var, DILoc,
-                                                  Value.Properties)));
+        MTracker->emitLoc(ResolvedDbgOps, Var, Value.Properties));
   }
 
   /// Load object with live-in variable values. \p mlocs contains the live-in
@@ -450,7 +445,7 @@ class TransferTracker {
   /// FIXME: could just examine mloctracker instead of passing in \p mlocs?
   void
   loadInlocs(MachineBasicBlock &MBB, ValueTable &MLocs, DbgOpIDMap &DbgOpStore,
-             const SmallVectorImpl<std::pair<DebugVariableID, DbgValue>> &VLocs,
+             const SmallVectorImpl<std::pair<DebugVariable, DbgValue>> &VLocs,
              unsigned NumLocs) {
     ActiveMLocs.clear();
     ActiveVLocs.clear();
@@ -511,11 +506,11 @@ class TransferTracker {
 
   /// Record that \p Var has value \p ID, a value that becomes available
   /// later in the function.
-  void addUseBeforeDef(DebugVariableID VarID,
+  void addUseBeforeDef(const DebugVariable &Var,
                        const DbgValueProperties &Properties,
                        const SmallVectorImpl<DbgOp> &DbgOps, unsigned Inst) {
-    UseBeforeDefs[Inst].emplace_back(DbgOps, VarID, Properties);
-    UseBeforeDefVariables.insert(VarID);
+    UseBeforeDefs[Inst].emplace_back(DbgOps, Var, Properties);
+    UseBeforeDefVariables.insert(Var);
   }
 
   /// After the instruction at index \p Inst and position \p pos has been
@@ -534,7 +529,7 @@ class TransferTracker {
     // Populate ValueToLoc with illegal default mappings for every value used by
     // any UseBeforeDef variables for this instruction.
     for (auto &Use : MIt->second) {
-      if (!UseBeforeDefVariables.count(Use.VarID))
+      if (!UseBeforeDefVariables.count(Use.Var))
         continue;
 
       for (DbgOp &Op : Use.Values) {
@@ -573,7 +568,7 @@ class TransferTracker {
     // Using the map of values to locations, produce a final set of values for
     // this variable.
     for (auto &Use : MIt->second) {
-      if (!UseBeforeDefVariables.count(Use.VarID))
+      if (!UseBeforeDefVariables.count(Use.Var))
         continue;
 
       SmallVector<ResolvedDbgOp> DbgOps;
@@ -596,9 +591,8 @@ class TransferTracker {
         continue;
 
       // Otherwise, we're good to go.
-      auto &[Var, DILoc] = DVMap.lookupDVID(Use.VarID);
-      PendingDbgValues.push_back(std::make_pair(
-          Use.VarID, MTracker->emitLoc(DbgOps, Var, DILoc, Use.Properties)));
+      PendingDbgValues.push_back(
+          MTracker->emitLoc(DbgOps, Use.Var, Use.Properties));
     }
     flushDbgValues(pos, nullptr);
   }
@@ -648,7 +642,7 @@ class TransferTracker {
     return Reg != SP && Reg != FP;
   }
 
-  bool recoverAsEntryValue(DebugVariableID VarID,
+  bool recoverAsEntryValue(const DebugVariable &Var,
                            const DbgValueProperties &Prop,
                            const ValueIDNum &Num) {
     // Is this variable location a candidate to be an entry value. First,
@@ -669,8 +663,6 @@ class TransferTracker {
       DIExpr = *NonVariadicExpression;
     }
 
-    auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
-
     // Is the variable appropriate for entry values (i.e., is a parameter).
     if (!isEntryValueVariable(Var, DIExpr))
       return false;
@@ -684,8 +676,9 @@ class TransferTracker {
         DIExpression::prepend(DIExpr, DIExpression::EntryValue);
     Register Reg = MTracker->LocIdxToLocID[Num.getLoc()];
     MachineOperand MO = MachineOperand::CreateReg(Reg, false);
-    PendingDbgValues.push_back(std::make_pair(
-        VarID, &*emitMOLoc(MO, Var, {NewExpr, Prop.Indirect, false})));
+
+    PendingDbgValues.push_back(
+        emitMOLoc(MO, Var, {NewExpr, Prop.Indirect, false}));
     return true;
   }
 
@@ -694,20 +687,19 @@ class TransferTracker {
     DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
                       MI.getDebugLoc()->getInlinedAt());
     DbgValueProperties Properties(MI);
-    DebugVariableID VarID = DVMap.getDVID(Var);
 
     // Ignore non-register locations, we don't transfer those.
     if (MI.isUndefDebugValue() ||
         all_of(MI.debug_operands(),
                [](const MachineOperand &MO) { return !MO.isReg(); })) {
-      auto It = ActiveVLocs.find(VarID);
+      auto It = ActiveVLocs.find(Var);
       if (It != ActiveVLocs.end()) {
         for (LocIdx Loc : It->second.loc_indices())
-          ActiveMLocs[Loc].erase(VarID);
+          ActiveMLocs[Loc].erase(Var);
         ActiveVLocs.erase(It);
       }
       // Any use-before-defs no longer apply.
-      UseBeforeDefVariables.erase(VarID);
+      UseBeforeDefVariables.erase(Var);
       return;
     }
 
@@ -733,15 +725,14 @@ class TransferTracker {
                 SmallVectorImpl<ResolvedDbgOp> &NewLocs) {
     DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
                       MI.getDebugLoc()->getInlinedAt());
-    DebugVariableID VarID = DVMap.getDVID(Var);
     // Any use-before-defs no longer apply.
-    UseBeforeDefVariables.erase(VarID);
+    UseBeforeDefVariables.erase(Var);
 
     // Erase any previous location.
-    auto It = ActiveVLocs.find(VarID);
+    auto It = ActiveVLocs.find(Var);
     if (It != ActiveVLocs.end()) {
       for (LocIdx Loc : It->second.loc_indices())
-        ActiveMLocs[Loc].erase(VarID);
+        ActiveMLocs[Loc].erase(Var);
     }
 
     // If there _is_ no new location, all we had to do was erase.
@@ -751,7 +742,7 @@ class TransferTracker {
       return;
     }
 
-    SmallVector<std::pair<LocIdx, DebugVariableID>> LostMLocs;
+    SmallVector<std::pair<LocIdx, DebugVariable>> LostMLocs;
     for (ResolvedDbgOp &Op : NewLocs) {
       if (Op.IsConst)
         continue;
@@ -778,17 +769,17 @@ class TransferTracker {
         for (const auto &LostMLoc : LostMLocs)
           ActiveMLocs[LostMLoc.first].erase(LostMLoc.second);
         LostMLocs.clear();
-        It = ActiveVLocs.find(VarID);
+        It = ActiveVLocs.find(Var);
         ActiveMLocs[NewLoc.asU64()].clear();
         VarLocs[NewLoc.asU64()] = MTracker->readMLoc(NewLoc);
       }
 
-      ActiveMLocs[NewLoc].insert(VarID);
+      ActiveMLocs[NewLoc].insert(Var);
     }
 
     if (It == ActiveVLocs.end()) {
       ActiveVLocs.insert(
-          std::make_pair(VarID, ResolvedDbgValue(NewLocs, Properties)));
+          std::make_pair(Var, ResolvedDbgValue(NewLocs, Properties)));
     } else {
       It->second.Ops.assign(NewLocs);
       It->second.Properties = Properties;
@@ -831,21 +822,21 @@ class TransferTracker {
     // explicitly undef, then stop here.
     if (!NewLoc && !MakeUndef) {
       // Try and recover a few more locations with entry values.
-      for (DebugVariableID VarID : ActiveMLocIt->second) {
-        auto &Prop = ActiveVLocs.find(VarID)->second.Properties;
-        recoverAsEntryValue(VarID, Prop, OldValue);
+      for (const auto &Var : ActiveMLocIt->second) {
+        auto &Prop = ActiveVLocs.find(Var)->second.Properties;
+        recoverAsEntryValue(Var, Prop, OldValue);
       }
       flushDbgValues(Pos, nullptr);
       return;
     }
 
     // Examine all the variables based on this location.
-    DenseSet<DebugVariableID> NewMLocs;
+    DenseSet<DebugVariable> NewMLocs;
     // If no new location has been found, every variable that depends on this
     // MLoc is dead, so end their existing MLoc->Var mappings as well.
-    SmallVector<std::pair<LocIdx, DebugVariableID>> LostMLocs;
-    for (DebugVariableID VarID : ActiveMLocIt->second) {
-      auto ActiveVLocIt = ActiveVLocs.find(VarID);
+    SmallVector<std::pair<LocIdx, DebugVariable>> LostMLocs;
+    for (const auto &Var : ActiveMLocIt->second) {
+      auto ActiveVLocIt = ActiveVLocs.find(Var);
       // Re-state the variable location: if there's no replacement then NewLoc
       // is std::nullopt and a $noreg DBG_VALUE will be created. Otherwise, a
       // DBG_VALUE identifying the alternative location will be emitted.
@@ -864,21 +855,19 @@ class TransferTracker {
         replace_copy(ActiveVLocIt->second.Ops, DbgOps.begin(), OldOp, NewOp);
       }
 
-      auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
-      PendingDbgValues.push_back(std::make_pair(
-          VarID, &*MTracker->emitLoc(DbgOps, Var, DILoc, Properties)));
+      PendingDbgValues.push_back(MTracker->emitLoc(DbgOps, Var, Properties));
 
       // Update machine locations <=> variable locations maps. Defer updating
       // ActiveMLocs to avoid invalidating the ActiveMLocIt iterator.
       if (!NewLoc) {
         for (LocIdx Loc : ActiveVLocIt->second.loc_indices()) {
           if (Loc != MLoc)
-            LostMLocs.emplace_back(Loc, VarID);
+            LostMLocs.emplace_back(Loc, Var);
         }
         ActiveVLocs.erase(ActiveVLocIt);
       } else {
         ActiveVLocIt->second.Ops = DbgOps;
-        NewMLocs.insert(VarID);
+        NewMLocs.insert(Var);
       }
     }
 
@@ -902,8 +891,8 @@ class TransferTracker {
     // Commit ActiveMLoc changes.
     ActiveMLocIt->second.clear();
     if (!NewMLocs.empty())
-      for (DebugVariableID VarID : NewMLocs)
-        ActiveMLocs[*NewLoc].insert(VarID);
+      for (auto &Var : NewMLocs)
+        ActiveMLocs[*NewLoc].insert(Var);
   }
 
   /// Transfer variables based on \p Src to be based on \p Dst. This handles
@@ -926,18 +915,17 @@ class TransferTracker {
     // For each variable based on Src; create a location at Dst.
     ResolvedDbgOp SrcOp(Src);
     ResolvedDbgOp DstOp(Dst);
-    for (DebugVariableID VarID : MovingVars) {
-      auto ActiveVLocIt = ActiveVLocs.find(VarID);
+    for (const auto &Var : MovingVars) {
+      auto ActiveVLocIt = ActiveVLocs.find(Var);
       assert(ActiveVLocIt != ActiveVLocs.end());
 
       // Update all instances of Src in the variable's tracked values to Dst.
       std::replace(ActiveVLocIt->second.Ops.begin(),
                    ActiveVLocIt->second.Ops.end(), SrcOp, DstOp);
 
-      auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
-      MachineInstr *MI = MTracker->emitLoc(ActiveVLocIt->second.Ops, Var, DILoc,
+      MachineInstr *MI = MTracker->emitLoc(ActiveVLocIt->second.Ops, Var,
                                            ActiveVLocIt->second.Properties);
-      PendingDbgValues.push_back(std::make_pair(VarID, MI));
+      PendingDbgValues.push_back(MI);
     }
     ActiveMLocs[Src].clear();
     flushDbgValues(Pos, nullptr);
@@ -1188,9 +1176,11 @@ LLVM_DUMP_METHOD void MLocTracker::dump_mloc_map() {
 
 MachineInstrBuilder
 MLocTracker::emitLoc(const SmallVectorImpl<ResolvedDbgOp> &DbgOps,
-                     const DebugVariable &Var, const DILocation *DILoc,
+                     const DebugVariable &Var,
                      const DbgValueProperties &Properties) {
-  DebugLoc DL = DebugLoc(DILoc);
+  DebugLoc DL = DILocation::get(Var.getVariable()->getContext(), 0, 0,
+                                Var.getVariable()->getScope(),
+                                const_cast<DILocation *>(Var.getInlinedAt()));
 
   const MCInstrDesc &Desc = Properties.IsVariadic
                                 ? TII.get(TargetOpcode::DBG_VALUE_LIST)
@@ -1736,8 +1726,7 @@ bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI,
       LastUseBeforeDef = std::max(LastUseBeforeDef, NewID.getInst());
     }
     if (IsValidUseBeforeDef) {
-      DebugVariableID VID = DVMap.insertDVID(V, MI.getDebugLoc().get());
-      TTracker->addUseBeforeDef(VID, {MI.getDebugExpression(), false, true},
+      TTracker->addUseBeforeDef(V, {MI.getDebugExpression(), false, true},
                                 DbgOps, LastUseBeforeDef);
     }
   }
@@ -1746,11 +1735,9 @@ bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI,
   // This DBG_VALUE is potentially a $noreg / undefined location, if
   // FoundLoc is illegal.
   // (XXX -- could morph the DBG_INSTR_REF in the future).
-  MachineInstr *DbgMI =
-      MTracker->emitLoc(NewLocs, V, MI.getDebugLoc().get(), Properties);
-  DebugVariableID ID = DVMap.getDVID(V);
+  MachineInstr *DbgMI = MTracker->emitLoc(NewLocs, V, Properties);
 
-  TTracker->PendingDbgValues.push_back(std::make_pair(ID, DbgMI));
+  TTracker->PendingDbgValues.push_back(DbgMI);
   TTracker->flushDbgValues(MI.getIterator(), nullptr);
   return true;
 }
@@ -3125,8 +3112,7 @@ void InstrRefBasedLDV::getBlocksForScope(
 }
 
 void InstrRefBasedLDV::buildVLocValueMap(
-    const DILocation *DILoc,
-    const SmallSet<DebugVariableID, 4> &VarsWeCareAbout,
+    const DILocation *DILoc, const SmallSet<DebugVariable, 4> &VarsWeCareAbout,
     SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks, LiveInsT &Output,
     FuncValueTable &MOutLocs, FuncValueTable &MInLocs,
     SmallVectorImpl<VLocTracker> &AllTheVLocs) {
@@ -3202,7 +3188,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
   // between blocks. This keeps the locality of working on one lexical scope at
   // at time, but avoids re-processing variable values because some other
   // variable has been assigned.
-  for (DebugVariableID VarID : VarsWeCareAbout) {
+  for (const auto &Var : VarsWeCareAbout) {
     // Re-initialize live-ins and live-outs, to clear the remains of previous
     // variables live-ins / live-outs.
     for (unsigned int I = 0; I < NumBlocks; ++I) {
@@ -3216,7 +3202,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
     SmallPtrSet<MachineBasicBlock *, 32> DefBlocks;
     for (const MachineBasicBlock *ExpMBB : BlocksToExplore) {
       auto &TransferFunc = AllTheVLocs[ExpMBB->getNumber()].Vars;
-      if (TransferFunc.contains(VarID))
+      if (TransferFunc.contains(Var))
         DefBlocks.insert(const_cast<MachineBasicBlock *>(ExpMBB));
     }
 
@@ -3226,7 +3212,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
     // only one value definition, things are very simple.
     if (DefBlocks.size() == 1) {
       placePHIsForSingleVarDefinition(MutBlocksToExplore, *DefBlocks.begin(),
-                                      AllTheVLocs, VarID, Output);
+                                      AllTheVLocs, Var, Output);
       continue;
     }
 
@@ -3299,7 +3285,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
 
         // Do transfer function.
         auto &VTracker = AllTheVLocs[MBB->getNumber()];
-        auto TransferIt = VTracker.Vars.find(VarID);
+        auto TransferIt = VTracker.Vars.find(Var);
         if (TransferIt != VTracker.Vars.end()) {
           // Erase on empty transfer (DBG_VALUE $noreg).
           if (TransferIt->second.Kind == DbgValue::Undef) {
@@ -3361,11 +3347,9 @@ void InstrRefBasedLDV::buildVLocValueMap(
         continue;
       if (BlockLiveIn->Kind == DbgValue::VPHI)
         BlockLiveIn->Kind = DbgValue::Def;
-      auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
       assert(BlockLiveIn->Properties.DIExpr->getFragmentInfo() ==
-                 Var.getFragment() &&
-             "Fragment info missing during value prop");
-      Output[MBB->getNumber()].push_back(std::make_pair(VarID, *BlockLiveIn));
+             Var.getFragment() && "Fragment info missing during value prop");
+      Output[MBB->getNumber()].push_back(std::make_pair(Var, *BlockLiveIn));
     }
   } // Per-variable loop.
 
@@ -3376,7 +3360,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
 void InstrRefBasedLDV::placePHIsForSingleVarDefinition(
     const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks,
     MachineBasicBlock *AssignMBB, SmallVectorImpl<VLocTracker> &AllTheVLocs,
-    DebugVariableID VarID, LiveInsT &Output) {
+    const DebugVariable &Var, LiveInsT &Output) {
   // If there is a single definition of the variable, then working out it's
   // value everywhere is very simple: it's every block dominated by the
   // definition. At the dominance frontier, the usual algorithm would:
@@ -3389,7 +3373,7 @@ void InstrRefBasedLDV::placePHIsForSingleVarDefinition(
 
   // Pick out the variables value from the block transfer function.
   VLocTracker &VLocs = AllTheVLocs[AssignMBB->getNumber()];
-  auto ValueIt = VLocs.Vars.find(VarID);
+  auto ValueIt = VLocs.Vars.find(Var);
   const DbgValue &Value = ValueIt->second;
 
   // If it's an explicit assignment of "undef", that means there is no location
@@ -3404,7 +3388,7 @@ void InstrRefBasedLDV::placePHIsForSingleVarDefinition(
     if (!DomTree->properlyDominates(AssignMBB, ScopeBlock))
       continue;
 
-    Output[ScopeBlock->getNumber()].push_back({VarID, Value});
+    Output[ScopeBlock->getNumber()].push_back({Var, Value});
   }
 
   // All blocks that aren't dominated have no live-in value, thus no variable
@@ -3531,9 +3515,9 @@ bool InstrRefBasedLDV::depthFirstVLocAndEmit(
     const ScopeToVarsT &ScopeToVars, ScopeToAssignBlocksT &ScopeToAssignBlocks,
     LiveInsT &Output, FuncValueTable &MOutLocs, FuncValueTable &MInLocs,
     SmallVectorImpl<VLocTracker> &AllTheVLocs, MachineFunction &MF,
+    DenseMap<DebugVariable, unsigned> &AllVarsNumbering,
     const TargetPassConfig &TPC) {
-  TTracker =
-      new TransferTracker(TII, MTracker, MF, DVMap, *TRI, CalleeSavedRegs, TPC);
+  TTracker = new TransferTracker(TII, MTracker, MF, *TRI, CalleeSavedRegs, TPC);
   unsigned NumLocs = MTracker->getNumLocs();
   VTracker = nullptr;
 
@@ -3638,24 +3622,31 @@ bool InstrRefBasedLDV::depthFirstVLocAndEmit(
     if (MInLocs.hasTableFor(*MBB))
       EjectBlock(*MBB);
 
-  return emitTransfers();
+  return emitTransfers(AllVarsNumbering);
 }
 
-bool InstrRefBasedLDV::emitTransfers() {
+bool InstrRefBasedLDV::emitTransfers(
+    DenseMap<DebugVariable, unsigned> &AllVarsNumbering) {
   // Go through all the transfers recorded in the TransferTracker -- this is
   // both the live-ins to a block, and any movements of values that happen
   // in the middle.
-  for (auto &P : TTracker->Transfers) {
+  for (const auto &P : TTracker->Transfers) {
     // We have to insert DBG_VALUEs in a consistent order, otherwise they
     // appear in DWARF in different orders. Use the order that they appear
     // when walking through each block / each instruction, stored in
-    // DVMap.
-    llvm::sort(P.Insts, llvm::less_first());
+    // AllVarsNumbering.
+    SmallVector<std::pair<unsigned, MachineInstr *>> Insts;
+    for (MachineInstr *MI : P.Insts) {
+      DebugVariable Var(MI->getDebugVariable(), MI->getDebugExpression(),
+                        MI->getDebugLoc()->getInlinedAt());
+      Insts.emplace_back(AllVarsNumbering.find(Var)->second, MI);
+    }
+    llvm::sort(Insts, llvm::less_first());
 
     // Insert either before or after the designated point...
     if (P.MBB) {
       MachineBasicBlock &MBB = *P.MBB;
-      for (const auto &Pair : P.Insts)
+      for (const auto &Pair : Insts)
         MBB.insert(P.Pos, Pair.second);
     } else {
       // Terminators, like tail calls, can clobber things. Don't try and place
@@ -3664,7 +3655,7 @@ bool InstrRefBasedLDV::emitTransfers() {
         continue;
 
       MachineBasicBlock &MBB = *P.Pos->getParent();
-      for (const auto &Pair : P.Insts)
+      for (const auto &Pair : Insts)
         MBB.insertAfterBundle(P.Pos, Pair.second);
     }
   }
@@ -3719,7 +3710,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
   initialSetup(MF);
 
   MLocTransfer.resize(MaxNumBlocks);
-  vlocs.resize(MaxNumBlocks, VLocTracker(DVMap, OverlapFragments, EmptyExpr));
+  vlocs.resize(MaxNumBlocks, VLocTracker(OverlapFragments, EmptyExpr));
   SavedLiveIns.resize(MaxNumBlocks);
 
   produceMLocTransferFunction(MF, MLocTransfer, MaxNumBlocks);
@@ -3775,6 +3766,10 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
     MTracker->reset();
   }
 
+  // Number all variables in the order that they appear, to be used as a stable
+  // insertion order later.
+  DenseMap<DebugVariable, unsigned> AllVarsNumbering;
+
   // Map from one LexicalScope to all the variables in that scope.
   ScopeToVarsT ScopeToVars;
 
@@ -3793,15 +3788,16 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
     auto *VTracker = &vlocs[MBB->getNumber()];
     // Collect each variable with a DBG_VALUE in this block.
     for (auto &idx : VTracker->Vars) {
-      DebugVariableID VarID = idx.first;
-      const DILocation *ScopeLoc = VTracker->Scopes[VarID];
+      const auto &Var = idx.first;
+      const DILocation *ScopeLoc = VTracker->Scopes[Var];
       assert(ScopeLoc != nullptr);
       auto *Scope = LS.findLexicalScope(ScopeLoc);
 
       // No insts in scope -> shouldn't have been recorded.
       assert(Scope != nullptr);
 
-      ScopeToVars[Scope].insert(VarID);
+      AllVarsNumbering.insert(std::make_pair(Var, AllVarsNumbering.size()));
+      ScopeToVars[Scope].insert(Var);
       ScopeToAssignBlocks[Scope].insert(VTracker->MBB);
       ScopeToDILocation[Scope] = ScopeLoc;
       ++VarAssignCount;
@@ -3825,7 +3821,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
     // the "else" block of this condition.
     Changed = depthFirstVLocAndEmit(
         MaxNumBlocks, ScopeToDILocation, ScopeToVars, ScopeToAssignBlocks,
-        SavedLiveIns, MOutLocs, MInLocs, vlocs, MF, *TPC);
+        SavedLiveIns, MOutLocs, MInLocs, vlocs, MF, AllVarsNumbering, *TPC);
   }
 
   delete MTracker;
@@ -3844,7 +3840,6 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
   SeenFragments.clear();
   SeenDbgPHIs.clear();
   DbgOpStore.clear();
-  DVMap.clear();
 
   return Changed;
 }
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
index 8c03e38eee062..8770983481c2f 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
@@ -35,44 +35,6 @@ class DbgOpIDMap;
 
 using namespace llvm;
 
-using DebugVariableID = unsigned;
-using VarAndLoc = std::pair<DebugVariable, const DILocation *>;
-
-/// Mapping from DebugVariable to/from a unique identifying number. Each
-/// DebugVariable consists of three pointers, and after a small amount of
-/// work to identify overlapping fragments of variables we mostly only use
-/// DebugVariables as identities of variables. It's much more compile-time
-/// efficient to use an ID number instead, which this class provides.
-class DebugVariableMap {
-  DenseMap<DebugVariable, unsigned> VarToIdx;
-  SmallVector<VarAndLoc> IdxToVar;
-
-public:
-  DebugVariableID getDVID(const DebugVariable &Var) const {
-    auto It = VarToIdx.find(Var);
-    assert(It != VarToIdx.end());
-    return It->second;
-  }
-
-  DebugVariableID insertDVID(DebugVariable &Var, const DILocation *Loc) {
-    unsigned Size = VarToIdx.size();
-    auto ItPair = VarToIdx.insert({Var, Size});
-    if (ItPair.second) {
-      IdxToVar.push_back({Var, Loc});
-      return Size;
-    }
-
-    return ItPair.first->second;
-  }
-
-  const VarAndLoc &lookupDVID(DebugVariableID ID) const { return IdxToVar[ID]; }
-
-  void clear() {
-    VarToIdx.clear();
-    IdxToVar.clear();
-  }
-};
-
 /// Handle-class for a particular "location". This value-type uniquely
 /// symbolises a register or stack location, allowing manipulation of locations
 /// without concern for where that location is. Practically, this allows us to
@@ -1023,7 +985,7 @@ class MLocTracker {
   /// information in \pProperties, for variable Var. Don't insert it anywhere,
   /// just return the builder for it.
   MachineInstrBuilder emitLoc(const SmallVectorImpl<ResolvedDbgOp> &DbgOps,
-                              const DebugVariable &Var, const DILocation *DILoc,
+                              const DebugVariable &Var,
                               const DbgValueProperties &Properties);
 };
 
@@ -1041,45 +1003,38 @@ using OverlapMap =
 /// identified.
 class VLocTracker {
 public:
-  /// Ref to function-wide map of DebugVariable <=> ID-numbers.
-  DebugVariableMap &DVMap;
   /// Map DebugVariable to the latest Value it's defined to have.
   /// Needs to be a MapVector because we determine order-in-the-input-MIR from
-  /// the order in this container. (FIXME: likely no longer true as the ordering
-  /// is now provided by DebugVariableMap).
+  /// the order in this container.
   /// We only retain the last DbgValue in each block for each variable, to
   /// determine the blocks live-out variable value. The Vars container forms the
   /// transfer function for this block, as part of the dataflow analysis. The
   /// movement of values between locations inside of a block is handled at a
   /// much later stage, in the TransferTracker class.
-  MapVector<DebugVariableID, DbgValue> Vars;
-  SmallDenseMap<DebugVariableID, const DILocation *, 8> Scopes;
+  MapVector<DebugVariable, DbgValue> Vars;
+  SmallDenseMap<DebugVariable, const DILocation *, 8> Scopes;
   MachineBasicBlock *MBB = nullptr;
   const OverlapMap &OverlappingFragments;
   DbgValueProperties EmptyProperties;
 
 public:
-  VLocTracker(DebugVariableMap &DVMap, const OverlapMap &O,
-              const DIExpression *EmptyExpr)
-      : DVMap(DVMap), OverlappingFragments(O),
-        EmptyProperties(EmptyExpr, false, false) {}
+  VLocTracker(const OverlapMap &O, const DIExpression *EmptyExpr)
+      : OverlappingFragments(O), EmptyProperties(EmptyExpr, false, false) {}
 
   void defVar(const MachineInstr &MI, const DbgValueProperties &Properties,
               const SmallVectorImpl<DbgOpID> &DebugOps) {
     assert(MI.isDebugValueLike());
     DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
                       MI.getDebugLoc()->getInlinedAt());
-    // Either insert or fetch an ID number for this variable.
-    DebugVariableID VarID = DVMap.insertDVID(Var, MI.getDebugLoc().get());
     DbgValue Rec = (DebugOps.size() > 0)
                        ? DbgValue(DebugOps, Properties)
                        : DbgValue(Properties, DbgValue::Undef);
 
     // Attempt insertion; overwrite if it's already mapped.
-    auto Result = Vars.insert(std::make_pair(VarID, Rec));
+    auto Result = Vars.insert(std::make_pair(Var, Rec));
     if (!Result.second)
       Result.first->second = Rec;
-    Scopes[VarID] = MI.getDebugLoc().get();
+    Scopes[Var] = MI.getDebugLoc().get();
 
     considerOverlaps(Var, MI.getDebugLoc().get());
   }
@@ -1101,15 +1056,13 @@ class VLocTracker {
 
       DebugVariable Overlapped(Var.getVariable(), OptFragmentInfo,
                                Var.getInlinedAt());
-      // Produce an ID number for this overlapping fragment of a variable.
-      DebugVariableID OverlappedID = DVMap.insertDVID(Overlapped, Loc);
       DbgValue Rec = DbgValue(EmptyProperties, DbgValue::Undef);
 
       // Attempt insertion; overwrite if it's already mapped.
-      auto Result = Vars.insert(std::make_pair(OverlappedID, Rec));
+      auto Result = Vars.insert(std::make_pair(Overlapped, Rec));
       if (!Result.second)
         Result.first->second = Rec;
-      Scopes[OverlappedID] = Loc;
+      Scopes[Overlapped] = Loc;
     }
   }
 
@@ -1140,7 +1093,7 @@ class InstrRefBasedLDV : public LDVImpl {
   /// variables to their values.
   using LiveIdxT = DenseMap<const MachineBasicBlock *, DbgValue *>;
 
-  using VarAndLoc = std::pair<DebugVariableID, DbgValue>;
+  using VarAndLoc = std::pair<DebugVariable, DbgValue>;
 
   /// Type for a live-in value: the predecessor block, and its value.
   using InValueT = std::pair<MachineBasicBlock *, DbgValue *>;
@@ -1153,8 +1106,7 @@ class InstrRefBasedLDV : public LDVImpl {
   using ScopeToDILocT = DenseMap<const LexicalScope *, const DILocation *>;
 
   /// Mapping from lexical scopes to variables in that scope.
-  using ScopeToVarsT =
-      DenseMap<const LexicalScope *, SmallSet<DebugVariableID, 4>>;
+  using ScopeToVarsT = DenseMap<const LexicalScope *, SmallSet<DebugVariable, 4>>;
 
   /// Mapping from lexical scopes to blocks where variables in that scope are
   /// assigned. Such blocks aren't necessarily "in" the lexical scope, it's
@@ -1248,11 +1200,6 @@ class InstrRefBasedLDV : public LDVImpl {
 
   DbgOpIDMap DbgOpStore;
 
-  /// Mapping between DebugVariables and unique ID numbers. This is a more
-  /// efficient way to represent the identity of a variable, versus a plain
-  /// DebugVariable.
-  DebugVariableMap DVMap;
-
   /// True if we need to examine call instructions for stack clobbers. We
   /// normally assume that they don't clobber SP, but stack probes on Windows
   /// do.
@@ -1383,9 +1330,9 @@ class InstrRefBasedLDV : public LDVImpl {
   /// performance as it doesn't have to find the dominance frontier between
   /// different assignments.
   void placePHIsForSingleVarDefinition(
-      const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks,
-      MachineBasicBlock *MBB, SmallVectorImpl<VLocTracker> &AllTheVLocs,
-      DebugVariableID Var, LiveInsT &Output);
+          const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks,
+          MachineBasicBlock *MBB, SmallVectorImpl<VLocTracker> &AllTheVLocs,
+          const DebugVariable &Var, LiveInsT &Output);
 
   /// Calculate the iterated-dominance-frontier for a set of defs, using the
   /// existing LLVM facilities for this. Works for a single "value" or
@@ -1434,7 +1381,7 @@ class InstrRefBasedLDV : public LDVImpl {
   /// scope, but which do contain DBG_VALUEs, which VarLocBasedImpl tracks
   /// locations through.
   void buildVLocValueMap(const DILocation *DILoc,
-                         const SmallSet<DebugVariableID, 4> &VarsWeCareAbout,
+                         const SmallSet<DebugVariable, 4> &VarsWeCareAbout,
                          SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks,
                          LiveInsT &Output, FuncValueTable &MOutLocs,
                          FuncValueTable &MInLocs,
@@ -1467,8 +1414,10 @@ class InstrRefBasedLDV : public LDVImpl {
       const SmallVectorImpl<const MachineBasicBlock *> &BlockOrders);
 
   /// Take collections of DBG_VALUE instructions stored in TTracker, and
-  /// install them into their output blocks.
-  bool emitTransfers();
+  /// install them into their output blocks. Preserves a stable order of
+  /// DBG_VALUEs produced (which would otherwise cause nondeterminism) through
+  /// the AllVarsNumbering order.
+  bool emitTransfers(DenseMap<DebugVariable, unsigned> &AllVarsNumbering);
 
   /// Boilerplate computation of some initial sets, artifical blocks and
   /// RPOT block ordering.
@@ -1488,14 +1437,13 @@ class InstrRefBasedLDV : public LDVImpl {
   /// block information can be fully computed before exploration finishes,
   /// allowing us to emit it and free data structures earlier than otherwise.
   /// It's also good for locality.
-  bool depthFirstVLocAndEmit(unsigned MaxNumBlocks,
-                             const ScopeToDILocT &ScopeToDILocation,
-                             const ScopeToVarsT &ScopeToVars,
-                             ScopeToAssignBlocksT &ScopeToBlocks,
-                             LiveInsT &Output, FuncValueTable &MOutLocs,
-                             FuncValueTable &MInLocs,
-                             SmallVectorImpl<VLocTracker> &AllTheVLocs,
-                             MachineFunction &MF, const TargetPassConfig &TPC);
+  bool depthFirstVLocAndEmit(
+      unsigned MaxNumBlocks, const ScopeToDILocT &ScopeToDILocation,
+      const ScopeToVarsT &ScopeToVars, ScopeToAssignBlocksT &ScopeToBlocks,
+      LiveInsT &Output, FuncValueTable &MOutLocs, FuncValueTable &MInLocs,
+      SmallVectorImpl<VLocTracker> &AllTheVLocs, MachineFunction &MF,
+      DenseMap<DebugVariable, unsigned> &AllVarsNumbering,
+      const TargetPassConfig &TPC);
 
   bool ExtendRanges(MachineFunction &MF, MachineDominatorTree *DomTree,
                     TargetPassConfig *TPC, unsigned InputBBLimit,
diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir
index 67bfd85dcb379..b54c748ac9e84 100644
--- a/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir
+++ b/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir
@@ -17,12 +17,12 @@
 # CHECK-LABEL: bb.3.bb3:
 # CHECK:      DBG_VALUE $ecx, $noreg, !{{[0-9]+}},
 # CHECK-SAME:                 !DIExpression(DW_OP_LLVM_fragment, 0, 32)
-# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}},
+# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}}, 
 # CHECK-SAME:                 !DIExpression({{[^)]+}}, DW_OP_LLVM_fragment, 0, 32)
 # CHECK-SAME:                 $ecx, $r8d
 # CHECK-NEXT: DBG_VALUE $ebx, $noreg, !{{[0-9]+}},
 # CHECK-SAME:                 !DIExpression(DW_OP_LLVM_fragment, 32, 32)
-# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}},
+# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}}, 
 # CHECK-SAME:                 !DIExpression({{[^)]+}}, DW_OP_LLVM_fragment, 32, 32)
 # CHECK-SAME:                 $ebx, $r10d
 # CHECK-NEXT: XOR32rr
diff --git a/llvm/unittests/CodeGen/InstrRefLDVTest.cpp b/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
index 50a8cb97ae061..306a97c3149cc 100644
--- a/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
+++ b/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
@@ -55,7 +55,6 @@ class InstrRefLDVTest : public testing::Test {
   DIBasicType *LongInt;
   DIExpression *EmptyExpr;
   LiveDebugValues::OverlapMap Overlaps;
-  LiveDebugValues::DebugVariableMap DVMap;
 
   DebugLoc OutermostLoc, InBlockLoc, NotNestedBlockLoc, InlinedLoc;
 
@@ -177,7 +176,7 @@ class InstrRefLDVTest : public testing::Test {
 
   void addVTracker() {
     ASSERT_TRUE(LDV);
-    VTracker = std::make_unique<VLocTracker>(DVMap, Overlaps, EmptyExpr);
+    VTracker = std::make_unique<VLocTracker>(Overlaps, EmptyExpr);
     LDV->VTracker = &*VTracker;
   }
 

From 1a80153ba91f1e623c042fa0ae1ee5ab67087c0e Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Thu, 18 Jul 2024 10:11:39 -0400
Subject: [PATCH 022/486] [LV][NFC]Simplify the structure and improve message
 of safe distance analysis for scalable vectorization. (#99487)

---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 40919c944d21f..7ca798a8b2d89 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4197,14 +4197,11 @@ bool LoopVectorizationCostModel::isScalableVectorizationAllowed() {
     return false;
   }
 
-  if (!Legal->isSafeForAnyVectorWidth()) {
-    std::optional<unsigned> MaxVScale = getMaxVScale(*TheFunction, TTI);
-    if (!MaxVScale) {
-      reportVectorizationInfo(
-          "The target does not provide maximum vscale value.",
-          "ScalableVFUnfeasible", ORE, TheLoop);
-      return false;
-    }
+  if (!Legal->isSafeForAnyVectorWidth() && !getMaxVScale(*TheFunction, TTI)) {
+    reportVectorizationInfo("The target does not provide maximum vscale value "
+                            "for safe distance analysis.",
+                            "ScalableVFUnfeasible", ORE, TheLoop);
+    return false;
   }
 
   IsScalableVectorizationAllowed = true;

From 92f9f014015554c5dd18df4699765cc42853a04d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 18 Jul 2024 15:09:40 +0100
Subject: [PATCH 023/486] [X86] getGFNICtrlMask - create a vXi8 mask instead of
 a bitcasted vXi64 mask.

Helps avoid some missed load-folds by stripping away bitcasts and make it easier to grok the GF2P8AFFINEQB masks.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp     |   8 +-
 llvm/test/CodeGen/X86/bitreverse.ll         |   2 +-
 llvm/test/CodeGen/X86/gfni-funnel-shifts.ll | 141 +++++++--------
 llvm/test/CodeGen/X86/gfni-rotates.ll       | 188 +++++++++-----------
 llvm/test/CodeGen/X86/gfni-shifts.ll        | 127 ++++++-------
 llvm/test/CodeGen/X86/vector-bitreverse.ll  |  71 +++-----
 6 files changed, 236 insertions(+), 301 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 56d08e7f76908..9d742be43408f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -29102,8 +29102,12 @@ SDValue getGFNICtrlMask(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, MVT
   assert(VT.getVectorElementType() == MVT::i8 &&
          (VT.getSizeInBits() % 64) == 0 && "Illegal GFNI control type");
   uint64_t Imm = getGFNICtrlImm(Opcode, Amt);
-  MVT MaskVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
-  return DAG.getBitcast(VT, DAG.getConstant(Imm, DL, MaskVT));
+  SmallVector<SDValue> MaskBits;
+  for (unsigned I = 0, E = VT.getSizeInBits(); I != E; I += 8) {
+    uint64_t Bits = (Imm >> (I % 64)) & 255;
+    MaskBits.push_back(DAG.getConstant(Bits, DL, MVT::i8));
+  }
+  return DAG.getBuildVector(VT, DL, MaskBits);
 }
 
 // Return true if the required (according to Opcode) shift-imm form is natively
diff --git a/llvm/test/CodeGen/X86/bitreverse.ll b/llvm/test/CodeGen/X86/bitreverse.ll
index 4f2654843728f..e256b811ee839 100644
--- a/llvm/test/CodeGen/X86/bitreverse.ll
+++ b/llvm/test/CodeGen/X86/bitreverse.ll
@@ -1340,7 +1340,7 @@ define i528 @large_promotion(i528 %A) nounwind {
 ; GFNI-NEXT:    pushq %r14
 ; GFNI-NEXT:    pushq %rbx
 ; GFNI-NEXT:    movq %rdi, %rax
-; GFNI-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [9241421688590303745,9241421688590303745]
+; GFNI-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
 ; GFNI-NEXT:    vmovq %xmm1, %r10
diff --git a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
index 5857ff1162ceb..c071f64dc66cd 100644
--- a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
+++ b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
@@ -699,7 +699,7 @@ define <32 x i8> @var_fshl_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; GFNIAVX512VL-NEXT:    vpandn %ymm3, %ymm2, %ymm4
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm6
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm4, %ymm6, %ymm1, %ymm1
@@ -743,7 +743,7 @@ define <32 x i8> @var_fshr_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm9
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm9
 ; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
@@ -751,12 +751,12 @@ define <32 x i8> @var_fshr_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ; GFNISSE-NEXT:    pand %xmm7, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm9, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [290499906672525312,290499906672525312]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm10
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm10
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [145249953336295424,145249953336295424]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm11
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm11
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
@@ -769,7 +769,7 @@ define <32 x i8> @var_fshr_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ; GFNISSE-NEXT:    psllw $5, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm12 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm12 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm13
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm13
 ; GFNISSE-NEXT:    paddb %xmm6, %xmm6
@@ -819,7 +819,7 @@ define <32 x i8> @var_fshr_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ; GFNIAVX1-LABEL: var_fshr_v32i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm4, %xmm6
 ; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
@@ -827,25 +827,25 @@ define <32 x i8> @var_fshr_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm7, %xmm8
 ; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm6, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [290499906672525312,290499906672525312]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX1-NEXT:    # xmm6 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm4, %xmm9
 ; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
 ; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm9, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [145249953336295424,145249953336295424]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX1-NEXT:    # xmm9 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm4, %xmm10
 ; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
 ; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm10, %xmm4, %xmm4
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm8
 ; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm10 = [16909320,16909320]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm10 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX1-NEXT:    # xmm10 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm10, %xmm8, %xmm11
 ; GFNIAVX1-NEXT:    vpxor %xmm3, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm11, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm11 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm11 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX1-NEXT:    # xmm11 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm11, %xmm8, %xmm12
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
@@ -880,33 +880,28 @@ define <32 x i8> @var_fshr_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ;
 ; GFNIAVX2-LABEL: var_fshr_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm1, %ymm3
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; GFNIAVX2-NEXT:    vpand %ymm4, %ymm2, %ymm5
-; GFNIAVX2-NEXT:    vpsllw $5, %ymm5, %ymm5
-; GFNIAVX2-NEXT:    vpblendvb %ymm5, %ymm3, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm1, %ymm3
-; GFNIAVX2-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; GFNIAVX2-NEXT:    vpblendvb %ymm5, %ymm3, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm1, %ymm3
+; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; GFNIAVX2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; GFNIAVX2-NEXT:    vpsllw $5, %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vpaddb %ymm4, %ymm4, %ymm5
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6
+; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm6, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4
 ; GFNIAVX2-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; GFNIAVX2-NEXT:    vpblendvb %ymm5, %ymm3, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [16909320,16909320,16909320,16909320]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
-; GFNIAVX2-NEXT:    vpandn %ymm4, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm3, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1108169199648,1108169199648,1108169199648,1108169199648]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
-; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm3, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
-; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm3, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
+; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
@@ -1288,7 +1283,7 @@ define <32 x i8> @constant_fshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 define <32 x i8> @splatconstant_fshl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNISSE-LABEL: splatconstant_fshl_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
 ; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm5 = [16909320,16909320]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm0
@@ -1307,10 +1302,8 @@ define <32 x i8> @splatconstant_fshl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind
 ;
 ; GFNIAVX2-LABEL: splatconstant_fshl_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [16909320,16909320,16909320,16909320]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
@@ -1328,9 +1321,9 @@ declare <32 x i8> @llvm.fshl.v32i8(<32 x i8>, <32 x i8>, <32 x i8>)
 define <32 x i8> @splatconstant_fshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNISSE-LABEL: splatconstant_fshr_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [4647714815446351872,4647714815446351872]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm0
 ; GFNISSE-NEXT:    por %xmm2, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm3
@@ -1347,10 +1340,8 @@ define <32 x i8> @splatconstant_fshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind
 ;
 ; GFNIAVX2-LABEL: splatconstant_fshr_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1108169199648,1108169199648,1108169199648,1108169199648]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
@@ -1686,9 +1677,9 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNIAVX512VL-LABEL: var_fshl_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm3, %ymm6
 ; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; GFNIAVX512VL-NEXT:    vpandq %zmm7, %zmm2, %zmm2
@@ -1696,7 +1687,7 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNIAVX512VL-NEXT:    vpxor %ymm7, %ymm8, %ymm9
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm9, %ymm9
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm9, %ymm6, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm3, %ymm10
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm9, %ymm9, %ymm9
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm9, %ymm10, %ymm3, %ymm3
@@ -1716,11 +1707,11 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [16909320,16909320,16909320,16909320]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm5
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm8, %ymm6
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm5, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm3, %ymm7
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm7, %ymm3, %ymm3
@@ -1805,19 +1796,19 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
 ; GFNISSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm8
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pand %xmm12, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm8
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm13 = [290499906672525312,290499906672525312]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm13 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm8
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm14 = [145249953336295424,145249953336295424]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm14 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm14, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm6
@@ -1830,7 +1821,7 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm8
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm0 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm0 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm0, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm9, %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
@@ -1917,7 +1908,7 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNIAVX1-LABEL: var_fshr_v64i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm8
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX1-NEXT:    # xmm7 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm8, %xmm9
 ; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} ymm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
@@ -1925,25 +1916,25 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm11, %xmm10
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm10, %xmm12
 ; GFNIAVX1-NEXT:    vpblendvb %xmm12, %xmm9, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [290499906672525312,290499906672525312]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm8, %xmm9
 ; GFNIAVX1-NEXT:    vpaddb %xmm12, %xmm12, %xmm12
 ; GFNIAVX1-NEXT:    vpblendvb %xmm12, %xmm9, %xmm8, %xmm9
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [145249953336295424,145249953336295424]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX1-NEXT:    # xmm8 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm9, %xmm13
 ; GFNIAVX1-NEXT:    vpaddb %xmm12, %xmm12, %xmm12
 ; GFNIAVX1-NEXT:    vpblendvb %xmm12, %xmm13, %xmm9, %xmm12
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm9
 ; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm13
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [16909320,16909320]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX1-NEXT:    # xmm9 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm13, %xmm14
 ; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm10, %xmm10
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm10, %xmm15
 ; GFNIAVX1-NEXT:    vpblendvb %xmm15, %xmm14, %xmm13, %xmm13
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm10 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm10 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX1-NEXT:    # xmm10 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm10, %xmm13, %xmm14
 ; GFNIAVX1-NEXT:    vpaddb %xmm15, %xmm15, %xmm15
@@ -2025,27 +2016,27 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ;
 ; GFNIAVX2-LABEL: var_fshr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm2, %ymm8
 ; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; GFNIAVX2-NEXT:    vpand %ymm6, %ymm4, %ymm9
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm9, %ymm9
 ; GFNIAVX2-NEXT:    vpblendvb %ymm9, %ymm8, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm2, %ymm10
 ; GFNIAVX2-NEXT:    vpaddb %ymm9, %ymm9, %ymm9
 ; GFNIAVX2-NEXT:    vpblendvb %ymm9, %ymm10, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm10, %ymm2, %ymm11
 ; GFNIAVX2-NEXT:    vpaddb %ymm9, %ymm9, %ymm9
 ; GFNIAVX2-NEXT:    vpblendvb %ymm9, %ymm11, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm9, %ymm0, %ymm11
 ; GFNIAVX2-NEXT:    vpandn %ymm6, %ymm4, %ymm4
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm4, %ymm4
 ; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm11, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm11 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm11 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm11, %ymm0, %ymm12
 ; GFNIAVX2-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
 ; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm12, %ymm0, %ymm0
@@ -2080,18 +2071,18 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNIAVX512VL-LABEL: var_fshr_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm5
 ; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; GFNIAVX512VL-NEXT:    vpandq %zmm6, %zmm2, %zmm2
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm7
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm7, %ymm8
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm8, %ymm5, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm3, %ymm9
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm8, %ymm8, %ymm8
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm8, %ymm9, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm9, %ymm3, %ymm10
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm8, %ymm8, %ymm8
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm8, %ymm10, %ymm3, %ymm3
@@ -2107,12 +2098,12 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [16909320,16909320,16909320,16909320]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm5
 ; GFNIAVX512VL-NEXT:    vpxor %ymm6, %ymm7, %ymm7
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm7, %ymm7
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm3, %ymm8
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm7
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm7, %ymm8, %ymm3, %ymm3
@@ -2727,7 +2718,7 @@ define <64 x i8> @constant_fshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 define <64 x i8> @splatconstant_fshl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE-LABEL: splatconstant_fshl_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    por %xmm4, %xmm0
@@ -2761,7 +2752,7 @@ define <64 x i8> @splatconstant_fshl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind
 ;
 ; GFNIAVX2-LABEL: splatconstant_fshl_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
@@ -2794,7 +2785,7 @@ declare <64 x i8> @llvm.fshl.v64i8(<64 x i8>, <64 x i8>, <64 x i8>)
 define <64 x i8> @splatconstant_fshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE-LABEL: splatconstant_fshr_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [290499906672525312,290499906672525312]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm4
 ; GFNISSE-NEXT:    pmovsxwq {{.*#+}} xmm9 = [258,258]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
@@ -2824,9 +2815,9 @@ define <64 x i8> @splatconstant_fshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind
 ;
 ; GFNIAVX2-LABEL: splatconstant_fshr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [258,258,258,258]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm2
diff --git a/llvm/test/CodeGen/X86/gfni-rotates.ll b/llvm/test/CodeGen/X86/gfni-rotates.ll
index cc077410228cb..5fd4dfa7cc262 100644
--- a/llvm/test/CodeGen/X86/gfni-rotates.ll
+++ b/llvm/test/CodeGen/X86/gfni-rotates.ll
@@ -415,7 +415,7 @@ define <32 x i8> @var_rotl_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm0
 ; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm6 = [16909320,16909320]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm7
@@ -424,17 +424,17 @@ define <32 x i8> @var_rotl_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNISSE-NEXT:    psllw $5, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm7, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [4647714815446351872,4647714815446351872]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm9
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm9
 ; GFNISSE-NEXT:    por %xmm0, %xmm9
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm9, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm10
@@ -473,26 +473,26 @@ define <32 x i8> @var_rotl_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNIAVX1-LABEL: var_rotl_v32i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX1-NEXT:    # xmm3 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm2, %xmm4
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [16909320,16909320]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm6
 ; GFNIAVX1-NEXT:    vpor %xmm4, %xmm6, %xmm4
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm6, %xmm6
 ; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [4647714815446351872,4647714815446351872]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX1-NEXT:    # xmm8 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm2, %xmm9
 ; GFNIAVX1-NEXT:    vpor %xmm7, %xmm9, %xmm7
 ; GFNIAVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
 ; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [9223372036854775808,9223372036854775808]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNIAVX1-NEXT:    # xmm7 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm2, %xmm9
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm10
@@ -519,22 +519,17 @@ define <32 x i8> @var_rotl_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_rotl_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [16909320,16909320,16909320,16909320]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1108169199648,1108169199648,1108169199648,1108169199648]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
@@ -581,7 +576,7 @@ define <32 x i8> @var_rotr_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNISSE-LABEL: var_rotr_v32i8:
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm5
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm0
 ; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm7 = [16909320,16909320]
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm8
@@ -592,16 +587,16 @@ define <32 x i8> @var_rotr_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNISSE-NEXT:    psubb %xmm2, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm5
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [4647714815446351872,4647714815446351872]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm9
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm9
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm10
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm10
 ; GFNISSE-NEXT:    por %xmm9, %xmm10
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm5
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm10
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm10
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm11
@@ -640,10 +635,10 @@ define <32 x i8> @var_rotr_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNIAVX1-LABEL: var_rotr_v32i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX1-NEXT:    # xmm3 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm2, %xmm4
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [16909320,16909320]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm6
 ; GFNIAVX1-NEXT:    vpor %xmm4, %xmm6, %xmm4
@@ -652,16 +647,16 @@ define <32 x i8> @var_rotr_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNIAVX1-NEXT:    vpsubb %xmm6, %xmm7, %xmm6
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm6, %xmm6
 ; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [4647714815446351872,4647714815446351872]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm8
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX1-NEXT:    # xmm9 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm2, %xmm10
 ; GFNIAVX1-NEXT:    vpor %xmm8, %xmm10, %xmm8
 ; GFNIAVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
 ; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm8, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNIAVX1-NEXT:    # xmm8 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm2, %xmm10
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm11
@@ -689,24 +684,19 @@ define <32 x i8> @var_rotr_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_rotr_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [16909320,16909320,16909320,16909320]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; GFNIAVX2-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1108169199648,1108169199648,1108169199648,1108169199648]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
@@ -1075,21 +1065,15 @@ define <32 x i8> @constant_rotr_v32i8(<32 x i8> %a) nounwind {
 define <32 x i8> @splatconstant_rotl_v32i8(<32 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_rotl_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [1161999622378488840,1161999622378488840]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
-; GFNIAVX1-LABEL: splatconstant_rotl_v32i8:
-; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX1-NEXT:    retq
-;
-; GFNIAVX2-LABEL: splatconstant_rotl_v32i8:
-; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [1161999622378488840,1161999622378488840,1161999622378488840,1161999622378488840]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    retq
+; GFNIAVX1OR2-LABEL: splatconstant_rotl_v32i8:
+; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_rotl_v32i8:
 ; GFNIAVX512:       # %bb.0:
@@ -1103,21 +1087,15 @@ declare <32 x i8> @llvm.fshl.v32i8(<32 x i8>, <32 x i8>, <32 x i8>)
 define <32 x i8> @splatconstant_rotr_v32i8(<32 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_rotr_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [4647715923615551520,4647715923615551520]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
-; GFNIAVX1-LABEL: splatconstant_rotr_v32i8:
-; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX1-NEXT:    retq
-;
-; GFNIAVX2-LABEL: splatconstant_rotr_v32i8:
-; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4647715923615551520,4647715923615551520,4647715923615551520,4647715923615551520]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    retq
+; GFNIAVX1OR2-LABEL: splatconstant_rotr_v32i8:
+; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_rotr_v32i8:
 ; GFNIAVX512:       # %bb.0:
@@ -1137,7 +1115,7 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
 ; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm10 = [16909320,16909320]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm11
@@ -1146,17 +1124,17 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNISSE-NEXT:    psllw $5, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm11, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [4647714815446351872,4647714815446351872]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm12 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm12 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm13
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm13
 ; GFNISSE-NEXT:    por %xmm0, %xmm13
 ; GFNISSE-NEXT:    paddb %xmm8, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm13, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm13 = [9223372036854775808,9223372036854775808]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm13 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm14
@@ -1243,26 +1221,26 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNIAVX1-LABEL: var_rotl_v64i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm6, %xmm7
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [16909320,16909320]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm6, %xmm8
 ; GFNIAVX1-NEXT:    vpor %xmm7, %xmm8, %xmm7
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm8
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm8, %xmm8
 ; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm7, %xmm6, %xmm9
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [4647714815446351872,4647714815446351872]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNIAVX1-NEXT:    # xmm6 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm9, %xmm10
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX1-NEXT:    # xmm7 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm9, %xmm11
 ; GFNIAVX1-NEXT:    vpor %xmm10, %xmm11, %xmm10
 ; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm11
 ; GFNIAVX1-NEXT:    vpblendvb %xmm11, %xmm10, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNIAVX1-NEXT:    # xmm8 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm9, %xmm10
 ; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm12
@@ -1322,21 +1300,21 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_rotl_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm5
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm7
 ; GFNIAVX2-NEXT:    vpor %ymm5, %ymm7, %ymm5
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm7
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm0, %ymm9
 ; GFNIAVX2-NEXT:    vpor %ymm7, %ymm9, %ymm7
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm7, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm0, %ymm9
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm10
 ; GFNIAVX2-NEXT:    vpor %ymm9, %ymm10, %ymm9
@@ -1362,22 +1340,22 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNIAVX512VL-LABEL: var_rotl_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm2, %ymm4
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [16909320,16909320,16909320,16909320]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm2, %ymm6
 ; GFNIAVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm6, %ymm6
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm4, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm7
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm2, %ymm9
 ; GFNIAVX512VL-NEXT:    vpor %ymm7, %ymm9, %ymm7
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm7, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm2, %ymm9
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm10
 ; GFNIAVX512VL-NEXT:    vpor %ymm9, %ymm10, %ymm9
@@ -1422,7 +1400,7 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNISSE-LABEL: var_rotr_v64i8:
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm9
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm0
 ; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm11 = [16909320,16909320]
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm12
@@ -1433,16 +1411,16 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNISSE-NEXT:    psubb %xmm4, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm9
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [4647714815446351872,4647714815446351872]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm13
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm13
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm12 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm12 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm14
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm14
 ; GFNISSE-NEXT:    por %xmm13, %xmm14
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm14, %xmm9
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm13 = [9223372036854775808,9223372036854775808]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm13 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm14
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm14
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm15
@@ -1527,10 +1505,10 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNIAVX1-LABEL: var_rotr_v64i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm7, %xmm6
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [16909320,16909320]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm7, %xmm8
 ; GFNIAVX1-NEXT:    vpor %xmm6, %xmm8, %xmm8
@@ -1539,16 +1517,16 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNIAVX1-NEXT:    vpsubb %xmm9, %xmm6, %xmm9
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm9, %xmm9
 ; GFNIAVX1-NEXT:    vpblendvb %xmm9, %xmm8, %xmm7, %xmm10
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [4647714815446351872,4647714815446351872]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNIAVX1-NEXT:    # xmm7 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm10, %xmm11
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX1-NEXT:    # xmm8 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm10, %xmm12
 ; GFNIAVX1-NEXT:    vpor %xmm11, %xmm12, %xmm11
 ; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm12
 ; GFNIAVX1-NEXT:    vpblendvb %xmm12, %xmm11, %xmm10, %xmm10
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNIAVX1-NEXT:    # xmm9 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm10, %xmm11
 ; GFNIAVX1-NEXT:    vpaddb %xmm10, %xmm10, %xmm13
@@ -1611,23 +1589,23 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_rotr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm5
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm7
 ; GFNIAVX2-NEXT:    vpor %ymm5, %ymm7, %ymm5
 ; GFNIAVX2-NEXT:    vpxor %xmm7, %xmm7, %xmm7
 ; GFNIAVX2-NEXT:    vpsubb %ymm2, %ymm7, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64,0,0,0,0,0,0,128,64]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm8
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm9, %ymm0, %ymm10
 ; GFNIAVX2-NEXT:    vpor %ymm8, %ymm10, %ymm8
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm8, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm0, %ymm10
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm11
 ; GFNIAVX2-NEXT:    vpor %ymm10, %ymm11, %ymm10
@@ -1654,24 +1632,24 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNIAVX512VL-LABEL: var_rotr_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [16909320,16909320,16909320,16909320]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm2, %ymm4
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm2, %ymm6
 ; GFNIAVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm6, %ymm6
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm4, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [258,258,258,258]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm7
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm2, %ymm9
 ; GFNIAVX512VL-NEXT:    vpor %ymm7, %ymm9, %ymm7
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm7, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [1,1,1,1]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm2, %ymm9
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm10, %ymm2, %ymm11
 ; GFNIAVX512VL-NEXT:    vpor %ymm9, %ymm11, %ymm9
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
@@ -2257,7 +2235,7 @@ define <64 x i8> @constant_rotr_v64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @splatconstant_rotl_v64i8(<64 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_rotl_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [9223655728169885760,9223655728169885760]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
@@ -2266,14 +2244,14 @@ define <64 x i8> @splatconstant_rotl_v64i8(<64 x i8> %a) nounwind {
 ;
 ; GFNIAVX1-LABEL: splatconstant_rotl_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [9223655728169885760,9223655728169885760,9223655728169885760,9223655728169885760]
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_rotl_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223655728169885760,9223655728169885760,9223655728169885760,9223655728169885760]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
@@ -2290,7 +2268,7 @@ declare <64 x i8> @llvm.fshl.v64i8(<64 x i8>, <64 x i8>, <64 x i8>)
 define <64 x i8> @splatconstant_rotr_v64i8(<64 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_rotr_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [290499906672525570,290499906672525570]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
@@ -2299,14 +2277,14 @@ define <64 x i8> @splatconstant_rotr_v64i8(<64 x i8> %a) nounwind {
 ;
 ; GFNIAVX1-LABEL: splatconstant_rotr_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [290499906672525570,290499906672525570,290499906672525570,290499906672525570]
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_rotr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [290499906672525570,290499906672525570,290499906672525570,290499906672525570]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/gfni-shifts.ll b/llvm/test/CodeGen/X86/gfni-shifts.ll
index fc57b84ab9f02..6ed524e406826 100644
--- a/llvm/test/CodeGen/X86/gfni-shifts.ll
+++ b/llvm/test/CodeGen/X86/gfni-shifts.ll
@@ -615,7 +615,7 @@ define <32 x i8> @var_shl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNISSE-NEXT:    psllw $5, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm7
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm7
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
@@ -647,13 +647,13 @@ define <32 x i8> @var_shl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNIAVX1-LABEL: var_shl_v32i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [16909320,16909320]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX1-NEXT:    # xmm3 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm2, %xmm4
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
 ; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm6
 ; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
@@ -675,12 +675,10 @@ define <32 x i8> @var_shl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_shl_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [16909320,16909320,16909320,16909320]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1108169199648,1108169199648,1108169199648,1108169199648]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
@@ -717,19 +715,19 @@ define <32 x i8> @var_lshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm6
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm6
 ; GFNISSE-NEXT:    psllw $5, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [290499906672525312,290499906672525312]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm7
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm7
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm7, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [145249953336295424,145249953336295424]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm8
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
@@ -756,18 +754,18 @@ define <32 x i8> @var_lshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNIAVX1-LABEL: var_lshr_v32i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX1-NEXT:    # xmm3 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm2, %xmm4
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
 ; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [290499906672525312,290499906672525312]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm6
 ; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
 ; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [145249953336295424,145249953336295424]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX1-NEXT:    # xmm6 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm2, %xmm7
 ; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
@@ -786,16 +784,13 @@ define <32 x i8> @var_lshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_lshr_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
@@ -1459,16 +1454,10 @@ define <32 x i8> @splatconstant_shl_v32i8(<32 x i8> %a) nounwind {
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
-; GFNIAVX1-LABEL: splatconstant_shl_v32i8:
-; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX1-NEXT:    retq
-;
-; GFNIAVX2-LABEL: splatconstant_shl_v32i8:
-; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [258,258,258,258]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    retq
+; GFNIAVX1OR2-LABEL: splatconstant_shl_v32i8:
+; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_shl_v32i8:
 ; GFNIAVX512:       # %bb.0:
@@ -1481,21 +1470,15 @@ define <32 x i8> @splatconstant_shl_v32i8(<32 x i8> %a) nounwind {
 define <32 x i8> @splatconstant_lshr_v32i8(<32 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_lshr_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [145249953336295424,145249953336295424]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
-; GFNIAVX1-LABEL: splatconstant_lshr_v32i8:
-; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX1-NEXT:    retq
-;
-; GFNIAVX2-LABEL: splatconstant_lshr_v32i8:
-; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    retq
+; GFNIAVX1OR2-LABEL: splatconstant_lshr_v32i8:
+; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_lshr_v32i8:
 ; GFNIAVX512:       # %bb.0:
@@ -1508,21 +1491,15 @@ define <32 x i8> @splatconstant_lshr_v32i8(<32 x i8> %a) nounwind {
 define <32 x i8> @splatconstant_ashr_v32i8(<32 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_ashr_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [290499906672558208,290499906672558208]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [128,128,128,64,32,16,8,4,128,128,128,64,32,16,8,4]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
-; GFNIAVX1-LABEL: splatconstant_ashr_v32i8:
-; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX1-NEXT:    retq
-;
-; GFNIAVX2-LABEL: splatconstant_ashr_v32i8:
-; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [290499906672558208,290499906672558208,290499906672558208,290499906672558208]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    retq
+; GFNIAVX1OR2-LABEL: splatconstant_ashr_v32i8:
+; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_ashr_v32i8:
 ; GFNIAVX512:       # %bb.0:
@@ -1547,7 +1524,7 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE-NEXT:    psllw $5, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm11
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm11
 ; GFNISSE-NEXT:    paddb %xmm8, %xmm8
@@ -1609,13 +1586,13 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX1-LABEL: var_shl_v64i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [16909320,16909320]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm5, %xmm6
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm6, %xmm5, %xmm6
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm6, %xmm8
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
@@ -1658,11 +1635,11 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_shl_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm5
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm6
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm6, %ymm0, %ymm0
@@ -1683,12 +1660,12 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX512VL-LABEL: var_shl_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [16909320,16909320,16909320,16909320]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0,8,4,2,1,0,0,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm2, %ymm4
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm5, %ymm5
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0,32,16,8,4,2,1,0,0]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm6
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm6, %ymm2, %ymm2
@@ -1728,19 +1705,19 @@ define <64 x i8> @var_lshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm10
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm10
 ; GFNISSE-NEXT:    psllw $5, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [290499906672525312,290499906672525312]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm11
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm11
 ; GFNISSE-NEXT:    paddb %xmm8, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm11, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [145249953336295424,145249953336295424]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm12
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm12
 ; GFNISSE-NEXT:    paddb %xmm8, %xmm8
@@ -1797,18 +1774,18 @@ define <64 x i8> @var_lshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX1-LABEL: var_lshr_v64i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm5, %xmm6
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm6, %xmm5, %xmm6
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [290499906672525312,290499906672525312]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm6, %xmm8
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm8, %xmm6, %xmm8
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [145249953336295424,145249953336295424]
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX1-NEXT:    # xmm6 = mem[0,0]
 ; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm8, %xmm9
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
@@ -1848,15 +1825,15 @@ define <64 x i8> @var_lshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_lshr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm5
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm6
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm6, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm7
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm7, %ymm0, %ymm0
@@ -1874,16 +1851,16 @@ define <64 x i8> @var_lshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX512VL-LABEL: var_lshr_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm2, %ymm4
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm5, %ymm5
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm6
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm6, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2,0,128,64,32,16,8,4,2]
 ; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm2, %ymm7
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm7, %ymm2, %ymm2
@@ -2999,7 +2976,7 @@ define <64 x i8> @splatconstant_shl_v64i8(<64 x i8> %a) nounwind {
 ;
 ; GFNIAVX2-LABEL: splatconstant_shl_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [66052,66052,66052,66052]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4,2,1,0,0,0,0,0,4,2,1,0,0,0,0,0,4,2,1,0,0,0,0,0,4,2,1,0,0,0,0,0]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
@@ -3015,7 +2992,7 @@ define <64 x i8> @splatconstant_shl_v64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @splatconstant_lshr_v64i8(<64 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_lshr_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
@@ -3031,7 +3008,7 @@ define <64 x i8> @splatconstant_lshr_v64i8(<64 x i8> %a) nounwind {
 ;
 ; GFNIAVX2-LABEL: splatconstant_lshr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
@@ -3047,7 +3024,7 @@ define <64 x i8> @splatconstant_lshr_v64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @splatconstant_ashr_v64i8(<64 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_ashr_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [145249953336295552,145249953336295552]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [128,128,64,32,16,8,4,2,128,128,64,32,16,8,4,2]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
@@ -3063,7 +3040,7 @@ define <64 x i8> @splatconstant_ashr_v64i8(<64 x i8> %a) nounwind {
 ;
 ; GFNIAVX2-LABEL: splatconstant_ashr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [145249953336295552,145249953336295552,145249953336295552,145249953336295552]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [128,128,64,32,16,8,4,2,128,128,64,32,16,8,4,2,128,128,64,32,16,8,4,2,128,128,64,32,16,8,4,2]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-bitreverse.ll b/llvm/test/CodeGen/X86/vector-bitreverse.ll
index c54a7f4642253..5dcf19013f0b7 100644
--- a/llvm/test/CodeGen/X86/vector-bitreverse.ll
+++ b/llvm/test/CodeGen/X86/vector-bitreverse.ll
@@ -875,27 +875,15 @@ define <32 x i8> @test_bitreverse_v32i8(<32 x i8> %a) nounwind {
 ;
 ; GFNISSE-LABEL: test_bitreverse_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [9241421688590303745,9241421688590303745]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
-; GFNIAVX1-LABEL: test_bitreverse_v32i8:
-; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX1-NEXT:    retq
-;
-; GFNIAVX2-LABEL: test_bitreverse_v32i8:
-; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    retq
-;
-; GFNIAVX512-LABEL: test_bitreverse_v32i8:
-; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
-; GFNIAVX512-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
-; GFNIAVX512-NEXT:    retq
+; GFNIAVX-LABEL: test_bitreverse_v32i8:
+; GFNIAVX:       # %bb.0:
+; GFNIAVX-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX-NEXT:    retq
   %b = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a)
   ret <32 x i8> %b
 }
@@ -1058,7 +1046,7 @@ define <16 x i16> @test_bitreverse_v16i16(<16 x i16> %a) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
 ; GFNISSE-NEXT:    pshufb %xmm2, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm3 = [9241421688590303745,9241421688590303745]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm3, %xmm0
 ; GFNISSE-NEXT:    pshufb %xmm2, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm3, %xmm1
@@ -1071,21 +1059,20 @@ define <16 x i16> @test_bitreverse_v16i16(<16 x i16> %a) nounwind {
 ; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: test_bitreverse_v16i16:
 ; GFNIAVX2:       # %bb.0:
 ; GFNIAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14,17,16,19,18,21,20,23,22,25,24,27,26,29,28,31,30]
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: test_bitreverse_v16i16:
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14,17,16,19,18,21,20,23,22,25,24,27,26,29,28,31,30]
-; GFNIAVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
-; GFNIAVX512-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX512-NEXT:    retq
   %b = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a)
   ret <16 x i16> %b
@@ -1258,7 +1245,7 @@ define <8 x i32> @test_bitreverse_v8i32(<8 x i32> %a) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
 ; GFNISSE-NEXT:    pshufb %xmm2, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm3 = [9241421688590303745,9241421688590303745]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm3, %xmm0
 ; GFNISSE-NEXT:    pshufb %xmm2, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm3, %xmm1
@@ -1271,21 +1258,20 @@ define <8 x i32> @test_bitreverse_v8i32(<8 x i32> %a) nounwind {
 ; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: test_bitreverse_v8i32:
 ; GFNIAVX2:       # %bb.0:
 ; GFNIAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,19,18,17,16,23,22,21,20,27,26,25,24,31,30,29,28]
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: test_bitreverse_v8i32:
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,19,18,17,16,23,22,21,20,27,26,25,24,31,30,29,28]
-; GFNIAVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
-; GFNIAVX512-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX512-NEXT:    retq
   %b = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a)
   ret <8 x i32> %b
@@ -1462,7 +1448,7 @@ define <4 x i64> @test_bitreverse_v4i64(<4 x i64> %a) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
 ; GFNISSE-NEXT:    pshufb %xmm2, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm3 = [9241421688590303745,9241421688590303745]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm3, %xmm0
 ; GFNISSE-NEXT:    pshufb %xmm2, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm3, %xmm1
@@ -1475,21 +1461,20 @@ define <4 x i64> @test_bitreverse_v4i64(<4 x i64> %a) nounwind {
 ; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: test_bitreverse_v4i64:
 ; GFNIAVX2:       # %bb.0:
 ; GFNIAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,23,22,21,20,19,18,17,16,31,30,29,28,27,26,25,24]
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: test_bitreverse_v4i64:
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,23,22,21,20,19,18,17,16,31,30,29,28,27,26,25,24]
-; GFNIAVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
-; GFNIAVX512-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX512-NEXT:    retq
   %b = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a)
   ret <4 x i64> %b
@@ -1741,7 +1726,7 @@ define <64 x i8> @test_bitreverse_v64i8(<64 x i8> %a) nounwind {
 ;
 ; GFNISSE-LABEL: test_bitreverse_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [9241421688590303745,9241421688590303745]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
@@ -1757,7 +1742,7 @@ define <64 x i8> @test_bitreverse_v64i8(<64 x i8> %a) nounwind {
 ;
 ; GFNIAVX2-LABEL: test_bitreverse_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
@@ -2054,7 +2039,7 @@ define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
 ; GFNISSE-NEXT:    pshufb %xmm4, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [9241421688590303745,9241421688590303745]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm0
 ; GFNISSE-NEXT:    pshufb %xmm4, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm1
@@ -2085,7 +2070,7 @@ define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
 ; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14,1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
 ; GFNIAVX2-NEXT:    # ymm2 = mem[0,1,0,1]
 ; GFNIAVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm1, %ymm1
@@ -2412,7 +2397,7 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
 ; GFNISSE-NEXT:    pshufb %xmm4, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [9241421688590303745,9241421688590303745]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm0
 ; GFNISSE-NEXT:    pshufb %xmm4, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm1
@@ -2443,7 +2428,7 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
 ; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
 ; GFNIAVX2-NEXT:    # ymm2 = mem[0,1,0,1]
 ; GFNIAVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm1, %ymm1
@@ -2778,7 +2763,7 @@ define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
 ; GFNISSE-NEXT:    pshufb %xmm4, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [9241421688590303745,9241421688590303745]
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm0
 ; GFNISSE-NEXT:    pshufb %xmm4, %xmm1
 ; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm1
@@ -2809,7 +2794,7 @@ define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
 ; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
 ; GFNIAVX2-NEXT:    # ymm2 = mem[0,1,0,1]
 ; GFNIAVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm1, %ymm1

From 9fae0c6f9c05915a5daac5b368258a40e1fab237 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Thu, 18 Jul 2024 15:30:42 +0200
Subject: [PATCH 024/486] Reapply "[clang][Interp] Fix CheckCallable for
 undefined-and-not-constexpr fns"

This reverts commit ad7aeb0ff58ebd29f68adb85c64e8010639e2a76.
---
 clang/lib/AST/Interp/Interp.cpp | 87 +++++++++++++++++----------------
 clang/lib/AST/Interp/Interp.h   |  6 +--
 clang/test/AST/Interp/cxx2a.cpp | 15 ++++++
 3 files changed, 64 insertions(+), 44 deletions(-)
 create mode 100644 clang/test/AST/Interp/cxx2a.cpp

diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index 2be9b5360d055..e6e9298982887 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -579,57 +579,62 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) {
     return false;
   }
 
-  if (!F->isConstexpr() || !F->hasBody()) {
-    const SourceLocation &Loc = S.Current->getLocation(OpPC);
-    if (S.getLangOpts().CPlusPlus11) {
-      const FunctionDecl *DiagDecl = F->getDecl();
+  if (F->isConstexpr() && F->hasBody() &&
+      (F->getDecl()->isConstexpr() || F->getDecl()->hasAttr<MSConstexprAttr>()))
+    return true;
 
-      // Invalid decls have been diagnosed before.
-      if (DiagDecl->isInvalidDecl())
-        return false;
+  // Implicitly constexpr.
+  if (F->isLambdaStaticInvoker())
+    return true;
 
-      // If this function is not constexpr because it is an inherited
-      // non-constexpr constructor, diagnose that directly.
-      const auto *CD = dyn_cast<CXXConstructorDecl>(DiagDecl);
-      if (CD && CD->isInheritingConstructor()) {
-        const auto *Inherited = CD->getInheritedConstructor().getConstructor();
-        if (!Inherited->isConstexpr())
-          DiagDecl = CD = Inherited;
-      }
+  const SourceLocation &Loc = S.Current->getLocation(OpPC);
+  if (S.getLangOpts().CPlusPlus11) {
+    const FunctionDecl *DiagDecl = F->getDecl();
+
+    // Invalid decls have been diagnosed before.
+    if (DiagDecl->isInvalidDecl())
+      return false;
+
+    // If this function is not constexpr because it is an inherited
+    // non-constexpr constructor, diagnose that directly.
+    const auto *CD = dyn_cast<CXXConstructorDecl>(DiagDecl);
+    if (CD && CD->isInheritingConstructor()) {
+      const auto *Inherited = CD->getInheritedConstructor().getConstructor();
+      if (!Inherited->isConstexpr())
+        DiagDecl = CD = Inherited;
+    }
 
-      // FIXME: If DiagDecl is an implicitly-declared special member function
-      // or an inheriting constructor, we should be much more explicit about why
-      // it's not constexpr.
-      if (CD && CD->isInheritingConstructor()) {
-        S.FFDiag(Loc, diag::note_constexpr_invalid_inhctor, 1)
+    // FIXME: If DiagDecl is an implicitly-declared special member function
+    // or an inheriting constructor, we should be much more explicit about why
+    // it's not constexpr.
+    if (CD && CD->isInheritingConstructor()) {
+      S.FFDiag(Loc, diag::note_constexpr_invalid_inhctor, 1)
           << CD->getInheritedConstructor().getConstructor()->getParent();
-        S.Note(DiagDecl->getLocation(), diag::note_declared_at);
-      } else {
-        // Don't emit anything if the function isn't defined and we're checking
-        // for a constant expression. It might be defined at the point we're
-        // actually calling it.
-        bool IsExtern = DiagDecl->getStorageClass() == SC_Extern;
-        if (!DiagDecl->isDefined() && !IsExtern &&
-            S.checkingPotentialConstantExpression())
-          return false;
+      S.Note(DiagDecl->getLocation(), diag::note_declared_at);
+    } else {
+      // Don't emit anything if the function isn't defined and we're checking
+      // for a constant expression. It might be defined at the point we're
+      // actually calling it.
+      bool IsExtern = DiagDecl->getStorageClass() == SC_Extern;
+      if (!DiagDecl->isDefined() && !IsExtern && DiagDecl->isConstexpr() &&
+          S.checkingPotentialConstantExpression())
+        return false;
 
-        // If the declaration is defined, declared 'constexpr' _and_ has a body,
-        // the below diagnostic doesn't add anything useful.
-        if (DiagDecl->isDefined() && DiagDecl->isConstexpr() &&
-            DiagDecl->hasBody())
-          return false;
+      // If the declaration is defined, declared 'constexpr' _and_ has a body,
+      // the below diagnostic doesn't add anything useful.
+      if (DiagDecl->isDefined() && DiagDecl->isConstexpr() &&
+          DiagDecl->hasBody())
+        return false;
 
-        S.FFDiag(Loc, diag::note_constexpr_invalid_function, 1)
+      S.FFDiag(Loc, diag::note_constexpr_invalid_function, 1)
           << DiagDecl->isConstexpr() << (bool)CD << DiagDecl;
-        S.Note(DiagDecl->getLocation(), diag::note_declared_at);
-      }
-    } else {
-      S.FFDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
+      S.Note(DiagDecl->getLocation(), diag::note_declared_at);
     }
-    return false;
+  } else {
+    S.FFDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
   }
 
-  return true;
+  return false;
 }
 
 bool CheckCallDepth(InterpState &S, CodePtr OpPC) {
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index 17b3157cb40a9..2e159012f5ffd 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -2531,14 +2531,14 @@ inline bool Call(InterpState &S, CodePtr OpPC, const Function *Func,
       if (!CheckInvoke(S, OpPC, ThisPtr))
         return false;
     }
-
-    if (S.checkingPotentialConstantExpression())
-      return false;
   }
 
   if (!CheckCallable(S, OpPC, Func))
     return false;
 
+  if (Func->hasThisPointer() && S.checkingPotentialConstantExpression())
+    return false;
+
   if (!CheckCallDepth(S, OpPC))
     return false;
 
diff --git a/clang/test/AST/Interp/cxx2a.cpp b/clang/test/AST/Interp/cxx2a.cpp
new file mode 100644
index 0000000000000..27d1aa1a27f75
--- /dev/null
+++ b/clang/test/AST/Interp/cxx2a.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -std=c++2a -fsyntax-only -fcxx-exceptions -verify=ref,both %s
+// RUN: %clang_cc1 -std=c++2a -fsyntax-only -fcxx-exceptions -verify=expected,both %s -fexperimental-new-constant-interpreter
+
+template <unsigned N>
+struct S {
+  S() requires (N==1) = default;
+  S() requires (N==2) {} // both-note {{declared here}}
+  consteval S() requires (N==3) = default;
+};
+
+consteval int aConstevalFunction() { // both-error {{consteval function never produces a constant expression}}
+  S<2> s4; // both-note {{non-constexpr constructor 'S' cannot be used in a constant expression}}
+  return 0;
+}
+/// We're NOT calling the above function. The diagnostics should appear anyway.

From cd495d2cdd84a22026a115c7e9923c27b196732e Mon Sep 17 00:00:00 2001
From: Utkarsh Saxena <usx@google.com>
Date: Thu, 18 Jul 2024 16:18:34 +0200
Subject: [PATCH 025/486] Add source file name for template instantiations in
 -ftime-trace (#98320)

This is helpful in identifying file and location which contain the particular template declaration.
---
 a-abfdec1d.o.tmp                              |   0
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/Driver/Options.td         |   4 +
 .../include/clang/Frontend/FrontendOptions.h  |   8 +-
 clang/lib/Driver/ToolChains/Clang.cpp         |   1 +
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  11 +-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |  11 +-
 clang/test/Driver/ftime-trace-sections.cpp    |   2 +-
 clang/test/Driver/ftime-trace.cpp             |  39 +++---
 clang/tools/driver/cc1_main.cpp               |   3 +-
 clang/unittests/Support/TimeProfilerTest.cpp  | 121 ++++++++++++++----
 llvm/include/llvm/Support/TimeProfiler.h      |  23 +++-
 llvm/lib/Support/TimeProfiler.cpp             |  61 +++++++--
 13 files changed, 223 insertions(+), 64 deletions(-)
 create mode 100644 a-abfdec1d.o.tmp

diff --git a/a-abfdec1d.o.tmp b/a-abfdec1d.o.tmp
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index e0e86af257a19..971df672b6ca1 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -736,6 +736,9 @@ Improvements to Clang's time-trace
 - Clang now specifies that using ``auto`` in a lambda parameter is a C++14 extension when
   appropriate. (`#46059: <https://github.com/llvm/llvm-project/issues/46059>`_).
 
+- Clang now adds source file infomation for template instantiations as ``event["args"]["filename"]``. This
+  added behind an option ``-ftime-trace-verbose``. This is expected to increase the size of trace by 2-3 times.
+
 Improvements to Coverage Mapping
 --------------------------------
 
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 1675e435d210c..d3068c1b30a7a 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3988,6 +3988,10 @@ def ftime_trace_granularity_EQ : Joined<["-"], "ftime-trace-granularity=">, Grou
   HelpText<"Minimum time granularity (in microseconds) traced by time profiler">,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
   MarshallingInfoInt<FrontendOpts<"TimeTraceGranularity">, "500u">;
+def ftime_trace_verbose : Joined<["-"], "ftime-trace-verbose">, Group<f_Group>,
+  HelpText<"Make time trace capture verbose event details (e.g. source filenames). This can increase the size of the output by 2-3 times">,
+  Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
+  MarshallingInfoFlag<FrontendOpts<"TimeTraceVerbose">>;
 def ftime_trace_EQ : Joined<["-"], "ftime-trace=">, Group<f_Group>,
   HelpText<"Similar to -ftime-trace. Specify the JSON file or a directory which will contain the JSON file">,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 5e5034fe01eb5..8241925c98476 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -580,6 +580,11 @@ class FrontendOptions {
   /// Minimum time granularity (in microseconds) traced by time profiler.
   unsigned TimeTraceGranularity;
 
+  /// Make time trace capture verbose event details (e.g. source filenames).
+  /// This can increase the size of the output by 2-3 times.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned TimeTraceVerbose : 1;
+
   /// Path which stores the output files for -ftime-trace
   std::string TimeTracePath;
 
@@ -601,7 +606,8 @@ class FrontendOptions {
         EmitSymbolGraph(false), EmitExtensionSymbolGraphs(false),
         EmitSymbolGraphSymbolLabelsForTesting(false),
         EmitPrettySymbolGraphs(false), GenReducedBMI(false),
-        UseClangIRPipeline(false), TimeTraceGranularity(500) {}
+        UseClangIRPipeline(false), TimeTraceGranularity(500),
+        TimeTraceVerbose(false) {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file
   /// extension. For example, "c" would return Language::C.
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 1fd6fba210042..6b33301d36401 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6754,6 +6754,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (const char *Name = C.getTimeTraceFile(&JA)) {
     CmdArgs.push_back(Args.MakeArgString("-ftime-trace=" + Twine(Name)));
     Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_granularity_EQ);
+    Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_verbose);
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_ftrapv_handler_EQ)) {
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index a7bc6749c5852..725b62db5e80a 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -3426,11 +3426,16 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
     return true;
 
   llvm::TimeTraceScope TimeScope("InstantiateClass", [&]() {
-    std::string Name;
-    llvm::raw_string_ostream OS(Name);
+    llvm::TimeTraceMetadata M;
+    llvm::raw_string_ostream OS(M.Detail);
     Instantiation->getNameForDiagnostic(OS, getPrintingPolicy(),
                                         /*Qualified=*/true);
-    return Name;
+    if (llvm::isTimeTraceVerbose()) {
+      auto Loc = SourceMgr.getExpansionLoc(Instantiation->getLocation());
+      M.File = SourceMgr.getFilename(Loc);
+      M.Line = SourceMgr.getExpansionLineNumber(Loc);
+    }
+    return M;
   });
 
   Pattern = PatternDef;
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 01432301633ed..4e619f4b491a6 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -4966,11 +4966,16 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   }
 
   llvm::TimeTraceScope TimeScope("InstantiateFunction", [&]() {
-    std::string Name;
-    llvm::raw_string_ostream OS(Name);
+    llvm::TimeTraceMetadata M;
+    llvm::raw_string_ostream OS(M.Detail);
     Function->getNameForDiagnostic(OS, getPrintingPolicy(),
                                    /*Qualified=*/true);
-    return Name;
+    if (llvm::isTimeTraceVerbose()) {
+      auto Loc = SourceMgr.getExpansionLoc(Function->getLocation());
+      M.File = SourceMgr.getFilename(Loc);
+      M.Line = SourceMgr.getExpansionLineNumber(Loc);
+    }
+    return M;
   });
 
   // If we're performing recursive template instantiation, create our own
diff --git a/clang/test/Driver/ftime-trace-sections.cpp b/clang/test/Driver/ftime-trace-sections.cpp
index 0c16052bc0c3a..da7109b9d81a6 100644
--- a/clang/test/Driver/ftime-trace-sections.cpp
+++ b/clang/test/Driver/ftime-trace-sections.cpp
@@ -1,5 +1,5 @@
 // RUN: rm -rf %t && mkdir %t && cd %t
-// RUN: %clangxx -S -ftime-trace -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: %python %S/ftime-trace-sections.py < out.json
 
 template <typename T>
diff --git a/clang/test/Driver/ftime-trace.cpp b/clang/test/Driver/ftime-trace.cpp
index 5fe63de915a71..60c5885704b58 100644
--- a/clang/test/Driver/ftime-trace.cpp
+++ b/clang/test/Driver/ftime-trace.cpp
@@ -1,18 +1,18 @@
 // RUN: rm -rf %t && mkdir -p %t && cd %t
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=new-name.json -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=new-name.json -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat new-name.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
 // RUN: mkdir dir1 dir2
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir1 -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir1 -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat dir1/out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir2/ -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir2/ -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat dir2/out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
@@ -34,32 +34,33 @@
 // RUN: mkdir d e f && cp %s d/a.cpp && touch d/b.c
 
 /// TODO: Support -fno-integrated-as.
-// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -fintegrated-as d/a.cpp -o e/a.o 2>&1 | FileCheck %s --check-prefix=COMPILE1
-// COMPILE1: -cc1{{.*}} "-ftime-trace=e/a.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -fintegrated-as d/a.cpp -o e/a.o 2>&1 | FileCheck %s --check-prefix=COMPILE1
+// COMPILE1: -cc1{{.*}} "-ftime-trace=e/a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
-// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=COMPILE2
-// COMPILE2: -cc1{{.*}} "-ftime-trace=f/a.json" "-ftime-trace-granularity=0"
-// COMPILE2: -cc1{{.*}} "-ftime-trace=f/b.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=COMPILE2
+// COMPILE2: -cc1{{.*}} "-ftime-trace=f/a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// COMPILE2: -cc1{{.*}} "-ftime-trace=f/b.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
 /// -o specifies the link output. Create ${output}-${basename}.json.
-// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -o e/x 2>&1 | FileCheck %s --check-prefix=LINK1
-// LINK1: -cc1{{.*}} "-ftime-trace=e/x-a.json" "-ftime-trace-granularity=0"
-// LINK1: -cc1{{.*}} "-ftime-trace=e/x-b.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o e/x 2>&1 | FileCheck %s --check-prefix=LINK1
+// LINK1: -cc1{{.*}} "-ftime-trace=e/x-a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// LINK1: -cc1{{.*}} "-ftime-trace=e/x-b.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
 /// -dumpdir is f/g, not ending with a path separator. We create f/g${basename}.json.
-// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -o e/x -dumpdir f/g 2>&1 | FileCheck %s --check-prefix=LINK2
-// LINK2: -cc1{{.*}} "-ftime-trace=f/ga.json" "-ftime-trace-granularity=0"
-// LINK2: -cc1{{.*}} "-ftime-trace=f/gb.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o e/x -dumpdir f/g 2>&1 | FileCheck %s --check-prefix=LINK2
+// LINK2: -cc1{{.*}} "-ftime-trace=f/ga.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// LINK2: -cc1{{.*}} "-ftime-trace=f/gb.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
-// RUN: %clang -### -ftime-trace=e -ftime-trace-granularity=0 d/a.cpp d/b.c -o f/x -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=LINK3
-// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}a-{{[^.]*}}.json" "-ftime-trace-granularity=0"
-// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}b-{{[^.]*}}.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -ftime-trace=e -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o f/x -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=LINK3
+// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}a-{{[^.]*}}.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}b-{{[^.]*}}.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
-// RUN: %clang -### -ftime-trace -ftime-trace=e -ftime-trace-granularity=1 -xassembler d/a.cpp 2>&1 | \
+// RUN: %clang -### -ftime-trace -ftime-trace=e -ftime-trace-granularity=1 -ftime-trace-verbose -xassembler d/a.cpp 2>&1 | \
 // RUN:   FileCheck %s --check-prefix=UNUSED
 // UNUSED:      warning: argument unused during compilation: '-ftime-trace'
 // UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace=e'
 // UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace-granularity=1'
+// UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace-verbose'
 // UNUSED-NOT:  warning:
 
 template <typename T>
diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp
index c2ccb47a15bc8..f5e5fad36573e 100644
--- a/clang/tools/driver/cc1_main.cpp
+++ b/clang/tools/driver/cc1_main.cpp
@@ -241,7 +241,8 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
 
   if (!Clang->getFrontendOpts().TimeTracePath.empty()) {
     llvm::timeTraceProfilerInitialize(
-        Clang->getFrontendOpts().TimeTraceGranularity, Argv0);
+        Clang->getFrontendOpts().TimeTraceGranularity, Argv0,
+        Clang->getFrontendOpts().TimeTraceVerbose);
   }
   // --print-supported-cpus takes priority over the actual compilation.
   if (Clang->getFrontendOpts().PrintSupportedCPUs)
diff --git a/clang/unittests/Support/TimeProfilerTest.cpp b/clang/unittests/Support/TimeProfilerTest.cpp
index 5f3950ff033f1..96e137508ed94 100644
--- a/clang/unittests/Support/TimeProfilerTest.cpp
+++ b/clang/unittests/Support/TimeProfilerTest.cpp
@@ -10,11 +10,14 @@
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Lex/PreprocessorOptions.h"
 
+#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/TimeProfiler.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include <stack>
 
 #include "gtest/gtest.h"
+#include <tuple>
 
 using namespace clang;
 using namespace llvm;
@@ -23,7 +26,8 @@ namespace {
 
 // Should be called before testing.
 void setupProfiler() {
-  timeTraceProfilerInitialize(/*TimeTraceGranularity=*/0, "test");
+  timeTraceProfilerInitialize(/*TimeTraceGranularity=*/0, "test",
+                              /*TimeTraceVerbose=*/true);
 }
 
 // Should be called after `compileFromString()`.
@@ -38,14 +42,24 @@ std::string teardownProfiler() {
 
 // Returns true if code compiles successfully.
 // We only parse AST here. This is enough for constexpr evaluation.
-bool compileFromString(StringRef Code, StringRef Standard, StringRef FileName) {
+bool compileFromString(StringRef Code, StringRef Standard, StringRef File,
+                       llvm::StringMap<std::string> Headers = {}) {
   CompilerInstance Compiler;
   Compiler.createDiagnostics();
 
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> FS(
+      new llvm::vfs::InMemoryFileSystem());
+  FS->addFile(File, 0, MemoryBuffer::getMemBuffer(Code));
+  for (const auto &Header : Headers) {
+    FS->addFile(Header.getKey(), 0,
+                MemoryBuffer::getMemBuffer(Header.getValue()));
+  }
+  llvm::IntrusiveRefCntPtr<FileManager> Files(
+      new FileManager(FileSystemOptions(), FS));
+  Compiler.setFileManager(Files.get());
+
   auto Invocation = std::make_shared<CompilerInvocation>();
-  Invocation->getPreprocessorOpts().addRemappedFile(
-      FileName, MemoryBuffer::getMemBuffer(Code).release());
-  const char *Args[] = {Standard.data(), FileName.data()};
+  std::vector<const char *> Args = {Standard.data(), File.data()};
   CompilerInvocation::CreateFromArgs(*Invocation, Args,
                                      Compiler.getDiagnostics());
   Compiler.setInvocation(std::move(Invocation));
@@ -60,13 +74,27 @@ bool compileFromString(StringRef Code, StringRef Standard, StringRef FileName) {
   return Compiler.ExecuteAction(Action);
 }
 
+std::string GetMetadata(json::Object *Event) {
+  std::string Metadata;
+  llvm::raw_string_ostream OS(Metadata);
+  if (json::Object *Args = Event->getObject("args")) {
+    if (auto Detail = Args->getString("detail"))
+      OS << Detail->str();
+    if (auto File = Args->getString("file"))
+      OS << ", " << File->str();
+    if (auto Line = Args->getInteger("line"))
+      OS << ":" << *Line;
+  }
+  return Metadata;
+}
+
 // Returns pretty-printed trace graph.
 std::string buildTraceGraph(StringRef Json) {
   struct EventRecord {
     int64_t TimestampBegin;
     int64_t TimestampEnd;
-    StringRef Name;
-    StringRef Detail;
+    std::string Name;
+    std::string Metadata;
   };
   std::vector<EventRecord> Events;
 
@@ -81,10 +109,13 @@ std::string buildTraceGraph(StringRef Json) {
     int64_t TimestampBegin = TraceEventObj->getInteger("ts").value_or(0);
     int64_t TimestampEnd =
         TimestampBegin + TraceEventObj->getInteger("dur").value_or(0);
-    StringRef Name = TraceEventObj->getString("name").value_or("");
-    StringRef Detail = "";
-    if (json::Object *Args = TraceEventObj->getObject("args"))
-      Detail = Args->getString("detail").value_or("");
+    std::string Name = TraceEventObj->getString("name").value_or("").str();
+    std::string Metadata = GetMetadata(TraceEventObj);
+
+    // Source events are asynchronous events and may not perfectly nest the
+    // synchronous events. Skip testing them.
+    if (Name == "Source")
+      continue;
 
     // This is a "summary" event, like "Total PerformPendingInstantiations",
     // skip it
@@ -92,7 +123,7 @@ std::string buildTraceGraph(StringRef Json) {
       continue;
 
     Events.emplace_back(
-        EventRecord{TimestampBegin, TimestampEnd, Name, Detail});
+        EventRecord{TimestampBegin, TimestampEnd, Name, Metadata});
   }
 
   // There can be nested events that are very fast, for example:
@@ -132,9 +163,9 @@ std::string buildTraceGraph(StringRef Json) {
       Stream << "| ";
     }
     Stream.write(Event.Name.data(), Event.Name.size());
-    if (!Event.Detail.empty()) {
+    if (!Event.Metadata.empty()) {
       Stream << " (";
-      Stream.write(Event.Detail.data(), Event.Detail.size());
+      Stream.write(Event.Metadata.data(), Event.Metadata.size());
       Stream << ")";
     }
     Stream << "\n";
@@ -145,7 +176,7 @@ std::string buildTraceGraph(StringRef Json) {
 } // namespace
 
 TEST(TimeProfilerTest, ConstantEvaluationCxx20) {
-  constexpr StringRef Code = R"(
+  std::string Code = R"(
 void print(double value);
 
 namespace slow_namespace {
@@ -175,8 +206,7 @@ constexpr int slow_init_list[] = {1, 1, 2, 3, 5, 8, 13, 21}; // 25th line
   setupProfiler();
   ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc"));
   std::string Json = teardownProfiler();
-  std::string TraceGraph = buildTraceGraph(Json);
-  ASSERT_TRUE(TraceGraph == R"(
+  ASSERT_EQ(R"(
 Frontend
 | ParseDeclarationOrFunctionDefinition (test.cc:2:1)
 | ParseDeclarationOrFunctionDefinition (test.cc:6:1)
@@ -202,14 +232,54 @@ Frontend
 | ParseDeclarationOrFunctionDefinition (test.cc:25:1)
 | | EvaluateAsInitializer (slow_init_list)
 | PerformPendingInstantiations
-)");
+)",
+            buildTraceGraph(Json));
+}
+
+TEST(TimeProfilerTest, TemplateInstantiations) {
+  std::string B_H = R"(
+    template <typename T>
+    T fooB(T t) {
+      return T();
+    }
 
-  // NOTE: If this test is failing, run this test with
-  // `llvm::errs() << TraceGraph;` and change the assert above.
+    #define MacroTemp(x) template <typename T> void foo##x(T) { T(); }
+  )";
+
+  std::string A_H = R"(
+    #include "b.h"
+
+    MacroTemp(MTA)
+
+    template <typename T>
+    void fooA(T t) { fooB(t); fooMTA(t); }
+  )";
+  std::string Code = R"(
+    #include "a.h"
+    void user() { fooA(0); }
+  )";
+
+  setupProfiler();
+  ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc",
+                                /*Headers=*/{{"a.h", A_H}, {"b.h", B_H}}));
+  std::string Json = teardownProfiler();
+  ASSERT_EQ(R"(
+Frontend
+| ParseFunctionDefinition (fooB)
+| ParseFunctionDefinition (fooMTA)
+| ParseFunctionDefinition (fooA)
+| ParseDeclarationOrFunctionDefinition (test.cc:3:5)
+| | ParseFunctionDefinition (user)
+| PerformPendingInstantiations
+| | InstantiateFunction (fooA<int>, ./a.h:7)
+| | | InstantiateFunction (fooB<int>, ./b.h:3)
+| | | InstantiateFunction (fooMTA<int>, ./a.h:4)
+)",
+            buildTraceGraph(Json));
 }
 
 TEST(TimeProfilerTest, ConstantEvaluationC99) {
-  constexpr StringRef Code = R"(
+  std::string Code = R"(
 struct {
   short quantval[4]; // 3rd line
 } value;
@@ -218,15 +288,12 @@ struct {
   setupProfiler();
   ASSERT_TRUE(compileFromString(Code, "-std=c99", "test.c"));
   std::string Json = teardownProfiler();
-  std::string TraceGraph = buildTraceGraph(Json);
-  ASSERT_TRUE(TraceGraph == R"(
+  ASSERT_EQ(R"(
 Frontend
 | ParseDeclarationOrFunctionDefinition (test.c:2:1)
 | | isIntegerConstantExpr (<test.c:3:18>)
 | | EvaluateKnownConstIntCheckOverflow (<test.c:3:18>)
 | PerformPendingInstantiations
-)");
-
-  // NOTE: If this test is failing, run this test with
-  // `llvm::errs() << TraceGraph;` and change the assert above.
+)",
+            buildTraceGraph(Json));
 }
diff --git a/llvm/include/llvm/Support/TimeProfiler.h b/llvm/include/llvm/Support/TimeProfiler.h
index 31f7df10916db..6eb92930b36fd 100644
--- a/llvm/include/llvm/Support/TimeProfiler.h
+++ b/llvm/include/llvm/Support/TimeProfiler.h
@@ -83,16 +83,28 @@ namespace llvm {
 
 class raw_pwrite_stream;
 
+struct TimeTraceMetadata {
+  std::string Detail;
+  // Source file and line number information for the event.
+  std::string File;
+  int Line;
+
+  bool isEmpty() const { return Detail.empty() && File.empty(); }
+};
+
 struct TimeTraceProfiler;
 TimeTraceProfiler *getTimeTraceProfilerInstance();
 
+bool isTimeTraceVerbose();
+
 struct TimeTraceProfilerEntry;
 
 /// Initialize the time trace profiler.
 /// This sets up the global \p TimeTraceProfilerInstance
 /// variable to be the profiler instance.
 void timeTraceProfilerInitialize(unsigned TimeTraceGranularity,
-                                 StringRef ProcName);
+                                 StringRef ProcName,
+                                 bool TimeTraceVerbose = false);
 
 /// Cleanup the time trace profiler, if it was initialized.
 void timeTraceProfilerCleanup();
@@ -128,6 +140,10 @@ TimeTraceProfilerEntry *
 timeTraceProfilerBegin(StringRef Name,
                        llvm::function_ref<std::string()> Detail);
 
+TimeTraceProfilerEntry *
+timeTraceProfilerBegin(StringRef Name,
+                       llvm::function_ref<TimeTraceMetadata()> MetaData);
+
 /// Manually begin a time section, with the given \p Name and \p Detail.
 /// This starts Async Events having \p Name as a category which is shown
 /// separately from other traces. See
@@ -164,6 +180,11 @@ class TimeTraceScope {
     if (getTimeTraceProfilerInstance() != nullptr)
       Entry = timeTraceProfilerBegin(Name, Detail);
   }
+  TimeTraceScope(StringRef Name,
+                 llvm::function_ref<TimeTraceMetadata()> Metadata) {
+    if (getTimeTraceProfilerInstance() != nullptr)
+      Entry = timeTraceProfilerBegin(Name, Metadata);
+  }
   ~TimeTraceScope() {
     if (getTimeTraceProfilerInstance() != nullptr)
       timeTraceProfilerEnd(Entry);
diff --git a/llvm/lib/Support/TimeProfiler.cpp b/llvm/lib/Support/TimeProfiler.cpp
index 9612db7d30f98..c2014028ddadc 100644
--- a/llvm/lib/Support/TimeProfiler.cpp
+++ b/llvm/lib/Support/TimeProfiler.cpp
@@ -73,12 +73,20 @@ struct llvm::TimeTraceProfilerEntry {
   const TimePointType Start;
   TimePointType End;
   const std::string Name;
-  const std::string Detail;
+  TimeTraceMetadata Metadata;
+
   const bool AsyncEvent = false;
   TimeTraceProfilerEntry(TimePointType &&S, TimePointType &&E, std::string &&N,
                          std::string &&Dt, bool Ae)
+      : Start(std::move(S)), End(std::move(E)), Name(std::move(N)), Metadata(),
+        AsyncEvent(Ae) {
+    Metadata.Detail = std::move(Dt);
+  }
+
+  TimeTraceProfilerEntry(TimePointType &&S, TimePointType &&E, std::string &&N,
+                         TimeTraceMetadata &&Mt, bool Ae)
       : Start(std::move(S)), End(std::move(E)), Name(std::move(N)),
-        Detail(std::move(Dt)), AsyncEvent(Ae) {}
+        Metadata(std::move(Mt)), AsyncEvent(Ae) {}
 
   // Calculate timings for FlameGraph. Cast time points to microsecond precision
   // rather than casting duration. This avoids truncation issues causing inner
@@ -97,10 +105,12 @@ struct llvm::TimeTraceProfilerEntry {
 };
 
 struct llvm::TimeTraceProfiler {
-  TimeTraceProfiler(unsigned TimeTraceGranularity = 0, StringRef ProcName = "")
+  TimeTraceProfiler(unsigned TimeTraceGranularity = 0, StringRef ProcName = "",
+                    bool TimeTraceVerbose = false)
       : BeginningOfTime(system_clock::now()), StartTime(ClockType::now()),
         ProcName(ProcName), Pid(sys::Process::getProcessId()),
-        Tid(llvm::get_threadid()), TimeTraceGranularity(TimeTraceGranularity) {
+        Tid(llvm::get_threadid()), TimeTraceGranularity(TimeTraceGranularity),
+        TimeTraceVerbose(TimeTraceVerbose) {
     llvm::get_thread_name(ThreadName);
   }
 
@@ -113,6 +123,15 @@ struct llvm::TimeTraceProfiler {
     return Stack.back().get();
   }
 
+  TimeTraceProfilerEntry *
+  begin(std::string Name, llvm::function_ref<TimeTraceMetadata()> Metadata,
+        bool AsyncEvent = false) {
+    Stack.emplace_back(std::make_unique<TimeTraceProfilerEntry>(
+        ClockType::now(), TimePointType(), std::move(Name), Metadata(),
+        AsyncEvent));
+    return Stack.back().get();
+  }
+
   void end() {
     assert(!Stack.empty() && "Must call begin() first");
     end(*Stack.back());
@@ -184,8 +203,15 @@ struct llvm::TimeTraceProfiler {
           J.attribute("dur", DurUs);
         }
         J.attribute("name", E.Name);
-        if (!E.Detail.empty()) {
-          J.attributeObject("args", [&] { J.attribute("detail", E.Detail); });
+        if (!E.Metadata.isEmpty()) {
+          J.attributeObject("args", [&] {
+            if (!E.Metadata.Detail.empty())
+              J.attribute("detail", E.Metadata.Detail);
+            if (!E.Metadata.File.empty())
+              J.attribute("file", E.Metadata.File);
+            if (E.Metadata.Line > 0)
+              J.attribute("line", E.Metadata.Line);
+          });
         }
       });
 
@@ -307,14 +333,25 @@ struct llvm::TimeTraceProfiler {
 
   // Minimum time granularity (in microseconds)
   const unsigned TimeTraceGranularity;
+
+  // Make time trace capture verbose event details (e.g. source filenames). This
+  // can increase the size of the output by 2-3 times.
+  const bool TimeTraceVerbose;
 };
 
+bool llvm::isTimeTraceVerbose() {
+  return getTimeTraceProfilerInstance() &&
+         getTimeTraceProfilerInstance()->TimeTraceVerbose;
+}
+
 void llvm::timeTraceProfilerInitialize(unsigned TimeTraceGranularity,
-                                       StringRef ProcName) {
+                                       StringRef ProcName,
+                                       bool TimeTraceVerbose) {
   assert(TimeTraceProfilerInstance == nullptr &&
          "Profiler should not be initialized");
   TimeTraceProfilerInstance = new TimeTraceProfiler(
-      TimeTraceGranularity, llvm::sys::path::filename(ProcName));
+      TimeTraceGranularity, llvm::sys::path::filename(ProcName),
+      TimeTraceVerbose);
 }
 
 // Removes all TimeTraceProfilerInstances.
@@ -381,6 +418,14 @@ llvm::timeTraceProfilerBegin(StringRef Name,
   return nullptr;
 }
 
+TimeTraceProfilerEntry *
+llvm::timeTraceProfilerBegin(StringRef Name,
+                             llvm::function_ref<TimeTraceMetadata()> Metadata) {
+  if (TimeTraceProfilerInstance != nullptr)
+    return TimeTraceProfilerInstance->begin(std::string(Name), Metadata, false);
+  return nullptr;
+}
+
 TimeTraceProfilerEntry *llvm::timeTraceAsyncProfilerBegin(StringRef Name,
                                                           StringRef Detail) {
   if (TimeTraceProfilerInstance != nullptr)

From 1cc107234969c33a7036b9694da57f4223e3e4d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Thu, 18 Jul 2024 16:22:37 +0200
Subject: [PATCH 026/486] [GlobalIsel] Add G_SCMP and G_UCMP instructions
 (#98894)

https://github.com/llvm/llvm-project/pull/83227
---
 llvm/docs/GlobalISel/GenericOpcode.rst        | 20 ++++++++++++
 .../CodeGen/GlobalISel/MachineIRBuilder.h     | 28 +++++++++++++++++
 llvm/include/llvm/Support/TargetOpcodes.def   |  6 ++++
 llvm/include/llvm/Target/GenericOpcodes.td    | 14 +++++++++
 .../CodeGen/GlobalISel/MachineIRBuilder.cpp   | 12 +++++++
 llvm/lib/CodeGen/MachineVerifier.cpp          | 30 ++++++++++++++++++
 .../GlobalISel/legalizer-info-validation.mir  |  6 ++++
 llvm/test/MachineVerifier/test_uscmp.mir      | 31 +++++++++++++++++++
 .../match-table-cxx.td                        | 30 +++++++++---------
 llvm/test/TableGen/GlobalISelEmitter.td       |  2 +-
 10 files changed, 163 insertions(+), 16 deletions(-)
 create mode 100644 llvm/test/MachineVerifier/test_uscmp.mir

diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index 18a53a4815722..d32aeff5a69bb 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -348,6 +348,26 @@ G_ICMP
 Perform integer comparison producing non-zero (true) or zero (false). It's
 target specific whether a true value is 1, ~0U, or some other non-zero value.
 
+G_SCMP
+^^^^^^
+
+Perform signed 3-way integer comparison producing -1 (smaller), 0 (equal), or 1 (larger).
+
+.. code-block:: none
+
+  %5:_(s32) = G_SCMP %6, %2
+
+
+G_UCMP
+^^^^^^
+
+Perform unsigned 3-way integer comparison producing -1 (smaller), 0 (equal), or 1 (larger).
+
+.. code-block:: none
+
+  %7:_(s32) = G_UCMP %2, %6
+
+
 G_SELECT
 ^^^^^^^^
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index e74136f34b234..56a77b8596a18 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1273,6 +1273,34 @@ class MachineIRBuilder {
                                 const SrcOp &Op0, const SrcOp &Op1,
                                 std::optional<unsigned> Flags = std::nullopt);
 
+  /// Build and insert a \p Res = G_SCMP \p Op0, \p Op1
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+
+  /// \pre \p Res must be a generic virtual register with scalar or
+  ///      vector type. Typically this starts as s2 or <N x s2>.
+  /// \pre \p Op0 and Op1 must be generic virtual registers with the
+  ///      same number of elements as \p Res. If \p Res is a scalar,
+  ///      \p Op0 must be a scalar.
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildSCmp(const DstOp &Res, const SrcOp &Op0,
+                                const SrcOp &Op1);
+
+  /// Build and insert a \p Res = G_UCMP \p Op0, \p Op1
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+
+  /// \pre \p Res must be a generic virtual register with scalar or
+  ///      vector type. Typically this starts as s2 or <N x s2>.
+  /// \pre \p Op0 and Op1 must be generic virtual registers with the
+  ///      same number of elements as \p Res. If \p Res is a scalar,
+  ///      \p Op0 must be a scalar.
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildUCmp(const DstOp &Res, const SrcOp &Op0,
+                                const SrcOp &Op1);
+
   /// Build and insert a \p Res = G_IS_FPCLASS \p Src, \p Mask
   MachineInstrBuilder buildIsFPClass(const DstOp &Res, const SrcOp &Src,
                                      unsigned Mask) {
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index a6672f87af977..9fb6de49fb205 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -503,6 +503,12 @@ HANDLE_TARGET_OPCODE(G_ICMP)
 /// Generic floating-point comparison, also applicable to vectors.
 HANDLE_TARGET_OPCODE(G_FCMP)
 
+/// Generic signed 3-way comparison.
+HANDLE_TARGET_OPCODE(G_SCMP)
+
+/// Generic unsigned 3-way comparison.
+HANDLE_TARGET_OPCODE(G_UCMP)
+
 /// Generic select.
 HANDLE_TARGET_OPCODE(G_SELECT)
 
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 7501048dfdd78..36a0a087ba457 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -430,6 +430,20 @@ def G_FCMP : GenericInstruction {
   let hasSideEffects = false;
 }
 
+// Generic signed three-way comparison.
+def G_SCMP : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type1:$src1, type1:$src2);
+  let hasSideEffects = false;
+}
+
+// Generic unsigned three-way comparison.
+def G_UCMP : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type1:$src1, type1:$src2);
+  let hasSideEffects = false;
+}
+
 // Generic select
 def G_SELECT : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 06a6c1f93ef1f..7eb6cd4e0d798 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -911,6 +911,18 @@ MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
   return buildInstr(TargetOpcode::G_FCMP, Res, {Pred, Op0, Op1}, Flags);
 }
 
+MachineInstrBuilder MachineIRBuilder::buildSCmp(const DstOp &Res,
+                                                const SrcOp &Op0,
+                                                const SrcOp &Op1) {
+  return buildInstr(TargetOpcode::G_SCMP, Res, {Op0, Op1});
+}
+
+MachineInstrBuilder MachineIRBuilder::buildUCmp(const DstOp &Res,
+                                                const SrcOp &Op0,
+                                                const SrcOp &Op1) {
+  return buildInstr(TargetOpcode::G_UCMP, Res, {Op0, Op1});
+}
+
 MachineInstrBuilder
 MachineIRBuilder::buildSelect(const DstOp &Res, const SrcOp &Tst,
                               const SrcOp &Op0, const SrcOp &Op1,
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 0a5b8bdbc9371..d22fbe322ec36 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1544,6 +1544,36 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
 
     break;
   }
+  case TargetOpcode::G_SCMP:
+  case TargetOpcode::G_UCMP: {
+    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
+    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
+    LLT SrcTy2 = MRI->getType(MI->getOperand(2).getReg());
+
+    if (SrcTy.isPointerOrPointerVector() || SrcTy2.isPointerOrPointerVector()) {
+      report("Generic scmp/ucmp does not support pointers as operands", MI);
+      break;
+    }
+
+    if (DstTy.isPointerOrPointerVector()) {
+      report("Generic scmp/ucmp does not support pointers as a result", MI);
+      break;
+    }
+
+    if ((DstTy.isVector() != SrcTy.isVector()) ||
+        (DstTy.isVector() &&
+         DstTy.getElementCount() != SrcTy.getElementCount())) {
+      report("Generic vector scmp/ucmp must preserve number of lanes", MI);
+      break;
+    }
+
+    if (SrcTy != SrcTy2) {
+      report("Generic scmp/ucmp must have same input types", MI);
+      break;
+    }
+
+    break;
+  }
   case TargetOpcode::G_EXTRACT: {
     const MachineOperand &SrcOp = MI->getOperand(1);
     if (!SrcOp.isReg()) {
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 61ea3fb998374..b8da462ed78a1 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -351,6 +351,12 @@
 # DEBUG-NEXT: G_FCMP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_SCMP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_UCMP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_SELECT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/MachineVerifier/test_uscmp.mir b/llvm/test/MachineVerifier/test_uscmp.mir
new file mode 100644
index 0000000000000..aa686c4ec73e6
--- /dev/null
+++ b/llvm/test/MachineVerifier/test_uscmp.mir
@@ -0,0 +1,31 @@
+# RUN: not --crash llc -verify-machineinstrs -run-pass none -mtriple=arm64 -o /dev/null %s 2>&1 | FileCheck %s
+# REQUIRES: aarch64-registered-target
+
+---
+name:            test_uscmp
+body:             |
+  bb.0:
+
+    %2:_(p0) = G_IMPLICIT_DEF
+    %3:_(p0) = G_IMPLICIT_DEF
+    ; CHECK: Generic scmp/ucmp does not support pointers as operands
+    %4:_(s1) = G_SCMP %2, %3
+
+    %12:_(s64) = G_IMPLICIT_DEF
+    %13:_(s64) = G_IMPLICIT_DEF
+    ; CHECK: Generic scmp/ucmp does not support pointers as a result
+    %14:_(p0) = G_SCMP %12, %13
+
+    %23:_(<2 x s32>) = G_IMPLICIT_DEF
+    %24:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: Generic vector scmp/ucmp must preserve number of lanes
+    %5:_(s1) = G_UCMP  %23, %24
+
+    %15:_(s32) = G_CONSTANT i32 0
+    %16:_(s64) = G_CONSTANT i64 2
+    ; CHECK: Generic scmp/ucmp must have same input types
+    %17:_(s1) = G_SCMP %15, %16
+
+
+
+...
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
index 7bbde818082ce..0dcd0c2dd50a8 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
@@ -86,12 +86,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
 // CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2([[#LOWER:]]), GIMT_Encode2([[#UPPER:]]), /*)*//*default:*//*Label 4*/ GIMT_Encode4([[#DEFAULT:]]),
-// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4(410), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4(428), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 2*/ GIMT_Encode4(440), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_FABS*//*Label 3*/ GIMT_Encode4(452),
+// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4(418), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4(436), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 2*/ GIMT_Encode4(448), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_FABS*//*Label 3*/ GIMT_Encode4(460),
 // CHECK-NEXT:     // Label 0: @[[#%u, mul(UPPER-LOWER, 4) + 10]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(427), // Rule ID 2 //
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(435), // Rule ID 2 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
 // CHECK-NEXT:       // MIs[0] x
 // CHECK-NEXT:       // No operand predicates
@@ -101,10 +101,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner1),
 // CHECK-NEXT:       // Combiner Rule #2: TwoMatchNoApply
 // CHECK-NEXT:       GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:     // Label 5: @427
+// CHECK-NEXT:     // Label 5: @435
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @428
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(439), // Rule ID 3 //
+// CHECK-NEXT:     // Label 1: @436
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(447), // Rule ID 3 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -112,10 +112,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #3: NoMatchTwoApply
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2),
-// CHECK-NEXT:     // Label 6: @439
+// CHECK-NEXT:     // Label 6: @447
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @440
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(451), // Rule ID 1 //
+// CHECK-NEXT:     // Label 2: @448
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(459), // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -123,10 +123,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #1: TwoMatchTwoApply
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
-// CHECK-NEXT:     // Label 7: @451
+// CHECK-NEXT:     // Label 7: @459
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @452
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(463), // Rule ID 0 //
+// CHECK-NEXT:     // Label 3: @460
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(471), // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -134,7 +134,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #0: OneMatchOneApply
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT:     // Label 8: @463
+// CHECK-NEXT:     // Label 8: @471
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     // Label 4: @[[#%u, DEFAULT]]
 // CHECK-NEXT:     GIM_Reject,
diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td
index 796f595930319..853831366fa53 100644
--- a/llvm/test/TableGen/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter.td
@@ -513,7 +513,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
 // R00O-NEXT:  GIM_Reject,
 // R00O:       // Label [[DEFAULT_NUM]]: @[[DEFAULT]]
 // R00O-NEXT:  GIM_Reject,
-// R00O-NEXT:  }; // Size: 1808 bytes
+// R00O-NEXT:  }; // Size: 1816 bytes
 
 def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4),
                  [(set GPR32:$dst,

From 47b63cd508f993d9fab2acfbf0dcf86cdc8c5335 Mon Sep 17 00:00:00 2001
From: Daniel Bertalan <dani@danielbertalan.dev>
Date: Thu, 18 Jul 2024 16:26:32 +0200
Subject: [PATCH 027/486] [lld-macho] Save all thin archive members in repro
 tarball (#97169)

Previously, we only saved those members of thin archives into a repro
file that were actually used during linking. However, -ObjC handling
requires us to inspect all members, even those that don't end up being
loaded.

We weren't handling missing members correctly and crashed with an
"unhandled `Error`" failure in LLVM_ENABLE_ABI_BREAKING_CHECKS builds.

To fix this, we now eagerly load all object files and warn when
encountering missing members (in the instances where it wasn't a hard
error before). To avoid having to patch out the checks when dealing
with older repro files, the `--no-warn-thin-archive-missing-members`
flag is added as an escape hatch.
---
 lld/MachO/Config.h                           |  1 +
 lld/MachO/Driver.cpp                         | 43 ++++++++++++++++++--
 lld/MachO/InputFiles.cpp                     |  4 --
 lld/MachO/Options.td                         |  3 ++
 lld/test/MachO/reproduce-thin-archive-objc.s | 25 ++++++++++++
 lld/test/MachO/reproduce-thin-archives.s     | 19 +++++++--
 6 files changed, 83 insertions(+), 12 deletions(-)
 create mode 100644 lld/test/MachO/reproduce-thin-archive-objc.s

diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index 5c354e0fe8821..e79812b16ec12 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -212,6 +212,7 @@ struct Configuration {
   bool csProfileGenerate = false;
   llvm::StringRef csProfilePath;
   bool pgoWarnMismatch;
+  bool warnThinArchiveMissingMembers;
 
   bool callGraphProfileSort = false;
   llvm::StringRef printSymbolOrder;
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index ffb3feae25ca4..dc9d635b48ec4 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -270,6 +270,20 @@ struct ArchiveFileInfo {
 
 static DenseMap<StringRef, ArchiveFileInfo> loadedArchives;
 
+static void saveThinArchiveToRepro(ArchiveFile const *file) {
+  assert(tar && file->getArchive().isThin());
+
+  Error e = Error::success();
+  for (const object::Archive::Child &c : file->getArchive().children(e)) {
+    MemoryBufferRef mb = CHECK(c.getMemoryBufferRef(),
+                               toString(file) + ": failed to get buffer");
+    tar->append(relativeToRoot(CHECK(c.getFullName(), file)), mb.getBuffer());
+  }
+  if (e)
+    error(toString(file) +
+          ": Archive::children failed: " + toString(std::move(e)));
+}
+
 static InputFile *addFile(StringRef path, LoadType loadType,
                           bool isLazy = false, bool isExplicit = true,
                           bool isBundleLoader = false,
@@ -301,6 +315,9 @@ static InputFile *addFile(StringRef path, LoadType loadType,
       if (!archive->isEmpty() && !archive->hasSymbolTable())
         error(path + ": archive has no index; run ranlib to add one");
       file = make<ArchiveFile>(std::move(archive), isForceHidden);
+
+      if (tar && file->getArchive().isThin())
+        saveThinArchiveToRepro(file);
     } else {
       file = entry->second.file;
       // Command-line loads take precedence. If file is previously loaded via
@@ -330,9 +347,13 @@ static InputFile *addFile(StringRef path, LoadType loadType,
               reason = "-all_load";
               break;
           }
-          if (Error e = file->fetch(c, reason))
-            error(toString(file) + ": " + reason +
-                  " failed to load archive member: " + toString(std::move(e)));
+          if (Error e = file->fetch(c, reason)) {
+            if (config->warnThinArchiveMissingMembers)
+              warn(toString(file) + ": " + reason +
+                   " failed to load archive member: " + toString(std::move(e)));
+            else
+              llvm::consumeError(std::move(e));
+          }
         }
         if (e)
           error(toString(file) +
@@ -349,7 +370,18 @@ static InputFile *addFile(StringRef path, LoadType loadType,
         Error e = Error::success();
         for (const object::Archive::Child &c : file->getArchive().children(e)) {
           Expected<MemoryBufferRef> mb = c.getMemoryBufferRef();
-          if (!mb || !hasObjCSection(*mb))
+          if (!mb) {
+            // We used to create broken repro tarballs that only included those
+            // object files from thin archives that ended up being used.
+            if (config->warnThinArchiveMissingMembers)
+              warn(toString(file) + ": -ObjC failed to open archive member: " +
+                   toString(mb.takeError()));
+            else
+              llvm::consumeError(mb.takeError());
+            continue;
+          }
+
+          if (!hasObjCSection(*mb))
             continue;
           if (Error e = file->fetch(c, "-ObjC"))
             error(toString(file) + ": -ObjC failed to load archive member: " +
@@ -1699,6 +1731,9 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
   config->csProfilePath = args.getLastArgValue(OPT_cs_profile_path);
   config->pgoWarnMismatch =
       args.hasFlag(OPT_pgo_warn_mismatch, OPT_no_pgo_warn_mismatch, true);
+  config->warnThinArchiveMissingMembers =
+      args.hasFlag(OPT_warn_thin_archive_missing_members,
+                   OPT_no_warn_thin_archive_missing_members, true);
   config->generateUuid = !args.hasArg(OPT_no_uuid);
 
   for (const Arg *arg : args.filtered(OPT_alias)) {
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index b40a812f30bd3..3086c9cc4729d 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -2200,10 +2200,6 @@ Error ArchiveFile::fetch(const object::Archive::Child &c, StringRef reason) {
   if (!mb)
     return mb.takeError();
 
-  // Thin archives refer to .o files, so --reproduce needs the .o files too.
-  if (tar && c.getParent()->isThin())
-    tar->append(relativeToRoot(CHECK(c.getFullName(), this)), mb->getBuffer());
-
   Expected<TimePoint<std::chrono::seconds>> modTime = c.getLastModified();
   if (!modTime)
     return modTime.takeError();
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index dc2212399222f..bbd8bf70c3a0c 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -153,6 +153,9 @@ def cs_profile_path: Joined<["--"], "cs-profile-path=">,
 defm pgo_warn_mismatch: BB<"pgo-warn-mismatch",
   "turn on warnings about profile cfg mismatch (default)",
   "turn off warnings about profile cfg mismatch">, Group<grp_lld>;
+defm warn_thin_archive_missing_members : BB<"warn-thin-archive-missing-members",
+  "Warn on missing object files referenced by thin archives (default)",
+  "Do not warn on missing object files referenced by thin archives">, Group<grp_lld>;
 
 // This is a complete Options.td compiled from Apple's ld(1) manpage
 // dated 2018-03-07 and cross checked with ld64 source code in repo
diff --git a/lld/test/MachO/reproduce-thin-archive-objc.s b/lld/test/MachO/reproduce-thin-archive-objc.s
new file mode 100644
index 0000000000000..c5fe42f130526
--- /dev/null
+++ b/lld/test/MachO/reproduce-thin-archive-objc.s
@@ -0,0 +1,25 @@
+# REQUIRES: x86
+
+## For a long time, LLD only included those members from thin archives that were actually used
+## during linking. However, we need to iterate over all members for -ObjC, check that we don't
+## crash when we encounter a missing member.
+
+# RUN: rm -rf %t; mkdir %t
+# RUN: sed s/SYM/_main/   %s | llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/main.o
+# RUN: sed s/SYM/_unused/ %s | llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/unused.o
+
+# RUN: cd %t; llvm-ar rcsT unused.a unused.o; rm unused.o
+## FIXME: Absolute paths don't end up relativized in the repro file.
+
+# RUN: %no-fatal-warnings-lld %t/main.o %t/unused.a -ObjC -o /dev/null 2>&1 \
+# RUN:                      | FileCheck %s --check-prefix=WARN
+
+# RUN: %lld %t/main.o %t/unused.a -ObjC --no-warn-thin-archive-missing-members -o /dev/null \
+# RUN:    | FileCheck %s --implicit-check-not 'warning' --allow-empty
+
+# WARN: ld64.lld: warning: {{.*}}unused.a: -ObjC failed to open archive member: 'unused.o'
+
+.text
+.globl SYM
+SYM:
+    ret
diff --git a/lld/test/MachO/reproduce-thin-archives.s b/lld/test/MachO/reproduce-thin-archives.s
index 9dee3f400e06a..33eeaede7aa41 100644
--- a/lld/test/MachO/reproduce-thin-archives.s
+++ b/lld/test/MachO/reproduce-thin-archives.s
@@ -1,10 +1,11 @@
 # REQUIRES: x86
 
-# RUN: rm -rf %t.dir
-# RUN: mkdir -p %t.dir
-# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %s -o %t.dir/foo.o
+# RUN: rm -rf %t.dir; split-file %s %t.dir
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %t.dir/foo.s -o %t.dir/foo.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %t.dir/unused.s -o %t.dir/unused.o
 # RUN: cd %t.dir
-# RUN: llvm-ar rcsT foo.a foo.o
+# RUN: llvm-ar rcsT foo.a foo.o unused.o
 
 # RUN: %lld foo.a -o /dev/null --reproduce repro.tar
 # RUN: tar tf repro.tar | FileCheck -DPATH='repro/%:t.dir' %s
@@ -12,9 +13,19 @@
 # RUN: %lld -all_load foo.a -o /dev/null --reproduce repro2.tar
 # RUN: tar tf repro2.tar | FileCheck -DPATH='repro2/%:t.dir' %s
 
+# RUN: %lld -ObjC foo.a -o /dev/null --reproduce repro3.tar
+# RUN: tar tf repro3.tar | FileCheck -DPATH='repro3/%:t.dir' %s
+
 # CHECK-DAG: [[PATH]]/foo.a
 # CHECK-DAG: [[PATH]]/foo.o
+# CHECK-DAG: [[PATH]]/unused.o
 
+#--- foo.s
 .globl _main
 _main:
   nop
+
+#--- unused.s
+.globl _unused
+_unused:
+  nop

From c0c4ad5d9a6e05e0b1f5f98ce2e08d479b281be8 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Thu, 18 Jul 2024 07:34:04 -0700
Subject: [PATCH 028/486] [clang][test] Split AArch64 target feature checks
 across multiple lines. NFC (#99365)

Whenever these tests change, it's difficult to see why they don't match,
and the diff after you've fixed them isn't easy to grok. By splitting
them with a sed pipe, we fix both issues simultaneously.
---
 .../Preprocessor/aarch64-target-features.c    | 351 +++++++++++++++---
 1 file changed, 303 insertions(+), 48 deletions(-)

diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index 71cc36acf3f0e..d811cb36e28d8 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -291,54 +291,309 @@
 // RUN: %clang -target aarch64 -mtune=CYCLONE -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MTUNE-CYCLONE %s
 // CHECK-MTUNE-CYCLONE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a"
 
-// RUN: %clang -target aarch64 -mcpu=apple-a7 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a9 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a10 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A10 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a11 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A11 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a12 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a13 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A13 %s
-// RUN: %clang -target aarch64 -mcpu=apple-s4 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
-// RUN: %clang -target aarch64 -mcpu=apple-s5 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
-// RUN: %clang -target aarch64 -mcpu=cyclone -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a34 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A34 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a35 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A35 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a53 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A53 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a57 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A57 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a72 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A72 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a73 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-CORTEX-A73 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-r82 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-CORTEX-R82 %s
-// RUN: %clang -target aarch64 -mcpu=exynos-m3 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-M3 %s
-// RUN: %clang -target aarch64 -mcpu=exynos-m4 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-M4 %s
-// RUN: %clang -target aarch64 -mcpu=exynos-m5 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-M4 %s
-// RUN: %clang -target aarch64 -mcpu=kryo -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-KRYO %s
-// RUN: %clang -target aarch64 -mcpu=thunderx2t99 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-THUNDERX2T99 %s
-// RUN: %clang -target aarch64 -mcpu=a64fx -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A64FX %s
-// RUN: %clang -target aarch64 -mcpu=carmel -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-CARMEL %s
-// CHECK-MCPU-APPLE-A7: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A10: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+lor" "-target-feature" "+neon" "-target-feature" "+pan" "-target-feature" "+perfmon" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+vh"
-// CHECK-MCPU-APPLE-A11: "-cc1"{{.*}} "-triple" "aarch64{{.*}}"{{.*}}"-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A12: "-cc1"{{.*}} "-triple" "aarch64"{{.*}} "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
-// CHECK-MCPU-A34: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A13: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+sha3"
-// CHECK-MCPU-A35: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-A53: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-A57: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-A72: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-CORTEX-A73: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-CORTEX-R82: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8r" "-target-feature" "+ccdp" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+flagm" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+predres" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sb" "-target-feature" "+ssbs"
-// CHECK-MCPU-M3: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-M4: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
-// CHECK-MCPU-KRYO: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-THUNDERX2T99: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.1a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+rdm" "-target-feature" "+sha2
-// CHECK-MCPU-A64FX: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+sve"
-// CHECK-MCPU-CARMEL: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
-
-// RUN: %clang -target x86_64-apple-macosx -arch arm64 -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH-ARM64 %s
-// CHECK-ARCH-ARM64: "-target-cpu" "apple-m1" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+altnzcv" "-target-feature" "+ccdp" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fptoint" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+predres" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sb" "-target-feature" "+sha2" "-target-feature" "+sha3" "-target-feature" "+specrestrict" "-target-feature" "+ssbs"
-
-// RUN: %clang -target x86_64-apple-macosx -arch arm64_32 -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH-ARM64_32 %s
-// CHECK-ARCH-ARM64_32: "-target-cpu" "apple-s4" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
+// RUN: %clang -target aarch64 -mcpu=apple-a7 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a8 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a9 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a10 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A10 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a11 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A11 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a12 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a13 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A13 %s
+// RUN: %clang -target aarch64 -mcpu=apple-s4 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
+// RUN: %clang -target aarch64 -mcpu=apple-s5 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
+// RUN: %clang -target aarch64 -mcpu=cyclone -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a34 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A34 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a35 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A35 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a53 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A53 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a57 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A57 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a72 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A72 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a73 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-CORTEX-A73 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-r82 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-CORTEX-R82 %s
+// RUN: %clang -target aarch64 -mcpu=exynos-m3 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-M3 %s
+// RUN: %clang -target aarch64 -mcpu=exynos-m4 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-M4 %s
+// RUN: %clang -target aarch64 -mcpu=exynos-m5 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-M4 %s
+// RUN: %clang -target aarch64 -mcpu=kryo -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-KRYO %s
+// RUN: %clang -target aarch64 -mcpu=thunderx2t99 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-THUNDERX2T99 %s
+// RUN: %clang -target aarch64 -mcpu=a64fx -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A64FX %s
+// RUN: %clang -target aarch64 -mcpu=carmel -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-CARMEL %s
+// CHECK-MCPU-APPLE-A7-LABEL: "-target-cpu" "apple-a7"
+// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+zcm"
+// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+zcz"
+// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+v8a"
+// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A7-NEXT: "-target-abi"
+// CHECK-MCPU-APPLE-A10-LABEL: "-target-cpu" "apple-a10"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+zcm"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+zcz"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+v8a"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+lor"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+pan"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+rdm"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+vh"
+// CHECK-MCPU-APPLE-A10-NEXT: "-target-abi"
+// CHECK-MCPU-APPLE-A11-LABEL: "-target-cpu" "apple-a11"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+zcm"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+zcz"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+v8.2a"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+fullfp16"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+lse"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+ras"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+rdm"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A11-NEXT: "-target-abi"
+// CHECK-MCPU-APPLE-A12-LABEL: "-target-cpu" "apple-a12"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+zcm"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+zcz"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+v8.3a"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+complxnum"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+fullfp16"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+jsconv"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+lse"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+pauth"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+ras"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+rcpc"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+rdm"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A12-NEXT: "-target-abi"
+// CHECK-MCPU-A34-LABEL: "-target-cpu" "cortex-a34"
+// CHECK-MCPU-A34-NEXT: "-target-feature" "+v8a"
+// CHECK-MCPU-A34-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-A34-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-A34-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-A34-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-A34-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-A34-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-A34-NEXT: "-target-abi"
+// CHECK-MCPU-APPLE-A13-LABEL: "-target-cpu" "apple-a13"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+zcm"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+zcz"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+v8.4a"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+complxnum"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+dotprod"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+fp16fml"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+fullfp16"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+jsconv"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+lse"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+pauth"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+ras"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+rcpc"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+rdm"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+sha3"
+// CHECK-MCPU-APPLE-A13-NEXT: "-target-abi"
+// CHECK-MCPU-A35-LABEL: "-target-cpu" "cortex-a35"
+// CHECK-MCPU-A35-NEXT: "-target-feature" "+v8a"
+// CHECK-MCPU-A35-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-A35-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-A35-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-A35-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-A35-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-A35-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-A35-NEXT: "-target-abi"
+// CHECK-MCPU-A53-LABEL: "-target-cpu" "cortex-a53"
+// CHECK-MCPU-A53-NEXT: "-target-feature" "+v8a"
+// CHECK-MCPU-A53-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-A53-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-A53-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-A53-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-A53-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-A53-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-A53-NEXT: "-target-abi"
+// CHECK-MCPU-A57-LABEL: "-target-cpu" "cortex-a57"
+// CHECK-MCPU-A57-NEXT: "-target-feature" "+v8a"
+// CHECK-MCPU-A57-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-A57-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-A57-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-A57-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-A57-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-A57-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-A57-NEXT: "-target-abi"
+// CHECK-MCPU-A72-LABEL: "-target-cpu" "cortex-a72"
+// CHECK-MCPU-A72-NEXT: "-target-feature" "+v8a"
+// CHECK-MCPU-A72-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-A72-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-A72-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-A72-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-A72-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-A72-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-A72-NEXT: "-target-abi"
+// CHECK-MCPU-CORTEX-A73-LABEL: "-target-cpu" "cortex-a73"
+// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+v8a"
+// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-CORTEX-A73-NEXT: "-target-abi"
+// CHECK-MCPU-CORTEX-R82-LABEL: "-target-cpu" "cortex-r82"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+v8r"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+ccdp"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+complxnum"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+dotprod"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+flagm"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+fp16fml"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+fullfp16"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+jsconv"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+lse"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+pauth"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+predres"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+ras"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+rcpc"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+rdm"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+sb"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+ssbs"
+// CHECK-MCPU-CORTEX-R82-NEXT: "-target-abi"
+// CHECK-MCPU-M3-LABEL: "-target-cpu" "exynos-m3"
+// CHECK-MCPU-M3-NEXT: "-target-feature" "+v8a"
+// CHECK-MCPU-M3-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-M3-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-M3-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-M3-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-M3-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-M3-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-M3-NEXT: "-target-abi"
+// CHECK-MCPU-M4-LABEL: "-target-cpu" "exynos-m{{[45]}}"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+v8.2a"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+dotprod"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+fullfp16"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+lse"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+ras"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+rdm"
+// CHECK-MCPU-M4-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-M4-NEXT: "-target-abi"
+// CHECK-MCPU-KRYO-LABEL: "-target-cpu" "kryo"
+// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+v8a"
+// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-KRYO-NEXT: "-target-abi"
+// CHECK-MCPU-THUNDERX2T99-LABEL: "-target-cpu" "thunderx2t99"
+// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+v8.1a"
+// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+lse"
+// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+rdm"
+// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+sha2
+// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-abi"
+// CHECK-MCPU-A64FX-LABEL: "-target-cpu" "a64fx"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+v8.2a"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+complxnum"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+fullfp16"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+lse"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+perfmon"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+ras"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+rdm"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+sve"
+// CHECK-MCPU-A64FX-NEXT: "-target-abi"
+// CHECK-MCPU-CARMEL-LABEL: "-target-cpu" "carmel"
+// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+v8.2a"
+// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+aes"
+// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+crc"
+// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+fullfp16"
+// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+lse"
+// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+neon"
+// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+ras"
+// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+rdm"
+// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+sha2"
+// CHECK-MCPU-CARMEL-NEXT: "-target-abi"
+
+
+// RUN: %clang -target x86_64-apple-macosx -arch arm64 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck --check-prefix=CHECK-ARCH-ARM64 %s
+// CHECK-ARCH-ARM64-LABEL: "-target-cpu" "apple-m1"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+zcm"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+zcz"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+v8.4a"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+aes"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+altnzcv"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+ccdp"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+complxnum"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+crc"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+dotprod"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+fp16fml"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+fptoint"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+fullfp16"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+jsconv"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+lse"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+neon"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+pauth"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+perfmon"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+predres"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+ras"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+rcpc"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+rdm"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+sb"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+sha2"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+sha3"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+specrestrict"
+// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+ssbs"
+// CHECK-ARCH-ARM64-NEXT: "-target-abi"
+
+// RUN: %clang -target x86_64-apple-macosx -arch arm64_32 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck --check-prefix=CHECK-ARCH-ARM64_32 %s
+// CHECK-ARCH-ARM64_32-LABEL: "-target-cpu" "apple-s4"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+zcm"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+zcz"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+v8.3a"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+aes"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+complxnum"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+crc"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+fp-armv8"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+fullfp16"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+jsconv"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+lse"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+neon"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+pauth"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+perfmon"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+ras"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+rcpc"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+rdm"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+sha2"
+// CHECK-ARCH-ARM64_32-NEXT: "-target-abi"
 
 // RUN: %clang -target aarch64 -march=armv8-a+fp+simd+crc+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MARCH-1 %s
 // RUN: %clang -target aarch64 -march=armv8-a+nofp+nosimd+nocrc+nocrypto+fp+simd+crc+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MARCH-1 %s

From 257a0d535ac052a4eb1bb847605eff1eb169087d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Thu, 18 Jul 2024 14:40:55 +0200
Subject: [PATCH 029/486] [clang][Interp] Diagnose out-of-range casts to enum
 types

---
 clang/lib/AST/Interp/Compiler.cpp | 10 ++++
 clang/lib/AST/Interp/Interp.cpp   | 26 +++++++++
 clang/lib/AST/Interp/Interp.h     | 14 +++++
 clang/lib/AST/Interp/Opcodes.td   |  7 +++
 clang/test/AST/Interp/cxx11.cpp   | 90 +++++++++++++++++++++++++++++++
 5 files changed, 147 insertions(+)

diff --git a/clang/lib/AST/Interp/Compiler.cpp b/clang/lib/AST/Interp/Compiler.cpp
index 28c4ffd071862..24140b23c1f0b 100644
--- a/clang/lib/AST/Interp/Compiler.cpp
+++ b/clang/lib/AST/Interp/Compiler.cpp
@@ -466,6 +466,16 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     if (!this->visit(SubExpr))
       return false;
 
+    // Possibly diagnose casts to enum types if the target type does not
+    // have a fixed size.
+    if (CE->getType()->isEnumeralType()) {
+      if (const auto *ET = CE->getType().getCanonicalType()->getAs<EnumType>();
+          ET && !ET->getDecl()->isFixed()) {
+        if (!this->emitCheckEnumValue(*FromT, ET->getDecl(), CE))
+          return false;
+      }
+    }
+
     if (ToT == PT_IntAP)
       return this->emitCastAP(*FromT, Ctx.getBitWidth(CE->getType()), CE);
     if (ToT == PT_IntAPS)
diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index e6e9298982887..cd6fc60400ebd 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "llvm/ADT/APSInt.h"
+#include "llvm/ADT/StringExtras.h"
 #include <limits>
 #include <vector>
 
@@ -899,6 +900,31 @@ bool RunDestructors(InterpState &S, CodePtr OpPC, const Block *B) {
   return runRecordDestructor(S, OpPC, Pointer(const_cast<Block *>(B)), Desc);
 }
 
+void diagnoseEnumValue(InterpState &S, CodePtr OpPC, const EnumDecl *ED,
+                       const APSInt &Value) {
+  llvm::APInt Min;
+  llvm::APInt Max;
+
+  if (S.EvaluatingDecl && !S.EvaluatingDecl->isConstexpr())
+    return;
+
+  ED->getValueRange(Max, Min);
+  --Max;
+
+  if (ED->getNumNegativeBits() &&
+      (Max.slt(Value.getSExtValue()) || Min.sgt(Value.getSExtValue()))) {
+    const SourceLocation &Loc = S.Current->getLocation(OpPC);
+    S.report(Loc, diag::warn_constexpr_unscoped_enum_out_of_range)
+        << llvm::toString(Value, 10) << Min.getSExtValue() << Max.getSExtValue()
+        << ED;
+  } else if (!ED->getNumNegativeBits() && Max.ult(Value.getZExtValue())) {
+    const SourceLocation &Loc = S.Current->getLocation(OpPC);
+    S.report(Loc, diag::warn_constexpr_unscoped_enum_out_of_range)
+        << llvm::toString(Value, 10) << Min.getZExtValue() << Max.getZExtValue()
+        << ED;
+  }
+}
+
 bool Interpret(InterpState &S, APValue &Result) {
   // The current stack frame when we started Interpret().
   // This is being used by the ops to determine wheter
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index 2e159012f5ffd..f86b787fb034e 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -2774,6 +2774,20 @@ inline bool CheckNonNullArg(InterpState &S, CodePtr OpPC) {
   return false;
 }
 
+void diagnoseEnumValue(InterpState &S, CodePtr OpPC, const EnumDecl *ED,
+                       const APSInt &Value);
+
+template <PrimType Name, class T = typename PrimConv<Name>::T>
+inline bool CheckEnumValue(InterpState &S, CodePtr OpPC, const EnumDecl *ED) {
+  assert(ED);
+  assert(!ED->isFixed());
+  const APSInt Val = S.Stk.peek<T>().toAPSInt();
+
+  if (S.inConstantContext())
+    diagnoseEnumValue(S, OpPC, ED, Val);
+  return true;
+}
+
 /// OldPtr -> Integer -> NewPtr.
 template <PrimType TIn, PrimType TOut>
 inline bool DecayPtr(InterpState &S, CodePtr OpPC) {
diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/Interp/Opcodes.td
index 3e69098570bd7..49ebb156ab2fb 100644
--- a/clang/lib/AST/Interp/Opcodes.td
+++ b/clang/lib/AST/Interp/Opcodes.td
@@ -66,6 +66,7 @@ def ArgDecl : ArgType { let Name = "const Decl*"; }
 def ArgVarDecl : ArgType { let Name = "const VarDecl*"; }
 def ArgDesc : ArgType { let Name = "const Descriptor *"; }
 def ArgPrimType : ArgType { let Name = "PrimType"; }
+def ArgEnumDecl : ArgType { let Name = "const EnumDecl *"; }
 
 //===----------------------------------------------------------------------===//
 // Classes of types instructions operate on.
@@ -389,6 +390,12 @@ def CheckDecl : Opcode {
   let Args = [ArgVarDecl];
 }
 
+def CheckEnumValue : Opcode {
+  let Args = [ArgEnumDecl];
+  let Types = [FixedSizeIntegralTypeClass];
+  let HasGroup = 1;
+}
+
 // [] -> [Value]
 def GetGlobal : AccessOpcode;
 def GetGlobalUnchecked : AccessOpcode;
diff --git a/clang/test/AST/Interp/cxx11.cpp b/clang/test/AST/Interp/cxx11.cpp
index 82b2727bbadbb..c0b88f0e567e0 100644
--- a/clang/test/AST/Interp/cxx11.cpp
+++ b/clang/test/AST/Interp/cxx11.cpp
@@ -62,3 +62,93 @@ namespace ReferenceToConst {
     }
   };
 }
+
+
+
+namespace GH50055 {
+// Enums without fixed underlying type
+enum E1 {e11=-4, e12=4};
+enum E2 {e21=0, e22=4};
+enum E3 {e31=-4, e32=1024};
+enum E4 {e41=0};
+// Empty but as-if it had a single enumerator with value 0
+enum EEmpty {};
+
+// Enum with fixed underlying type because the underlying type is explicitly specified
+enum EFixed : int {efixed1=-4, efixed2=4};
+// Enum with fixed underlying type because it is scoped
+enum class EScoped {escoped1=-4, escoped2=4};
+
+enum EMaxInt {emaxint1=-1, emaxint2=__INT_MAX__};
+
+enum NumberType {};
+
+E2 testDefaultArgForParam(E2 e2Param = (E2)-1) { // ok, not a constant expression context
+  E2 e2LocalInit = e2Param; // ok, not a constant expression context
+  return e2LocalInit;
+}
+
+// #include <enum-constexpr-conversion-system-header.h>
+
+void testValueInRangeOfEnumerationValues() {
+  constexpr E1 x1 = static_cast<E1>(-8);
+  constexpr E1 x2 = static_cast<E1>(8);
+  // both-error@-1 {{integer value 8 is outside the valid range of values [-8, 7] for the enumeration type 'E1'}}
+  E1 x2b = static_cast<E1>(8); // ok, not a constant expression context
+
+  constexpr E2 x3 = static_cast<E2>(-8);
+  // both-error@-1 {{integer value -8 is outside the valid range of values [0, 7] for the enumeration type 'E2'}}
+  constexpr E2 x4 = static_cast<E2>(0);
+  constexpr E2 x5 = static_cast<E2>(8);
+  // both-error@-1 {{integer value 8 is outside the valid range of values [0, 7] for the enumeration type 'E2'}}
+
+  constexpr E3 x6 = static_cast<E3>(-2048);
+  constexpr E3 x7 = static_cast<E3>(-8);
+  constexpr E3 x8 = static_cast<E3>(0);
+  constexpr E3 x9 = static_cast<E3>(8);
+  constexpr E3 x10 = static_cast<E3>(2048);
+  // both-error@-1 {{integer value 2048 is outside the valid range of values [-2048, 2047] for the enumeration type 'E3'}}
+
+  constexpr E4 x11 = static_cast<E4>(0);
+  constexpr E4 x12 = static_cast<E4>(1);
+  constexpr E4 x13 = static_cast<E4>(2);
+  // both-error@-1 {{integer value 2 is outside the valid range of values [0, 1] for the enumeration type 'E4'}}
+
+  constexpr EEmpty x14 = static_cast<EEmpty>(0);
+  constexpr EEmpty x15 = static_cast<EEmpty>(1);
+  constexpr EEmpty x16 = static_cast<EEmpty>(2);
+  // both-error@-1 {{integer value 2 is outside the valid range of values [0, 1] for the enumeration type 'EEmpty'}}
+
+  constexpr EFixed x17 = static_cast<EFixed>(100);
+  constexpr EScoped x18 = static_cast<EScoped>(100);
+
+  constexpr EMaxInt x19 = static_cast<EMaxInt>(__INT_MAX__-1);
+  constexpr EMaxInt x20 = static_cast<EMaxInt>((long)__INT_MAX__+1);
+  // both-error@-1 {{integer value 2147483648 is outside the valid range of values [-2147483648, 2147483647] for the enumeration type 'EMaxInt'}}
+
+  const NumberType neg_one = (NumberType) ((NumberType) 0 - (NumberType) 1); // ok, not a constant expression context
+}
+
+template<class T, unsigned size> struct Bitfield {
+  static constexpr T max = static_cast<T>((1 << size) - 1); // #enum
+};
+
+void testValueInRangeOfEnumerationValuesViaTemplate() {
+  Bitfield<E2, 3> good;
+  Bitfield<E2, 4> bad; // both-error@#enum {{integer value 15 is outside the valid range of values [0, 7] for the enumeration type 'E2'}}
+}
+
+enum SortOrder {
+  AscendingOrder,
+  DescendingOrder
+};
+
+class A {
+  static void f(SortOrder order);
+};
+
+void A::f(SortOrder order) {
+  if (order == SortOrder(-1)) // ok, not a constant expression context
+    return;
+}
+}

From 7d74ca9513a3fa53b482230c20b1977a1f3d121b Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 18 Jul 2024 16:48:21 +0200
Subject: [PATCH 030/486] [ValueLattice] Support constant vectors in mergeIn()
 (#99466)

This is a followup to vector support in LVI/CVP/SCCP. In mergeIn(), if
one of the operands is a vector of integer constant, we should try to
convert it into a constant range, in case that allows performing a range
union to something better than overdefined.
---
 llvm/include/llvm/Analysis/ValueLattice.h     | 34 ++++++++++++-------
 .../CorrelatedValuePropagation/vectors.ll     |  4 +--
 llvm/test/Transforms/SCCP/phis.ll             |  4 +--
 3 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/llvm/include/llvm/Analysis/ValueLattice.h b/llvm/include/llvm/Analysis/ValueLattice.h
index b81eb5f60ab7e..fa56d838a3859 100644
--- a/llvm/include/llvm/Analysis/ValueLattice.h
+++ b/llvm/include/llvm/Analysis/ValueLattice.h
@@ -281,18 +281,21 @@ class ValueLatticeElement {
     return std::nullopt;
   }
 
-  ConstantRange asConstantRange(Type *Ty, bool UndefAllowed = false) const {
-    assert(Ty->isIntOrIntVectorTy() && "Must be integer type");
+  ConstantRange asConstantRange(unsigned BW, bool UndefAllowed = false) const {
     if (isConstantRange(UndefAllowed))
       return getConstantRange();
     if (isConstant())
       return getConstant()->toConstantRange();
-    unsigned BW = Ty->getScalarSizeInBits();
     if (isUnknown())
       return ConstantRange::getEmpty(BW);
     return ConstantRange::getFull(BW);
   }
 
+  ConstantRange asConstantRange(Type *Ty, bool UndefAllowed = false) const {
+    assert(Ty->isIntOrIntVectorTy() && "Must be integer type");
+    return asConstantRange(Ty->getScalarSizeInBits(), UndefAllowed);
+  }
+
   bool markOverdefined() {
     if (isOverdefined())
       return false;
@@ -384,7 +387,9 @@ class ValueLatticeElement {
       return true;
     }
 
-    assert(isUnknown() || isUndef());
+    assert(isUnknown() || isUndef() || isConstant());
+    assert((!isConstant() || NewR.contains(getConstant()->toConstantRange())) &&
+           "Constant must be subset of new range");
 
     NumRangeExtensions = 0;
     Tag = NewTag;
@@ -426,6 +431,16 @@ class ValueLatticeElement {
         return false;
       if (RHS.isUndef())
         return false;
+      // If the constant is a vector of integers, try to treat it as a range.
+      if (getConstant()->getType()->isVectorTy() &&
+          getConstant()->getType()->getScalarType()->isIntegerTy()) {
+        ConstantRange L = getConstant()->toConstantRange();
+        ConstantRange NewR = L.unionWith(
+            RHS.asConstantRange(L.getBitWidth(), /*UndefAllowed=*/true));
+        return markConstantRange(
+            std::move(NewR),
+            Opts.setMayIncludeUndef(RHS.isConstantRangeIncludingUndef()));
+      }
       markOverdefined();
       return true;
     }
@@ -444,14 +459,9 @@ class ValueLatticeElement {
       return OldTag != Tag;
     }
 
-    if (!RHS.isConstantRange()) {
-      // We can get here if we've encountered a constantexpr of integer type
-      // and merge it with a constantrange.
-      markOverdefined();
-      return true;
-    }
-
-    ConstantRange NewR = getConstantRange().unionWith(RHS.getConstantRange());
+    const ConstantRange &L = getConstantRange();
+    ConstantRange NewR = L.unionWith(
+        RHS.asConstantRange(L.getBitWidth(), /*UndefAllowed=*/true));
     return markConstantRange(
         std::move(NewR),
         Opts.setMayIncludeUndef(RHS.isConstantRangeIncludingUndef()));
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll b/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll
index 43e680cd25cdb..6254b54d42554 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll
@@ -286,7 +286,7 @@ define <2 x i16> @phi_merge1(i1 %c, <2 x i8> %a) {
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
 ; CHECK-NEXT:    [[PHI:%.*]] = phi <2 x i16> [ [[ZEXT]], %[[ENTRY]] ], [ <i16 1, i16 2>, %[[IF]] ]
-; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i16> [[PHI]], <i16 2, i16 3>
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw <2 x i16> [[PHI]], <i16 2, i16 3>
 ; CHECK-NEXT:    ret <2 x i16> [[ADD]]
 ;
 entry:
@@ -312,7 +312,7 @@ define <2 x i16> @phi_merge2(i1 %c, <2 x i8> %a) {
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
 ; CHECK-NEXT:    [[PHI:%.*]] = phi <2 x i16> [ <i16 1, i16 2>, %[[ENTRY]] ], [ [[ZEXT]], %[[IF]] ]
-; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i16> [[PHI]], <i16 2, i16 3>
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw <2 x i16> [[PHI]], <i16 2, i16 3>
 ; CHECK-NEXT:    ret <2 x i16> [[ADD]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SCCP/phis.ll b/llvm/test/Transforms/SCCP/phis.ll
index 83daae0a7c0c8..9264a6eaefb85 100644
--- a/llvm/test/Transforms/SCCP/phis.ll
+++ b/llvm/test/Transforms/SCCP/phis.ll
@@ -109,7 +109,7 @@ define <2 x i16> @phi_vector_merge1(i1 %c, <2 x i8> %a) {
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
 ; CHECK-NEXT:    [[PHI:%.*]] = phi <2 x i16> [ [[ZEXT]], %[[ENTRY]] ], [ <i16 1, i16 2>, %[[IF]] ]
-; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i16> [[PHI]], <i16 2, i16 3>
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw <2 x i16> [[PHI]], <i16 2, i16 3>
 ; CHECK-NEXT:    ret <2 x i16> [[ADD]]
 ;
 entry:
@@ -135,7 +135,7 @@ define <2 x i16> @phi_vector_merge2(i1 %c, <2 x i8> %a) {
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
 ; CHECK-NEXT:    [[PHI:%.*]] = phi <2 x i16> [ <i16 1, i16 2>, %[[ENTRY]] ], [ [[ZEXT]], %[[IF]] ]
-; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i16> [[PHI]], <i16 2, i16 3>
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw <2 x i16> [[PHI]], <i16 2, i16 3>
 ; CHECK-NEXT:    ret <2 x i16> [[ADD]]
 ;
 entry:

From 9711f6bda1363eb3b8850ee67958ab90357db006 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 18 Jul 2024 16:47:27 +0200
Subject: [PATCH 031/486] [GVN] Add additional tests for pointer replacement
 (NFC)

---
 llvm/test/Transforms/GVN/condprop.ll | 195 +++++++++++++++++++++++++++
 1 file changed, 195 insertions(+)

diff --git a/llvm/test/Transforms/GVN/condprop.ll b/llvm/test/Transforms/GVN/condprop.ll
index 6402a23157729..02f4bb97d7ebd 100644
--- a/llvm/test/Transforms/GVN/condprop.ll
+++ b/llvm/test/Transforms/GVN/condprop.ll
@@ -804,5 +804,200 @@ bb5:
   br label %bb5
 }
 
+define void @select_same_obj(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: @select_same_obj(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[P3:%.*]] = select i1 [[C:%.*]], ptr [[P]], ptr [[P2]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[P]], [[P3]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[EXIT:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[P3]])
+; CHECK-NEXT:    ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p2 = getelementptr i8, ptr %p, i64 %x
+  %p3 = select i1 %c, ptr %p, ptr %p2
+  %cmp = icmp eq ptr %p, %p3
+  br i1 %cmp, label %if, label %exit
+
+if:
+  call void @use_ptr(ptr %p)
+  call void @use_ptr(ptr %p3)
+  ret void
+
+exit:
+  ret void
+}
+
+define void @select_different_obj(i1 %c, ptr %p, ptr %p2) {
+; CHECK-LABEL: @select_different_obj(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = select i1 [[C:%.*]], ptr [[P:%.*]], ptr [[P2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[P]], [[P3]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[EXIT:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[P3]])
+; CHECK-NEXT:    ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = select i1 %c, ptr %p, ptr %p2
+  %cmp = icmp eq ptr %p, %p3
+  br i1 %cmp, label %if, label %exit
+
+if:
+  call void @use_ptr(ptr %p)
+  call void @use_ptr(ptr %p3)
+  ret void
+
+exit:
+  ret void
+}
+
+define void @select_same_obj_is_select(i1 %c, ptr %p, ptr %p2, i64 %x) {
+; CHECK-LABEL: @select_same_obj_is_select(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = select i1 [[C:%.*]], ptr [[P:%.*]], ptr [[P2:%.*]]
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr i8, ptr [[P3]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[P3]], [[P4]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[EXIT:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @use_ptr(ptr [[P3]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[P3]])
+; CHECK-NEXT:    ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = select i1 %c, ptr %p, ptr %p2
+  %p4 = getelementptr i8, ptr %p3, i64 %x
+  %cmp = icmp eq ptr %p3, %p4
+  br i1 %cmp, label %if, label %exit
+
+if:
+  call void @use_ptr(ptr %p3)
+  call void @use_ptr(ptr %p4)
+  ret void
+
+exit:
+  ret void
+}
+
+define void @phi_same_obj(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: @phi_same_obj(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[JOIN:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[JOIN]]
+; CHECK:       join:
+; CHECK-NEXT:    [[P3:%.*]] = phi ptr [ [[P]], [[IF]] ], [ [[P2]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[P]], [[P3]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF2:%.*]], label [[EXIT:%.*]]
+; CHECK:       if2:
+; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[P3]])
+; CHECK-NEXT:    ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p2 = getelementptr i8, ptr %p, i64 %x
+  br i1 %c, label %if, label %join
+
+if:
+  br label %join
+
+join:
+  %p3 = phi ptr [ %p, %if ], [ %p2, %entry ]
+  %cmp = icmp eq ptr %p, %p3
+  br i1 %cmp, label %if2, label %exit
+
+if2:
+  call void @use_ptr(ptr %p)
+  call void @use_ptr(ptr %p3)
+  ret void
+
+exit:
+  ret void
+}
+
+define void @phi_different_obj(i1 %c, ptr %p, ptr %p2) {
+; CHECK-LABEL: @phi_different_obj(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[JOIN:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[JOIN]]
+; CHECK:       join:
+; CHECK-NEXT:    [[P3:%.*]] = phi ptr [ [[P:%.*]], [[IF]] ], [ [[P2:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[P]], [[P3]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF2:%.*]], label [[EXIT:%.*]]
+; CHECK:       if2:
+; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[P3]])
+; CHECK-NEXT:    ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %c, label %if, label %join
+
+if:
+  br label %join
+
+join:
+  %p3 = phi ptr [ %p, %if ], [ %p2, %entry ]
+  %cmp = icmp eq ptr %p, %p3
+  br i1 %cmp, label %if2, label %exit
+
+if2:
+  call void @use_ptr(ptr %p)
+  call void @use_ptr(ptr %p3)
+  ret void
+
+exit:
+  ret void
+}
+
+define void @phi_same_obj_cycle(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: @phi_same_obj_cycle(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[P_IV:%.*]] = phi ptr [ [[P:%.*]], [[ENTRY:%.*]] ], [ [[P_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[P_NEXT]] = getelementptr i8, ptr [[P_IV]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[P_IV]], [[P]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[LOOP_LATCH]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @use_ptr(ptr [[P_IV]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
+; CHECK-NEXT:    br label [[LOOP_LATCH]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %p.iv = phi ptr [ %p, %entry ], [ %p.next, %loop.latch ]
+  %p.next = getelementptr i8, ptr %p.iv, i64 %x
+  %cmp = icmp eq ptr %p.iv, %p
+  br i1 %cmp, label %if, label %loop.latch
+
+if:
+  call void @use_ptr(ptr %p.iv)
+  call void @use_ptr(ptr %p)
+  br label %loop.latch
+
+loop.latch:
+  br label %loop
+}
+
 declare void @use_bool(i1)
 declare void @use_ptr(ptr)

From f6b06b42a3f4f59ff33da20d42358f2768eaf726 Mon Sep 17 00:00:00 2001
From: Akira Hatanaka <ahatanak@gmail.com>
Date: Thu, 18 Jul 2024 07:51:17 -0700
Subject: [PATCH 032/486] [PAC] Implement function pointer re-signing (#98847)

Re-signing occurs when function type discrimination is enabled and a
function pointer is converted to another function pointer type that
requires signing using a different discriminator. A function pointer is
re-signed using discriminator zero when it's converted to a pointer to a
non-function type such as `void*`.

---------

Co-authored-by: Ahmed Bougacha <ahmed@bougacha.org>
Co-authored-by: John McCall <rjmccall@apple.com>
---
 clang/lib/CodeGen/Address.h                   |  42 +++-
 clang/lib/CodeGen/CGBuilder.h                 |   4 +-
 clang/lib/CodeGen/CGExpr.cpp                  |   3 +-
 clang/lib/CodeGen/CGExprScalar.cpp            |   7 +-
 clang/lib/CodeGen/CGPointerAuth.cpp           | 230 ++++++++++++++++++
 clang/lib/CodeGen/CGValue.h                   |  27 +-
 clang/lib/CodeGen/CodeGenFunction.cpp         |  30 ++-
 clang/lib/CodeGen/CodeGenFunction.h           |  35 ++-
 clang/lib/Headers/ptrauth.h                   |  15 ++
 .../ptrauth-function-lvalue-cast-disc.c       |  68 ++++++
 ...ptrauth-function-type-discriminator-cast.c |  94 +++++++
 11 files changed, 513 insertions(+), 42 deletions(-)
 create mode 100644 clang/test/CodeGen/ptrauth-function-lvalue-cast-disc.c
 create mode 100644 clang/test/CodeGen/ptrauth-function-type-discriminator-cast.c

diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index 35ec370a139c9..1c4d2e103b5e7 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_LIB_CODEGEN_ADDRESS_H
 #define LLVM_CLANG_LIB_CODEGEN_ADDRESS_H
 
+#include "CGPointerAuthInfo.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Type.h"
 #include "llvm/ADT/PointerIntPair.h"
@@ -108,6 +109,22 @@ class RawAddress {
 
 /// Like RawAddress, an abstract representation of an aligned address, but the
 /// pointer contained in this class is possibly signed.
+///
+/// This is designed to be an IR-level abstraction, carrying just the
+/// information necessary to perform IR operations on an address like loads and
+/// stores.  In particular, it doesn't carry C type information or allow the
+/// representation of things like bit-fields; clients working at that level
+/// should generally be using `LValue`.
+///
+/// An address may be either *raw*, meaning that it's an ordinary machine
+/// pointer, or *signed*, meaning that the pointer carries an embedded
+/// pointer-authentication signature. Representing signed pointers directly in
+/// this abstraction allows the authentication to be delayed as long as possible
+/// without forcing IRGen to use totally different code paths for signed and
+/// unsigned values or to separately propagate signature information through
+/// every API that manipulates addresses. Pointer arithmetic on signed addresses
+/// (e.g. drilling down to a struct field) is accumulated into a separate offset
+/// which is applied when the address is finally accessed.
 class Address {
   friend class CGBuilderTy;
 
@@ -121,7 +138,11 @@ class Address {
 
   CharUnits Alignment;
 
-  /// Offset from the base pointer.
+  /// The ptrauth information needed to authenticate the base pointer.
+  CGPointerAuthInfo PtrAuthInfo;
+
+  /// Offset from the base pointer. This is non-null only when the base
+  /// pointer is signed.
   llvm::Value *Offset = nullptr;
 
   llvm::Value *emitRawPointerSlow(CodeGenFunction &CGF) const;
@@ -140,12 +161,14 @@ class Address {
   }
 
   Address(llvm::Value *BasePtr, llvm::Type *ElementType, CharUnits Alignment,
-          llvm::Value *Offset, KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
+          CGPointerAuthInfo PtrAuthInfo, llvm::Value *Offset,
+          KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
       : Pointer(BasePtr, IsKnownNonNull), ElementType(ElementType),
-        Alignment(Alignment), Offset(Offset) {}
+        Alignment(Alignment), PtrAuthInfo(PtrAuthInfo), Offset(Offset) {}
 
   Address(RawAddress RawAddr)
-      : Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr),
+      : Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr,
+                RawAddr.isValid() ? RawAddr.isKnownNonNull() : NotKnownNonNull),
         ElementType(RawAddr.isValid() ? RawAddr.getElementType() : nullptr),
         Alignment(RawAddr.isValid() ? RawAddr.getAlignment()
                                     : CharUnits::Zero()) {}
@@ -192,6 +215,9 @@ class Address {
   /// Return the IR name of the pointer value.
   llvm::StringRef getName() const { return Pointer.getPointer()->getName(); }
 
+  const CGPointerAuthInfo &getPointerAuthInfo() const { return PtrAuthInfo; }
+  void setPointerAuthInfo(const CGPointerAuthInfo &Info) { PtrAuthInfo = Info; }
+
   // This function is called only in CGBuilderBaseTy::CreateElementBitCast.
   void setElementType(llvm::Type *Ty) {
     assert(hasOffset() &&
@@ -199,6 +225,8 @@ class Address {
     ElementType = Ty;
   }
 
+  bool isSigned() const { return PtrAuthInfo.isSigned(); }
+
   /// Whether the pointer is known not to be null.
   KnownNonNull_t isKnownNonNull() const {
     assert(isValid());
@@ -215,6 +243,9 @@ class Address {
 
   llvm::Value *getOffset() const { return Offset; }
 
+  Address getResignedAddress(const CGPointerAuthInfo &NewInfo,
+                             CodeGenFunction &CGF) const;
+
   /// Return the pointer contained in this class after authenticating it and
   /// adding offset to it if necessary.
   llvm::Value *emitRawPointer(CodeGenFunction &CGF) const {
@@ -240,7 +271,8 @@ class Address {
   /// alignment.
   Address withElementType(llvm::Type *ElemTy) const {
     if (!hasOffset())
-      return Address(getBasePointer(), ElemTy, getAlignment(), nullptr,
+      return Address(getBasePointer(), ElemTy, getAlignment(),
+                     getPointerAuthInfo(), /*Offset=*/nullptr,
                      isKnownNonNull());
     Address A(*this);
     A.ElementType = ElemTy;
diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h
index 0bc4fda62979c..5d59d5a4ae2c1 100644
--- a/clang/lib/CodeGen/CGBuilder.h
+++ b/clang/lib/CodeGen/CGBuilder.h
@@ -190,8 +190,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
                               const llvm::Twine &Name = "") {
     if (!Addr.hasOffset())
       return Address(CreateAddrSpaceCast(Addr.getBasePointer(), Ty, Name),
-                     ElementTy, Addr.getAlignment(), nullptr,
-                     Addr.isKnownNonNull());
+                     ElementTy, Addr.getAlignment(), Addr.getPointerAuthInfo(),
+                     /*Offset=*/nullptr, Addr.isKnownNonNull());
     // Eagerly force a raw address if these is an offset.
     return RawAddress(
         CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name),
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 5fdd3cc490e59..6a0af00b9e186 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -1312,7 +1312,8 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
         if (CE->getCastKind() == CK_AddressSpaceConversion)
           Addr = CGF.Builder.CreateAddrSpaceCast(
               Addr, CGF.ConvertType(E->getType()), ElemTy);
-        return Addr;
+        return CGF.authPointerToPointerCast(Addr, CE->getSubExpr()->getType(),
+                                            CE->getType());
       }
       break;
 
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 084dc54537eb7..a17d68424bbce 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -2374,7 +2374,9 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
       DestLV.setTBAAInfo(TBAAAccessInfo::getMayAliasInfo());
       return EmitLoadOfLValue(DestLV, CE->getExprLoc());
     }
-    return Builder.CreateBitCast(Src, DstTy);
+
+    llvm::Value *Result = Builder.CreateBitCast(Src, DstTy);
+    return CGF.authPointerToPointerCast(Result, E->getType(), DestTy);
   }
   case CK_AddressSpaceConversion: {
     Expr::EvalResult Result;
@@ -2524,6 +2526,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
       if (DestTy.mayBeDynamicClass())
         IntToPtr = Builder.CreateLaunderInvariantGroup(IntToPtr);
     }
+
+    IntToPtr = CGF.authPointerToPointerCast(IntToPtr, E->getType(), DestTy);
     return IntToPtr;
   }
   case CK_PointerToIntegral: {
@@ -2539,6 +2543,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
         PtrExpr = Builder.CreateStripInvariantGroup(PtrExpr);
     }
 
+    PtrExpr = CGF.authPointerToPointerCast(PtrExpr, E->getType(), DestTy);
     return Builder.CreatePtrToInt(PtrExpr, ConvertType(DestTy));
   }
   case CK_ToVoid: {
diff --git a/clang/lib/CodeGen/CGPointerAuth.cpp b/clang/lib/CodeGen/CGPointerAuth.cpp
index 621d567dde721..7fe62c0788742 100644
--- a/clang/lib/CodeGen/CGPointerAuth.cpp
+++ b/clang/lib/CodeGen/CGPointerAuth.cpp
@@ -15,6 +15,7 @@
 #include "CodeGenModule.h"
 #include "clang/CodeGen/CodeGenABITypes.h"
 #include "clang/CodeGen/ConstantInitBuilder.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Support/SipHash.h"
 
 using namespace clang;
@@ -165,6 +166,128 @@ CGPointerAuthInfo CodeGenModule::getPointerAuthInfoForType(QualType T) {
   return ::getPointerAuthInfoForType(*this, T);
 }
 
+static bool isZeroConstant(const llvm::Value *Value) {
+  if (const auto *CI = dyn_cast<llvm::ConstantInt>(Value))
+    return CI->isZero();
+  return false;
+}
+
+static bool equalAuthPolicies(const CGPointerAuthInfo &Left,
+                              const CGPointerAuthInfo &Right) {
+  assert((Left.isSigned() || Right.isSigned()) &&
+         "shouldn't be called if neither is signed");
+  if (Left.isSigned() != Right.isSigned())
+    return false;
+  return Left.getKey() == Right.getKey() &&
+         Left.getAuthenticationMode() == Right.getAuthenticationMode();
+}
+
+// Return the discriminator or return zero if the discriminator is null.
+static llvm::Value *getDiscriminatorOrZero(const CGPointerAuthInfo &Info,
+                                           CGBuilderTy &Builder) {
+  llvm::Value *Discriminator = Info.getDiscriminator();
+  return Discriminator ? Discriminator : Builder.getSize(0);
+}
+
+llvm::Value *
+CodeGenFunction::emitPointerAuthResignCall(llvm::Value *Value,
+                                           const CGPointerAuthInfo &CurAuth,
+                                           const CGPointerAuthInfo &NewAuth) {
+  assert(CurAuth && NewAuth);
+
+  if (CurAuth.getAuthenticationMode() !=
+          PointerAuthenticationMode::SignAndAuth ||
+      NewAuth.getAuthenticationMode() !=
+          PointerAuthenticationMode::SignAndAuth) {
+    llvm::Value *AuthedValue = EmitPointerAuthAuth(CurAuth, Value);
+    return EmitPointerAuthSign(NewAuth, AuthedValue);
+  }
+  // Convert the pointer to intptr_t before signing it.
+  auto *OrigType = Value->getType();
+  Value = Builder.CreatePtrToInt(Value, IntPtrTy);
+
+  auto *CurKey = Builder.getInt32(CurAuth.getKey());
+  auto *NewKey = Builder.getInt32(NewAuth.getKey());
+
+  llvm::Value *CurDiscriminator = getDiscriminatorOrZero(CurAuth, Builder);
+  llvm::Value *NewDiscriminator = getDiscriminatorOrZero(NewAuth, Builder);
+
+  // call i64 @llvm.ptrauth.resign(i64 %pointer,
+  //                               i32 %curKey, i64 %curDiscriminator,
+  //                               i32 %newKey, i64 %newDiscriminator)
+  auto *Intrinsic = CGM.getIntrinsic(llvm::Intrinsic::ptrauth_resign);
+  Value = EmitRuntimeCall(
+      Intrinsic, {Value, CurKey, CurDiscriminator, NewKey, NewDiscriminator});
+
+  // Convert back to the original type.
+  Value = Builder.CreateIntToPtr(Value, OrigType);
+  return Value;
+}
+
+llvm::Value *CodeGenFunction::emitPointerAuthResign(
+    llvm::Value *Value, QualType Type, const CGPointerAuthInfo &CurAuthInfo,
+    const CGPointerAuthInfo &NewAuthInfo, bool IsKnownNonNull) {
+  // Fast path: if neither schema wants a signature, we're done.
+  if (!CurAuthInfo && !NewAuthInfo)
+    return Value;
+
+  llvm::Value *Null = nullptr;
+  // If the value is obviously null, we're done.
+  if (auto *PointerValue = dyn_cast<llvm::PointerType>(Value->getType())) {
+    Null = CGM.getNullPointer(PointerValue, Type);
+  } else {
+    assert(Value->getType()->isIntegerTy());
+    Null = llvm::ConstantInt::get(IntPtrTy, 0);
+  }
+  if (Value == Null)
+    return Value;
+
+  // If both schemas sign the same way, we're done.
+  if (equalAuthPolicies(CurAuthInfo, NewAuthInfo)) {
+    const llvm::Value *CurD = CurAuthInfo.getDiscriminator();
+    const llvm::Value *NewD = NewAuthInfo.getDiscriminator();
+    if (CurD == NewD)
+      return Value;
+
+    if ((CurD == nullptr && isZeroConstant(NewD)) ||
+        (NewD == nullptr && isZeroConstant(CurD)))
+      return Value;
+  }
+
+  llvm::BasicBlock *InitBB = Builder.GetInsertBlock();
+  llvm::BasicBlock *ResignBB = nullptr, *ContBB = nullptr;
+
+  // Null pointers have to be mapped to null, and the ptrauth_resign
+  // intrinsic doesn't do that.
+  if (!IsKnownNonNull && !llvm::isKnownNonZero(Value, CGM.getDataLayout())) {
+    ContBB = createBasicBlock("resign.cont");
+    ResignBB = createBasicBlock("resign.nonnull");
+
+    auto *IsNonNull = Builder.CreateICmpNE(Value, Null);
+    Builder.CreateCondBr(IsNonNull, ResignBB, ContBB);
+    EmitBlock(ResignBB);
+  }
+
+  // Perform the auth/sign/resign operation.
+  if (!NewAuthInfo)
+    Value = EmitPointerAuthAuth(CurAuthInfo, Value);
+  else if (!CurAuthInfo)
+    Value = EmitPointerAuthSign(NewAuthInfo, Value);
+  else
+    Value = emitPointerAuthResignCall(Value, CurAuthInfo, NewAuthInfo);
+
+  // Clean up with a phi if we branched before.
+  if (ContBB) {
+    EmitBlock(ContBB);
+    auto *Phi = Builder.CreatePHI(Value->getType(), 2);
+    Phi->addIncoming(Null, InitBB);
+    Phi->addIncoming(Value, ResignBB);
+    Value = Phi;
+  }
+
+  return Value;
+}
+
 llvm::Constant *
 CodeGenModule::getConstantSignedPointer(llvm::Constant *Pointer, unsigned Key,
                                         llvm::Constant *StorageAddress,
@@ -351,3 +474,110 @@ CodeGenModule::getVTablePointerAuthInfo(CodeGenFunction *CGF,
                            /* IsIsaPointer */ false,
                            /* AuthenticatesNullValues */ false, Discriminator);
 }
+
+llvm::Value *CodeGenFunction::authPointerToPointerCast(llvm::Value *ResultPtr,
+                                                       QualType SourceType,
+                                                       QualType DestType) {
+  CGPointerAuthInfo CurAuthInfo, NewAuthInfo;
+  if (SourceType->isSignableType())
+    CurAuthInfo = getPointerAuthInfoForType(CGM, SourceType);
+
+  if (DestType->isSignableType())
+    NewAuthInfo = getPointerAuthInfoForType(CGM, DestType);
+
+  if (!CurAuthInfo && !NewAuthInfo)
+    return ResultPtr;
+
+  // If only one side of the cast is a function pointer, then we still need to
+  // resign to handle casts to/from opaque pointers.
+  if (!CurAuthInfo && DestType->isFunctionPointerType())
+    CurAuthInfo = CGM.getFunctionPointerAuthInfo(SourceType);
+
+  if (!NewAuthInfo && SourceType->isFunctionPointerType())
+    NewAuthInfo = CGM.getFunctionPointerAuthInfo(DestType);
+
+  return emitPointerAuthResign(ResultPtr, DestType, CurAuthInfo, NewAuthInfo,
+                               /*IsKnownNonNull=*/false);
+}
+
+Address CodeGenFunction::authPointerToPointerCast(Address Ptr,
+                                                  QualType SourceType,
+                                                  QualType DestType) {
+  CGPointerAuthInfo CurAuthInfo, NewAuthInfo;
+  if (SourceType->isSignableType())
+    CurAuthInfo = getPointerAuthInfoForType(CGM, SourceType);
+
+  if (DestType->isSignableType())
+    NewAuthInfo = getPointerAuthInfoForType(CGM, DestType);
+
+  if (!CurAuthInfo && !NewAuthInfo)
+    return Ptr;
+
+  if (!CurAuthInfo && DestType->isFunctionPointerType()) {
+    // When casting a non-signed pointer to a function pointer, just set the
+    // auth info on Ptr to the assumed schema. The pointer will be resigned to
+    // the effective type when used.
+    Ptr.setPointerAuthInfo(CGM.getFunctionPointerAuthInfo(SourceType));
+    return Ptr;
+  }
+
+  if (!NewAuthInfo && SourceType->isFunctionPointerType()) {
+    NewAuthInfo = CGM.getFunctionPointerAuthInfo(DestType);
+    Ptr = Ptr.getResignedAddress(NewAuthInfo, *this);
+    Ptr.setPointerAuthInfo(CGPointerAuthInfo());
+    return Ptr;
+  }
+
+  return Ptr;
+}
+
+Address CodeGenFunction::getAsNaturalAddressOf(Address Addr,
+                                               QualType PointeeTy) {
+  CGPointerAuthInfo Info =
+      PointeeTy.isNull() ? CGPointerAuthInfo()
+                         : CGM.getPointerAuthInfoForPointeeType(PointeeTy);
+  return Addr.getResignedAddress(Info, *this);
+}
+
+Address Address::getResignedAddress(const CGPointerAuthInfo &NewInfo,
+                                    CodeGenFunction &CGF) const {
+  assert(isValid() && "pointer isn't valid");
+  CGPointerAuthInfo CurInfo = getPointerAuthInfo();
+  llvm::Value *Val;
+
+  // Nothing to do if neither the current or the new ptrauth info needs signing.
+  if (!CurInfo.isSigned() && !NewInfo.isSigned())
+    return Address(getBasePointer(), getElementType(), getAlignment(),
+                   isKnownNonNull());
+
+  assert(ElementType && "Effective type has to be set");
+  assert(!Offset && "unexpected non-null offset");
+
+  // If the current and the new ptrauth infos are the same and the offset is
+  // null, just cast the base pointer to the effective type.
+  if (CurInfo == NewInfo && !hasOffset())
+    Val = getBasePointer();
+  else
+    Val = CGF.emitPointerAuthResign(getBasePointer(), QualType(), CurInfo,
+                                    NewInfo, isKnownNonNull());
+
+  Val = CGF.Builder.CreateBitCast(Val, getType());
+  return Address(Val, getElementType(), getAlignment(), NewInfo,
+                 /*Offset=*/nullptr, isKnownNonNull());
+}
+
+llvm::Value *LValue::getPointer(CodeGenFunction &CGF) const {
+  assert(isSimple());
+  return emitResignedPointer(getType(), CGF);
+}
+
+llvm::Value *LValue::emitResignedPointer(QualType PointeeTy,
+                                         CodeGenFunction &CGF) const {
+  assert(isSimple());
+  return CGF.getAsNaturalAddressOf(Addr, PointeeTy).getBasePointer();
+}
+
+llvm::Value *LValue::emitRawPointer(CodeGenFunction &CGF) const {
+  assert(isSimple());
+  return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr;
+}
diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h
index f1ba3cf95ae59..c4ec8d207d2e3 100644
--- a/clang/lib/CodeGen/CGValue.h
+++ b/clang/lib/CodeGen/CGValue.h
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_LIB_CODEGEN_CGVALUE_H
 
 #include "Address.h"
+#include "CGPointerAuthInfo.h"
 #include "CodeGenTBAA.h"
 #include "EHScopeStack.h"
 #include "clang/AST/ASTContext.h"
@@ -233,9 +234,6 @@ class LValue {
   // this lvalue.
   bool Nontemporal : 1;
 
-  // The pointer is known not to be null.
-  bool IsKnownNonNull : 1;
-
   LValueBaseInfo BaseInfo;
   TBAAAccessInfo TBAAInfo;
 
@@ -263,7 +261,6 @@ class LValue {
     this->ImpreciseLifetime = false;
     this->Nontemporal = false;
     this->ThreadLocalRef = false;
-    this->IsKnownNonNull = false;
     this->BaseIvarExp = nullptr;
   }
 
@@ -349,28 +346,26 @@ class LValue {
   LValueBaseInfo getBaseInfo() const { return BaseInfo; }
   void setBaseInfo(LValueBaseInfo Info) { BaseInfo = Info; }
 
-  KnownNonNull_t isKnownNonNull() const {
-    return (KnownNonNull_t)IsKnownNonNull;
-  }
+  KnownNonNull_t isKnownNonNull() const { return Addr.isKnownNonNull(); }
   LValue setKnownNonNull() {
-    IsKnownNonNull = true;
+    Addr.setKnownNonNull();
     return *this;
   }
 
   // simple lvalue
-  llvm::Value *getPointer(CodeGenFunction &CGF) const {
-    assert(isSimple());
-    return Addr.getBasePointer();
-  }
-  llvm::Value *emitRawPointer(CodeGenFunction &CGF) const {
-    assert(isSimple());
-    return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr;
-  }
+  llvm::Value *getPointer(CodeGenFunction &CGF) const;
+  llvm::Value *emitResignedPointer(QualType PointeeTy,
+                                   CodeGenFunction &CGF) const;
+  llvm::Value *emitRawPointer(CodeGenFunction &CGF) const;
 
   Address getAddress() const { return Addr; }
 
   void setAddress(Address address) { Addr = address; }
 
+  CGPointerAuthInfo getPointerAuthInfo() const {
+    return Addr.getPointerAuthInfo();
+  }
+
   // vector elt lvalue
   Address getVectorAddress() const {
     assert(isVectorElt());
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index ea4635c039cb2..551db09165dbe 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -195,34 +195,46 @@ CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
   CGF.Builder.setDefaultConstrainedRounding(OldRounding);
 }
 
-static LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
-                                         bool ForPointeeType,
-                                         CodeGenFunction &CGF) {
+static LValue
+makeNaturalAlignAddrLValue(llvm::Value *V, QualType T, bool ForPointeeType,
+                           bool MightBeSigned, CodeGenFunction &CGF,
+                           KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
   LValueBaseInfo BaseInfo;
   TBAAAccessInfo TBAAInfo;
   CharUnits Alignment =
       CGF.CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, ForPointeeType);
-  Address Addr = Address(V, CGF.ConvertTypeForMem(T), Alignment);
+  Address Addr =
+      MightBeSigned
+          ? CGF.makeNaturalAddressForPointer(V, T, Alignment, false, nullptr,
+                                             nullptr, IsKnownNonNull)
+          : Address(V, CGF.ConvertTypeForMem(T), Alignment, IsKnownNonNull);
   return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
 }
 
-LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
-  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this);
+LValue
+CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
+                                            KnownNonNull_t IsKnownNonNull) {
+  return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
+                                      /*MightBeSigned*/ true, *this,
+                                      IsKnownNonNull);
 }
 
 LValue
 CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
-  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this);
+  return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
+                                      /*MightBeSigned*/ true, *this);
 }
 
 LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,
                                                       QualType T) {
-  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this);
+  return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
+                                      /*MightBeSigned*/ false, *this);
 }
 
 LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,
                                                              QualType T) {
-  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this);
+  return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
+                                      /*MightBeSigned*/ false, *this);
 }
 
 llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 9fe4391237819..d83e38cab8e2d 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -202,7 +202,7 @@ template <> struct DominatingValue<Address> {
   }
   static type restore(CodeGenFunction &CGF, saved_type value) {
     return Address(DominatingLLVMValue::restore(CGF, value.BasePtr),
-                   value.ElementType, value.Alignment,
+                   value.ElementType, value.Alignment, CGPointerAuthInfo(),
                    DominatingLLVMValue::restore(CGF, value.Offset));
   }
 };
@@ -2689,7 +2689,8 @@ class CodeGenFunction : public CodeGenTypeCache {
     if (Alignment.isZero())
       Alignment =
           CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, ForPointeeType);
-    return Address(Ptr, ConvertTypeForMem(T), Alignment, nullptr,
+    return Address(Ptr, ConvertTypeForMem(T), Alignment,
+                   CGM.getPointerAuthInfoForPointeeType(T), /*Offset=*/nullptr,
                    IsKnownNonNull);
   }
 
@@ -2730,7 +2731,9 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// an l-value with the natural pointee alignment of T.
   LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
 
-  LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T);
+  LValue
+  MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
+                             KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
 
   /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
   /// to be unsigned.
@@ -4424,10 +4427,6 @@ class CodeGenFunction : public CodeGenTypeCache {
                                                CXXDtorType Type,
                                                const CXXRecordDecl *RD);
 
-  llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
-    return Addr.getBasePointer();
-  }
-
   bool isPointerKnownNonNull(const Expr *E);
 
   /// Create the discriminator from the storage address and the entity hash.
@@ -4437,16 +4436,36 @@ class CodeGenFunction : public CodeGenTypeCache {
                                         llvm::Value *StorageAddress,
                                         GlobalDecl SchemaDecl,
                                         QualType SchemaType);
-  llvm::Value *EmitPointerAuthSign(QualType PointeeType, llvm::Value *Pointer);
+
   llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info,
                                    llvm::Value *Pointer);
+
   llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info,
                                    llvm::Value *Pointer);
 
+  llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType,
+                                     const CGPointerAuthInfo &CurAuthInfo,
+                                     const CGPointerAuthInfo &NewAuthInfo,
+                                     bool IsKnownNonNull);
+  llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer,
+                                         const CGPointerAuthInfo &CurInfo,
+                                         const CGPointerAuthInfo &NewInfo);
+
   void EmitPointerAuthOperandBundle(
       const CGPointerAuthInfo &Info,
       SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
 
+  llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,
+                                        QualType SourceType, QualType DestType);
+  Address authPointerToPointerCast(Address Ptr, QualType SourceType,
+                                   QualType DestType);
+
+  Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy);
+
+  llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
+    return getAsNaturalAddressOf(Addr, PointeeType).getBasePointer();
+  }
+
   // Return the copy constructor name with the prefix "__copy_constructor_"
   // removed.
   static std::string getNonTrivialCopyConstructorStr(QualType QT,
diff --git a/clang/lib/Headers/ptrauth.h b/clang/lib/Headers/ptrauth.h
index 40ac6dcac2ab8..e0bc8c4f9acf7 100644
--- a/clang/lib/Headers/ptrauth.h
+++ b/clang/lib/Headers/ptrauth.h
@@ -58,6 +58,21 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
 /* Authenticating a pointer that was not signed with the given key
    and extra-data value will (likely) fail by trapping. */
 
+/* The null function pointer is always the all-zero bit pattern.
+   Signing an all-zero bit pattern will embed a (likely) non-zero
+   signature in the result, and so the result will not seem to be
+   a null function pointer.  Authenticating this value will yield
+   a null function pointer back.  However, authenticating an
+   all-zero bit pattern will probably fail, because the
+   authentication will expect a (likely) non-zero signature to
+   embedded in the value.
+
+   Because of this, if a pointer may validly be null, you should
+   check for null before attempting to authenticate it with one
+   of these intrinsics.  This is not necessary when using the
+   __ptrauth qualifier; the compiler will perform this check
+   automatically. */
+
 #if __has_feature(ptrauth_intrinsics)
 
 /* Strip the signature from a value without authenticating it.
diff --git a/clang/test/CodeGen/ptrauth-function-lvalue-cast-disc.c b/clang/test/CodeGen/ptrauth-function-lvalue-cast-disc.c
new file mode 100644
index 0000000000000..7d76649e2e49c
--- /dev/null
+++ b/clang/test/CodeGen/ptrauth-function-lvalue-cast-disc.c
@@ -0,0 +1,68 @@
+// RUN: %clang_cc1 %s -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -emit-llvm -o- -fptrauth-function-pointer-type-discrimination | FileCheck -check-prefixes CHECK,TYPE %s
+// RUN: %clang_cc1 %s -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -emit-llvm -o- | FileCheck -check-prefixes CHECK,ZERO %s
+
+typedef void (*fptr_t)(void);
+
+char *cptr;
+void (*fptr)(void);
+
+// CHECK-LABEL: define void @test1
+void test1() {
+  // TYPE: [[LOAD:%.*]] = load ptr, ptr @cptr
+  // TYPE: [[TOINT:%.*]] = ptrtoint ptr [[LOAD]] to i64
+  // TYPE: call i64 @llvm.ptrauth.resign(i64 [[TOINT]], i32 0, i64 0, i32 0, i64 18983)
+  // TYPE: call void {{.*}}() [ "ptrauth"(i32 0, i64 18983) ]
+  // ZERO-NOT: @llvm.ptrauth.resign
+
+  (*(fptr_t)cptr)();
+}
+
+// CHECK-LABEL: define i8 @test2
+char test2() {
+  return *(char *)fptr;
+
+  // TYPE: [[LOAD:%.*]] = load ptr, ptr @fptr
+  // TYPE: [[CMP:%.*]] = icmp ne ptr [[LOAD]], null
+  // TYPE-NEXT: br i1 [[CMP]], label %[[NONNULL:.*]], label %[[CONT:.*]]
+
+  // TYPE: [[NONNULL]]:
+  // TYPE: [[TOINT:%.*]] = ptrtoint ptr [[LOAD]] to i64
+  // TYPE: [[CALL:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[TOINT]], i32 0, i64 18983, i32 0, i64 0)
+  // TYPE: [[TOPTR:%.*]] = inttoptr i64 [[CALL]] to ptr
+
+  // TYPE: [[CONT]]:
+  // TYPE: phi ptr [ null, {{.*}} ], [ [[TOPTR]], %[[NONNULL]] ]
+  // ZERO-NOT: @llvm.ptrauth.resign
+}
+
+// CHECK-LABEL: define void @test4
+void test4() {
+  (*((fptr_t)(&*((char *)(&*(fptr_t)cptr)))))();
+
+  // CHECK: [[LOAD:%.*]] = load ptr, ptr @cptr
+  // TYPE-NEXT: [[CAST4:%.*]] = ptrtoint ptr [[LOAD]] to i64
+  // TYPE-NEXT: [[RESIGN:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[CAST4]], i32 0, i64 0, i32 0, i64 18983)
+  // TYPE-NEXT: [[CAST5:%.*]] = inttoptr i64 [[RESIGN]] to ptr
+  // TYPE-NEXT: call void [[CAST5]]() [ "ptrauth"(i32 0, i64 18983) ]
+  // ZERO-NOT: @llvm.ptrauth.resign
+  // ZERO: call void [[LOAD]]() [ "ptrauth"(i32 0, i64 0) ]
+}
+
+void *vptr;
+// CHECK-LABEL: define void @test5
+void test5() {
+  vptr = &*(char *)fptr;
+
+  // TYPE: [[LOAD:%.*]] = load ptr, ptr @fptr
+  // TYPE-NEXT: [[CMP]] = icmp ne ptr [[LOAD]], null
+  // TYPE-NEXT: br i1 [[CMP]], label %[[NONNULL:.*]], label %[[CONT:.*]]
+
+  // TYPE: [[NONNULL]]:
+  // TYPE: [[RESIGN:%.*]] = call i64 @llvm.ptrauth.resign(i64 {{.*}}, i32 0, i64 18983, i32 0, i64 0)
+  // TYPE: [[CAST:%.*]] = inttoptr i64 [[RESIGN]] to ptr
+
+  // TYPE: [[CONT]]:
+  // TYPE: [[PHI:%.*]] = phi ptr [ null, {{.*}} ], [ [[CAST]], %[[NONNULL]] ]
+  // TYPE: store ptr [[PHI]], ptr @vptr
+  // ZERO-NOT: @llvm.ptrauth.resign
+}
diff --git a/clang/test/CodeGen/ptrauth-function-type-discriminator-cast.c b/clang/test/CodeGen/ptrauth-function-type-discriminator-cast.c
new file mode 100644
index 0000000000000..cdf9ee4907525
--- /dev/null
+++ b/clang/test/CodeGen/ptrauth-function-type-discriminator-cast.c
@@ -0,0 +1,94 @@
+// RUN: %clang_cc1 %s -fptrauth-function-pointer-type-discrimination -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck %s --check-prefixes=CHECK,TYPE
+// RUN: %clang_cc1 %s -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck %s --check-prefixes=CHECK,ZERO
+// RUN: %clang_cc1 -xc++ %s -fptrauth-function-pointer-type-discrimination -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck %s --check-prefixes=CHECK,CHECKCXX,TYPE,TYPECXX
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void f(void);
+void f2(int);
+void (*fptr)(void);
+void *opaque;
+unsigned long uintptr;
+
+#ifdef __cplusplus
+struct ptr_member {
+  void (*fptr_)(int) = 0;
+};
+ptr_member pm;
+void (*test_member)() = (void (*)())pm.fptr_;
+
+// CHECKCXX-LABEL: define internal void @__cxx_global_var_init
+// TYPECXX: call i64 @llvm.ptrauth.resign(i64 {{.*}}, i32 0, i64 2712, i32 0, i64 18983)
+#endif
+
+
+// CHECK-LABEL: define void @test_cast_to_opaque
+void test_cast_to_opaque() {
+  opaque = (void *)f;
+
+  // TYPE: [[RESIGN_VAL:%.*]] = call i64 @llvm.ptrauth.resign(i64 ptrtoint (ptr ptrauth (ptr @f, i32 0, i64 18983) to i64), i32 0, i64 18983, i32 0, i64 0)
+  // TYPE: [[RESIGN_PTR:%.*]] = inttoptr i64 [[RESIGN_VAL]] to ptr
+  // ZERO-NOT: @llvm.ptrauth.resign
+}
+
+// CHECK-LABEL: define void @test_cast_from_opaque
+void test_cast_from_opaque() {
+  fptr = (void (*)(void))opaque;
+
+  // TYPE: [[LOAD:%.*]] = load ptr, ptr @opaque
+  // TYPE: [[CMP:%.*]] = icmp ne ptr [[LOAD]], null
+  // TYPE: br i1 [[CMP]], label %[[RESIGN_LAB:.*]], label
+
+  // TYPE: [[RESIGN_LAB]]:
+  // TYPE: [[INT:%.*]] = ptrtoint ptr [[LOAD]] to i64
+  // TYPE: [[RESIGN_INT:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[INT]], i32 0, i64 0, i32 0, i64 18983)
+
+  // ZERO-NOT: @llvm.ptrauth.resign
+}
+
+// CHECK-LABEL: define void @test_cast_to_intptr
+void test_cast_to_intptr() {
+  uintptr = (unsigned long)fptr;
+
+  // TYPE: [[ENTRY:.*]]:
+  // TYPE: [[LOAD:%.*]] = load ptr, ptr @fptr
+  // TYPE: [[CMP:%.*]] = icmp ne ptr [[LOAD]], null
+  // TYPE: br i1 [[CMP]], label %[[RESIGN_LAB:.*]], label %[[RESIGN_CONT:.*]]
+
+  // TYPE: [[RESIGN_LAB]]:
+  // TYPE: [[INT:%.*]] = ptrtoint ptr [[LOAD]] to i64
+  // TYPE: [[RESIGN_INT:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[INT]], i32 0, i64 18983, i32 0, i64 0)
+  // TYPE: [[RESIGN:%.*]] = inttoptr i64 [[RESIGN_INT]] to ptr
+  // TYPE: br label %[[RESIGN_CONT]]
+
+  // TYPE: [[RESIGN_CONT]]:
+  // TYPE: phi ptr [ null, %[[ENTRY]] ], [ [[RESIGN]], %[[RESIGN_LAB]] ]
+
+  // ZERO-NOT: @llvm.ptrauth.resign
+}
+
+// CHECK-LABEL: define void @test_function_to_function_cast
+void test_function_to_function_cast() {
+  void (*fptr2)(int) = (void (*)(int))fptr;
+  // TYPE: call i64 @llvm.ptrauth.resign(i64 {{.*}}, i32 0, i64 18983, i32 0, i64 2712)
+  // ZERO-NOT: @llvm.ptrauth.resign
+}
+
+// CHECK-LABEL: define void @test_call_lvalue_cast
+void test_call_lvalue_cast() {
+  (*(void (*)(int))f)(42);
+
+  // TYPE: entry:
+  // TYPE-NEXT: [[RESIGN:%.*]] = call i64 @llvm.ptrauth.resign(i64 ptrtoint (ptr ptrauth (ptr @f, i32 0, i64 18983) to i64), i32 0, i64 18983, i32 0, i64 2712)
+  // TYPE-NEXT: [[RESIGN_INT:%.*]] = inttoptr i64 [[RESIGN]] to ptr
+  // TYPE-NEXT: call void [[RESIGN_INT]](i32 noundef 42) [ "ptrauth"(i32 0, i64 2712) ]
+  // ZERO-NOT: @llvm.ptrauth.resign
+  // ZERO: call void ptrauth (ptr @f, i32 0)(i32 noundef 42) [ "ptrauth"(i32 0, i64 0) ]
+}
+
+
+#ifdef __cplusplus
+}
+#endif

From 676efd0ffb717215c752f200fe14163732290dcc Mon Sep 17 00:00:00 2001
From: Jeremy Morse <jeremy.morse@sony.com>
Date: Thu, 18 Jul 2024 15:22:07 +0100
Subject: [PATCH 033/486] Reapply 078198f310d5 "Index DebugVariables and some
 DILocations"

Now revised to actually make the unit test compile, which I'd been
ignoring. No actual functional change, it's a type difference.
Original commit message follows.

[DebugInfo][InstrRef] Index DebugVariables and some DILocations (#99318)

A lot of time in LiveDebugValues is spent computing DenseMap keys for
DebugVariables, and they're made up of three pointers, so are large.
This patch installs an index for them: for the SSA and value-to-location
mapping parts of InstrRefBasedLDV we don't need to access things like
the variable declaration or the inlining site, so just use a uint32_t
identifier for each variable fragment that's tracked. The compile-time
performance improvements are substantial (almost 0.4% on the tracker).

About 80% of this patch is just replacing DebugVariable references with
DebugVariableIDs instead, however there are some larger consequences. We
spend lots of time fetching DILocations when emitting DBG_VALUE
instructions, so index those with the DebugVariables: this means all
DILocations on all new DBG_VALUE instructions will normalise to the
first-seen DILocation for the variable (which should be fine).

We also used to keep an ordering of when each variable was seen first in
a DBG_* instruction, in the AllVarsNumbering collection, so that we can
emit new DBG_* instructions in a stable order. We can hang this off the
DebugVariable index instead, so AllVarsNumbering is deleted.

Finally, rather than ordering by AllVarsNumbering just before DBG_*
instructions are linked into the output MIR, store instructions along
with their DebugVariableID, so that they can be sorted by that instead.
---
 .../LiveDebugValues/InstrRefBasedImpl.cpp     | 201 +++++++++---------
 .../LiveDebugValues/InstrRefBasedImpl.h       | 111 +++++++---
 .../MIR/X86/live-debug-values-fragments.mir   |   4 +-
 llvm/unittests/CodeGen/InstrRefLDVTest.cpp    | 129 +++++------
 4 files changed, 256 insertions(+), 189 deletions(-)

diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
index 247258a1ff553..b9cf36a07846c 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
@@ -183,6 +183,7 @@ class TransferTracker {
   /// information from it. (XXX make it const?)
   MLocTracker *MTracker;
   MachineFunction &MF;
+  const DebugVariableMap &DVMap;
   bool ShouldEmitDebugEntryValues;
 
   /// Record of all changes in variable locations at a block position. Awkwardly
@@ -191,7 +192,9 @@ class TransferTracker {
   struct Transfer {
     MachineBasicBlock::instr_iterator Pos; /// Position to insert DBG_VALUes
     MachineBasicBlock *MBB; /// non-null if we should insert after.
-    SmallVector<MachineInstr *, 4> Insts; /// Vector of DBG_VALUEs to insert.
+    /// Vector of DBG_VALUEs to insert. Store with their DebugVariableID so that
+    /// they can be sorted into a stable order for emission at a later time.
+    SmallVector<std::pair<DebugVariableID, MachineInstr *>, 4> Insts;
   };
 
   /// Stores the resolved operands (machine locations and constants) and
@@ -227,15 +230,15 @@ class TransferTracker {
   /// Map from LocIdxes to which DebugVariables are based that location.
   /// Mantained while stepping through the block. Not accurate if
   /// VarLocs[Idx] != MTracker->LocIdxToIDNum[Idx].
-  DenseMap<LocIdx, SmallSet<DebugVariable, 4>> ActiveMLocs;
+  DenseMap<LocIdx, SmallSet<DebugVariableID, 4>> ActiveMLocs;
 
   /// Map from DebugVariable to it's current location and qualifying meta
   /// information. To be used in conjunction with ActiveMLocs to construct
   /// enough information for the DBG_VALUEs for a particular LocIdx.
-  DenseMap<DebugVariable, ResolvedDbgValue> ActiveVLocs;
+  DenseMap<DebugVariableID, ResolvedDbgValue> ActiveVLocs;
 
   /// Temporary cache of DBG_VALUEs to be entered into the Transfers collection.
-  SmallVector<MachineInstr *, 4> PendingDbgValues;
+  SmallVector<std::pair<DebugVariableID, MachineInstr *>, 4> PendingDbgValues;
 
   /// Record of a use-before-def: created when a value that's live-in to the
   /// current block isn't available in any machine location, but it will be
@@ -244,12 +247,12 @@ class TransferTracker {
     /// Value of this variable, def'd in block.
     SmallVector<DbgOp> Values;
     /// Identity of this variable.
-    DebugVariable Var;
+    DebugVariableID VarID;
     /// Additional variable properties.
     DbgValueProperties Properties;
-    UseBeforeDef(ArrayRef<DbgOp> Values, const DebugVariable &Var,
+    UseBeforeDef(ArrayRef<DbgOp> Values, DebugVariableID VarID,
                  const DbgValueProperties &Properties)
-        : Values(Values.begin(), Values.end()), Var(Var),
+        : Values(Values.begin(), Values.end()), VarID(VarID),
           Properties(Properties) {}
   };
 
@@ -260,15 +263,16 @@ class TransferTracker {
   /// The set of variables that are in UseBeforeDefs and can become a location
   /// once the relevant value is defined. An element being erased from this
   /// collection prevents the use-before-def materializing.
-  DenseSet<DebugVariable> UseBeforeDefVariables;
+  DenseSet<DebugVariableID> UseBeforeDefVariables;
 
   const TargetRegisterInfo &TRI;
   const BitVector &CalleeSavedRegs;
 
   TransferTracker(const TargetInstrInfo *TII, MLocTracker *MTracker,
-                  MachineFunction &MF, const TargetRegisterInfo &TRI,
+                  MachineFunction &MF, const DebugVariableMap &DVMap,
+                  const TargetRegisterInfo &TRI,
                   const BitVector &CalleeSavedRegs, const TargetPassConfig &TPC)
-      : TII(TII), MTracker(MTracker), MF(MF), TRI(TRI),
+      : TII(TII), MTracker(MTracker), MF(MF), DVMap(DVMap), TRI(TRI),
         CalleeSavedRegs(CalleeSavedRegs) {
     TLI = MF.getSubtarget().getTargetLowering();
     auto &TM = TPC.getTM<TargetMachine>();
@@ -352,7 +356,7 @@ class TransferTracker {
   ///    determine the values used by Value.
   void loadVarInloc(MachineBasicBlock &MBB, DbgOpIDMap &DbgOpStore,
                     const SmallVectorImpl<ValueLocPair> &ValueToLoc,
-                    DebugVariable Var, DbgValue Value) {
+                    DebugVariableID VarID, DbgValue Value) {
     SmallVector<DbgOp> DbgOps;
     SmallVector<ResolvedDbgOp> ResolvedDbgOps;
     bool IsValueValid = true;
@@ -401,7 +405,7 @@ class TransferTracker {
                                       static_cast<unsigned>(Num.getInst()));
           continue;
         }
-        recoverAsEntryValue(Var, Value.Properties, Num);
+        recoverAsEntryValue(VarID, Value.Properties, Num);
         IsValueValid = false;
         break;
       }
@@ -419,8 +423,7 @@ class TransferTracker {
 
     // Add UseBeforeDef entry for the last value to be defined in this block.
     if (LastUseBeforeDef) {
-      addUseBeforeDef(Var, Value.Properties, DbgOps,
-                      LastUseBeforeDef);
+      addUseBeforeDef(VarID, Value.Properties, DbgOps, LastUseBeforeDef);
       return;
     }
 
@@ -428,13 +431,15 @@ class TransferTracker {
     // the transfer.
     for (const ResolvedDbgOp &Op : ResolvedDbgOps)
       if (!Op.IsConst)
-        ActiveMLocs[Op.Loc].insert(Var);
+        ActiveMLocs[Op.Loc].insert(VarID);
     auto NewValue = ResolvedDbgValue{ResolvedDbgOps, Value.Properties};
-    auto Result = ActiveVLocs.insert(std::make_pair(Var, NewValue));
+    auto Result = ActiveVLocs.insert(std::make_pair(VarID, NewValue));
     if (!Result.second)
       Result.first->second = NewValue;
+    auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
     PendingDbgValues.push_back(
-        MTracker->emitLoc(ResolvedDbgOps, Var, Value.Properties));
+        std::make_pair(VarID, &*MTracker->emitLoc(ResolvedDbgOps, Var, DILoc,
+                                                  Value.Properties)));
   }
 
   /// Load object with live-in variable values. \p mlocs contains the live-in
@@ -445,7 +450,7 @@ class TransferTracker {
   /// FIXME: could just examine mloctracker instead of passing in \p mlocs?
   void
   loadInlocs(MachineBasicBlock &MBB, ValueTable &MLocs, DbgOpIDMap &DbgOpStore,
-             const SmallVectorImpl<std::pair<DebugVariable, DbgValue>> &VLocs,
+             const SmallVectorImpl<std::pair<DebugVariableID, DbgValue>> &VLocs,
              unsigned NumLocs) {
     ActiveMLocs.clear();
     ActiveVLocs.clear();
@@ -506,11 +511,11 @@ class TransferTracker {
 
   /// Record that \p Var has value \p ID, a value that becomes available
   /// later in the function.
-  void addUseBeforeDef(const DebugVariable &Var,
+  void addUseBeforeDef(DebugVariableID VarID,
                        const DbgValueProperties &Properties,
                        const SmallVectorImpl<DbgOp> &DbgOps, unsigned Inst) {
-    UseBeforeDefs[Inst].emplace_back(DbgOps, Var, Properties);
-    UseBeforeDefVariables.insert(Var);
+    UseBeforeDefs[Inst].emplace_back(DbgOps, VarID, Properties);
+    UseBeforeDefVariables.insert(VarID);
   }
 
   /// After the instruction at index \p Inst and position \p pos has been
@@ -529,7 +534,7 @@ class TransferTracker {
     // Populate ValueToLoc with illegal default mappings for every value used by
     // any UseBeforeDef variables for this instruction.
     for (auto &Use : MIt->second) {
-      if (!UseBeforeDefVariables.count(Use.Var))
+      if (!UseBeforeDefVariables.count(Use.VarID))
         continue;
 
       for (DbgOp &Op : Use.Values) {
@@ -568,7 +573,7 @@ class TransferTracker {
     // Using the map of values to locations, produce a final set of values for
     // this variable.
     for (auto &Use : MIt->second) {
-      if (!UseBeforeDefVariables.count(Use.Var))
+      if (!UseBeforeDefVariables.count(Use.VarID))
         continue;
 
       SmallVector<ResolvedDbgOp> DbgOps;
@@ -591,8 +596,9 @@ class TransferTracker {
         continue;
 
       // Otherwise, we're good to go.
-      PendingDbgValues.push_back(
-          MTracker->emitLoc(DbgOps, Use.Var, Use.Properties));
+      auto &[Var, DILoc] = DVMap.lookupDVID(Use.VarID);
+      PendingDbgValues.push_back(std::make_pair(
+          Use.VarID, MTracker->emitLoc(DbgOps, Var, DILoc, Use.Properties)));
     }
     flushDbgValues(pos, nullptr);
   }
@@ -642,7 +648,7 @@ class TransferTracker {
     return Reg != SP && Reg != FP;
   }
 
-  bool recoverAsEntryValue(const DebugVariable &Var,
+  bool recoverAsEntryValue(DebugVariableID VarID,
                            const DbgValueProperties &Prop,
                            const ValueIDNum &Num) {
     // Is this variable location a candidate to be an entry value. First,
@@ -663,6 +669,8 @@ class TransferTracker {
       DIExpr = *NonVariadicExpression;
     }
 
+    auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
+
     // Is the variable appropriate for entry values (i.e., is a parameter).
     if (!isEntryValueVariable(Var, DIExpr))
       return false;
@@ -676,9 +684,8 @@ class TransferTracker {
         DIExpression::prepend(DIExpr, DIExpression::EntryValue);
     Register Reg = MTracker->LocIdxToLocID[Num.getLoc()];
     MachineOperand MO = MachineOperand::CreateReg(Reg, false);
-
-    PendingDbgValues.push_back(
-        emitMOLoc(MO, Var, {NewExpr, Prop.Indirect, false}));
+    PendingDbgValues.push_back(std::make_pair(
+        VarID, &*emitMOLoc(MO, Var, {NewExpr, Prop.Indirect, false})));
     return true;
   }
 
@@ -687,19 +694,20 @@ class TransferTracker {
     DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
                       MI.getDebugLoc()->getInlinedAt());
     DbgValueProperties Properties(MI);
+    DebugVariableID VarID = DVMap.getDVID(Var);
 
     // Ignore non-register locations, we don't transfer those.
     if (MI.isUndefDebugValue() ||
         all_of(MI.debug_operands(),
                [](const MachineOperand &MO) { return !MO.isReg(); })) {
-      auto It = ActiveVLocs.find(Var);
+      auto It = ActiveVLocs.find(VarID);
       if (It != ActiveVLocs.end()) {
         for (LocIdx Loc : It->second.loc_indices())
-          ActiveMLocs[Loc].erase(Var);
+          ActiveMLocs[Loc].erase(VarID);
         ActiveVLocs.erase(It);
       }
       // Any use-before-defs no longer apply.
-      UseBeforeDefVariables.erase(Var);
+      UseBeforeDefVariables.erase(VarID);
       return;
     }
 
@@ -725,14 +733,15 @@ class TransferTracker {
                 SmallVectorImpl<ResolvedDbgOp> &NewLocs) {
     DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
                       MI.getDebugLoc()->getInlinedAt());
+    DebugVariableID VarID = DVMap.getDVID(Var);
     // Any use-before-defs no longer apply.
-    UseBeforeDefVariables.erase(Var);
+    UseBeforeDefVariables.erase(VarID);
 
     // Erase any previous location.
-    auto It = ActiveVLocs.find(Var);
+    auto It = ActiveVLocs.find(VarID);
     if (It != ActiveVLocs.end()) {
       for (LocIdx Loc : It->second.loc_indices())
-        ActiveMLocs[Loc].erase(Var);
+        ActiveMLocs[Loc].erase(VarID);
     }
 
     // If there _is_ no new location, all we had to do was erase.
@@ -742,7 +751,7 @@ class TransferTracker {
       return;
     }
 
-    SmallVector<std::pair<LocIdx, DebugVariable>> LostMLocs;
+    SmallVector<std::pair<LocIdx, DebugVariableID>> LostMLocs;
     for (ResolvedDbgOp &Op : NewLocs) {
       if (Op.IsConst)
         continue;
@@ -769,17 +778,17 @@ class TransferTracker {
         for (const auto &LostMLoc : LostMLocs)
           ActiveMLocs[LostMLoc.first].erase(LostMLoc.second);
         LostMLocs.clear();
-        It = ActiveVLocs.find(Var);
+        It = ActiveVLocs.find(VarID);
         ActiveMLocs[NewLoc.asU64()].clear();
         VarLocs[NewLoc.asU64()] = MTracker->readMLoc(NewLoc);
       }
 
-      ActiveMLocs[NewLoc].insert(Var);
+      ActiveMLocs[NewLoc].insert(VarID);
     }
 
     if (It == ActiveVLocs.end()) {
       ActiveVLocs.insert(
-          std::make_pair(Var, ResolvedDbgValue(NewLocs, Properties)));
+          std::make_pair(VarID, ResolvedDbgValue(NewLocs, Properties)));
     } else {
       It->second.Ops.assign(NewLocs);
       It->second.Properties = Properties;
@@ -822,21 +831,21 @@ class TransferTracker {
     // explicitly undef, then stop here.
     if (!NewLoc && !MakeUndef) {
       // Try and recover a few more locations with entry values.
-      for (const auto &Var : ActiveMLocIt->second) {
-        auto &Prop = ActiveVLocs.find(Var)->second.Properties;
-        recoverAsEntryValue(Var, Prop, OldValue);
+      for (DebugVariableID VarID : ActiveMLocIt->second) {
+        auto &Prop = ActiveVLocs.find(VarID)->second.Properties;
+        recoverAsEntryValue(VarID, Prop, OldValue);
       }
       flushDbgValues(Pos, nullptr);
       return;
     }
 
     // Examine all the variables based on this location.
-    DenseSet<DebugVariable> NewMLocs;
+    DenseSet<DebugVariableID> NewMLocs;
     // If no new location has been found, every variable that depends on this
     // MLoc is dead, so end their existing MLoc->Var mappings as well.
-    SmallVector<std::pair<LocIdx, DebugVariable>> LostMLocs;
-    for (const auto &Var : ActiveMLocIt->second) {
-      auto ActiveVLocIt = ActiveVLocs.find(Var);
+    SmallVector<std::pair<LocIdx, DebugVariableID>> LostMLocs;
+    for (DebugVariableID VarID : ActiveMLocIt->second) {
+      auto ActiveVLocIt = ActiveVLocs.find(VarID);
       // Re-state the variable location: if there's no replacement then NewLoc
       // is std::nullopt and a $noreg DBG_VALUE will be created. Otherwise, a
       // DBG_VALUE identifying the alternative location will be emitted.
@@ -855,19 +864,21 @@ class TransferTracker {
         replace_copy(ActiveVLocIt->second.Ops, DbgOps.begin(), OldOp, NewOp);
       }
 
-      PendingDbgValues.push_back(MTracker->emitLoc(DbgOps, Var, Properties));
+      auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
+      PendingDbgValues.push_back(std::make_pair(
+          VarID, &*MTracker->emitLoc(DbgOps, Var, DILoc, Properties)));
 
       // Update machine locations <=> variable locations maps. Defer updating
       // ActiveMLocs to avoid invalidating the ActiveMLocIt iterator.
       if (!NewLoc) {
         for (LocIdx Loc : ActiveVLocIt->second.loc_indices()) {
           if (Loc != MLoc)
-            LostMLocs.emplace_back(Loc, Var);
+            LostMLocs.emplace_back(Loc, VarID);
         }
         ActiveVLocs.erase(ActiveVLocIt);
       } else {
         ActiveVLocIt->second.Ops = DbgOps;
-        NewMLocs.insert(Var);
+        NewMLocs.insert(VarID);
       }
     }
 
@@ -891,8 +902,8 @@ class TransferTracker {
     // Commit ActiveMLoc changes.
     ActiveMLocIt->second.clear();
     if (!NewMLocs.empty())
-      for (auto &Var : NewMLocs)
-        ActiveMLocs[*NewLoc].insert(Var);
+      for (DebugVariableID VarID : NewMLocs)
+        ActiveMLocs[*NewLoc].insert(VarID);
   }
 
   /// Transfer variables based on \p Src to be based on \p Dst. This handles
@@ -915,17 +926,18 @@ class TransferTracker {
     // For each variable based on Src; create a location at Dst.
     ResolvedDbgOp SrcOp(Src);
     ResolvedDbgOp DstOp(Dst);
-    for (const auto &Var : MovingVars) {
-      auto ActiveVLocIt = ActiveVLocs.find(Var);
+    for (DebugVariableID VarID : MovingVars) {
+      auto ActiveVLocIt = ActiveVLocs.find(VarID);
       assert(ActiveVLocIt != ActiveVLocs.end());
 
       // Update all instances of Src in the variable's tracked values to Dst.
       std::replace(ActiveVLocIt->second.Ops.begin(),
                    ActiveVLocIt->second.Ops.end(), SrcOp, DstOp);
 
-      MachineInstr *MI = MTracker->emitLoc(ActiveVLocIt->second.Ops, Var,
+      auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
+      MachineInstr *MI = MTracker->emitLoc(ActiveVLocIt->second.Ops, Var, DILoc,
                                            ActiveVLocIt->second.Properties);
-      PendingDbgValues.push_back(MI);
+      PendingDbgValues.push_back(std::make_pair(VarID, MI));
     }
     ActiveMLocs[Src].clear();
     flushDbgValues(Pos, nullptr);
@@ -1176,11 +1188,9 @@ LLVM_DUMP_METHOD void MLocTracker::dump_mloc_map() {
 
 MachineInstrBuilder
 MLocTracker::emitLoc(const SmallVectorImpl<ResolvedDbgOp> &DbgOps,
-                     const DebugVariable &Var,
+                     const DebugVariable &Var, const DILocation *DILoc,
                      const DbgValueProperties &Properties) {
-  DebugLoc DL = DILocation::get(Var.getVariable()->getContext(), 0, 0,
-                                Var.getVariable()->getScope(),
-                                const_cast<DILocation *>(Var.getInlinedAt()));
+  DebugLoc DL = DebugLoc(DILoc);
 
   const MCInstrDesc &Desc = Properties.IsVariadic
                                 ? TII.get(TargetOpcode::DBG_VALUE_LIST)
@@ -1726,7 +1736,8 @@ bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI,
       LastUseBeforeDef = std::max(LastUseBeforeDef, NewID.getInst());
     }
     if (IsValidUseBeforeDef) {
-      TTracker->addUseBeforeDef(V, {MI.getDebugExpression(), false, true},
+      DebugVariableID VID = DVMap.insertDVID(V, MI.getDebugLoc().get());
+      TTracker->addUseBeforeDef(VID, {MI.getDebugExpression(), false, true},
                                 DbgOps, LastUseBeforeDef);
     }
   }
@@ -1735,9 +1746,11 @@ bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI,
   // This DBG_VALUE is potentially a $noreg / undefined location, if
   // FoundLoc is illegal.
   // (XXX -- could morph the DBG_INSTR_REF in the future).
-  MachineInstr *DbgMI = MTracker->emitLoc(NewLocs, V, Properties);
+  MachineInstr *DbgMI =
+      MTracker->emitLoc(NewLocs, V, MI.getDebugLoc().get(), Properties);
+  DebugVariableID ID = DVMap.getDVID(V);
 
-  TTracker->PendingDbgValues.push_back(DbgMI);
+  TTracker->PendingDbgValues.push_back(std::make_pair(ID, DbgMI));
   TTracker->flushDbgValues(MI.getIterator(), nullptr);
   return true;
 }
@@ -3112,7 +3125,8 @@ void InstrRefBasedLDV::getBlocksForScope(
 }
 
 void InstrRefBasedLDV::buildVLocValueMap(
-    const DILocation *DILoc, const SmallSet<DebugVariable, 4> &VarsWeCareAbout,
+    const DILocation *DILoc,
+    const SmallSet<DebugVariableID, 4> &VarsWeCareAbout,
     SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks, LiveInsT &Output,
     FuncValueTable &MOutLocs, FuncValueTable &MInLocs,
     SmallVectorImpl<VLocTracker> &AllTheVLocs) {
@@ -3188,7 +3202,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
   // between blocks. This keeps the locality of working on one lexical scope at
   // at time, but avoids re-processing variable values because some other
   // variable has been assigned.
-  for (const auto &Var : VarsWeCareAbout) {
+  for (DebugVariableID VarID : VarsWeCareAbout) {
     // Re-initialize live-ins and live-outs, to clear the remains of previous
     // variables live-ins / live-outs.
     for (unsigned int I = 0; I < NumBlocks; ++I) {
@@ -3202,7 +3216,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
     SmallPtrSet<MachineBasicBlock *, 32> DefBlocks;
     for (const MachineBasicBlock *ExpMBB : BlocksToExplore) {
       auto &TransferFunc = AllTheVLocs[ExpMBB->getNumber()].Vars;
-      if (TransferFunc.contains(Var))
+      if (TransferFunc.contains(VarID))
         DefBlocks.insert(const_cast<MachineBasicBlock *>(ExpMBB));
     }
 
@@ -3212,7 +3226,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
     // only one value definition, things are very simple.
     if (DefBlocks.size() == 1) {
       placePHIsForSingleVarDefinition(MutBlocksToExplore, *DefBlocks.begin(),
-                                      AllTheVLocs, Var, Output);
+                                      AllTheVLocs, VarID, Output);
       continue;
     }
 
@@ -3285,7 +3299,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
 
         // Do transfer function.
         auto &VTracker = AllTheVLocs[MBB->getNumber()];
-        auto TransferIt = VTracker.Vars.find(Var);
+        auto TransferIt = VTracker.Vars.find(VarID);
         if (TransferIt != VTracker.Vars.end()) {
           // Erase on empty transfer (DBG_VALUE $noreg).
           if (TransferIt->second.Kind == DbgValue::Undef) {
@@ -3347,9 +3361,11 @@ void InstrRefBasedLDV::buildVLocValueMap(
         continue;
       if (BlockLiveIn->Kind == DbgValue::VPHI)
         BlockLiveIn->Kind = DbgValue::Def;
+      auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
       assert(BlockLiveIn->Properties.DIExpr->getFragmentInfo() ==
-             Var.getFragment() && "Fragment info missing during value prop");
-      Output[MBB->getNumber()].push_back(std::make_pair(Var, *BlockLiveIn));
+                 Var.getFragment() &&
+             "Fragment info missing during value prop");
+      Output[MBB->getNumber()].push_back(std::make_pair(VarID, *BlockLiveIn));
     }
   } // Per-variable loop.
 
@@ -3360,7 +3376,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
 void InstrRefBasedLDV::placePHIsForSingleVarDefinition(
     const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks,
     MachineBasicBlock *AssignMBB, SmallVectorImpl<VLocTracker> &AllTheVLocs,
-    const DebugVariable &Var, LiveInsT &Output) {
+    DebugVariableID VarID, LiveInsT &Output) {
   // If there is a single definition of the variable, then working out it's
   // value everywhere is very simple: it's every block dominated by the
   // definition. At the dominance frontier, the usual algorithm would:
@@ -3373,7 +3389,7 @@ void InstrRefBasedLDV::placePHIsForSingleVarDefinition(
 
   // Pick out the variables value from the block transfer function.
   VLocTracker &VLocs = AllTheVLocs[AssignMBB->getNumber()];
-  auto ValueIt = VLocs.Vars.find(Var);
+  auto ValueIt = VLocs.Vars.find(VarID);
   const DbgValue &Value = ValueIt->second;
 
   // If it's an explicit assignment of "undef", that means there is no location
@@ -3388,7 +3404,7 @@ void InstrRefBasedLDV::placePHIsForSingleVarDefinition(
     if (!DomTree->properlyDominates(AssignMBB, ScopeBlock))
       continue;
 
-    Output[ScopeBlock->getNumber()].push_back({Var, Value});
+    Output[ScopeBlock->getNumber()].push_back({VarID, Value});
   }
 
   // All blocks that aren't dominated have no live-in value, thus no variable
@@ -3515,9 +3531,9 @@ bool InstrRefBasedLDV::depthFirstVLocAndEmit(
     const ScopeToVarsT &ScopeToVars, ScopeToAssignBlocksT &ScopeToAssignBlocks,
     LiveInsT &Output, FuncValueTable &MOutLocs, FuncValueTable &MInLocs,
     SmallVectorImpl<VLocTracker> &AllTheVLocs, MachineFunction &MF,
-    DenseMap<DebugVariable, unsigned> &AllVarsNumbering,
     const TargetPassConfig &TPC) {
-  TTracker = new TransferTracker(TII, MTracker, MF, *TRI, CalleeSavedRegs, TPC);
+  TTracker =
+      new TransferTracker(TII, MTracker, MF, DVMap, *TRI, CalleeSavedRegs, TPC);
   unsigned NumLocs = MTracker->getNumLocs();
   VTracker = nullptr;
 
@@ -3622,31 +3638,24 @@ bool InstrRefBasedLDV::depthFirstVLocAndEmit(
     if (MInLocs.hasTableFor(*MBB))
       EjectBlock(*MBB);
 
-  return emitTransfers(AllVarsNumbering);
+  return emitTransfers();
 }
 
-bool InstrRefBasedLDV::emitTransfers(
-    DenseMap<DebugVariable, unsigned> &AllVarsNumbering) {
+bool InstrRefBasedLDV::emitTransfers() {
   // Go through all the transfers recorded in the TransferTracker -- this is
   // both the live-ins to a block, and any movements of values that happen
   // in the middle.
-  for (const auto &P : TTracker->Transfers) {
+  for (auto &P : TTracker->Transfers) {
     // We have to insert DBG_VALUEs in a consistent order, otherwise they
     // appear in DWARF in different orders. Use the order that they appear
     // when walking through each block / each instruction, stored in
-    // AllVarsNumbering.
-    SmallVector<std::pair<unsigned, MachineInstr *>> Insts;
-    for (MachineInstr *MI : P.Insts) {
-      DebugVariable Var(MI->getDebugVariable(), MI->getDebugExpression(),
-                        MI->getDebugLoc()->getInlinedAt());
-      Insts.emplace_back(AllVarsNumbering.find(Var)->second, MI);
-    }
-    llvm::sort(Insts, llvm::less_first());
+    // DVMap.
+    llvm::sort(P.Insts, llvm::less_first());
 
     // Insert either before or after the designated point...
     if (P.MBB) {
       MachineBasicBlock &MBB = *P.MBB;
-      for (const auto &Pair : Insts)
+      for (const auto &Pair : P.Insts)
         MBB.insert(P.Pos, Pair.second);
     } else {
       // Terminators, like tail calls, can clobber things. Don't try and place
@@ -3655,7 +3664,7 @@ bool InstrRefBasedLDV::emitTransfers(
         continue;
 
       MachineBasicBlock &MBB = *P.Pos->getParent();
-      for (const auto &Pair : Insts)
+      for (const auto &Pair : P.Insts)
         MBB.insertAfterBundle(P.Pos, Pair.second);
     }
   }
@@ -3710,7 +3719,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
   initialSetup(MF);
 
   MLocTransfer.resize(MaxNumBlocks);
-  vlocs.resize(MaxNumBlocks, VLocTracker(OverlapFragments, EmptyExpr));
+  vlocs.resize(MaxNumBlocks, VLocTracker(DVMap, OverlapFragments, EmptyExpr));
   SavedLiveIns.resize(MaxNumBlocks);
 
   produceMLocTransferFunction(MF, MLocTransfer, MaxNumBlocks);
@@ -3766,10 +3775,6 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
     MTracker->reset();
   }
 
-  // Number all variables in the order that they appear, to be used as a stable
-  // insertion order later.
-  DenseMap<DebugVariable, unsigned> AllVarsNumbering;
-
   // Map from one LexicalScope to all the variables in that scope.
   ScopeToVarsT ScopeToVars;
 
@@ -3788,16 +3793,15 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
     auto *VTracker = &vlocs[MBB->getNumber()];
     // Collect each variable with a DBG_VALUE in this block.
     for (auto &idx : VTracker->Vars) {
-      const auto &Var = idx.first;
-      const DILocation *ScopeLoc = VTracker->Scopes[Var];
+      DebugVariableID VarID = idx.first;
+      const DILocation *ScopeLoc = VTracker->Scopes[VarID];
       assert(ScopeLoc != nullptr);
       auto *Scope = LS.findLexicalScope(ScopeLoc);
 
       // No insts in scope -> shouldn't have been recorded.
       assert(Scope != nullptr);
 
-      AllVarsNumbering.insert(std::make_pair(Var, AllVarsNumbering.size()));
-      ScopeToVars[Scope].insert(Var);
+      ScopeToVars[Scope].insert(VarID);
       ScopeToAssignBlocks[Scope].insert(VTracker->MBB);
       ScopeToDILocation[Scope] = ScopeLoc;
       ++VarAssignCount;
@@ -3821,7 +3825,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
     // the "else" block of this condition.
     Changed = depthFirstVLocAndEmit(
         MaxNumBlocks, ScopeToDILocation, ScopeToVars, ScopeToAssignBlocks,
-        SavedLiveIns, MOutLocs, MInLocs, vlocs, MF, AllVarsNumbering, *TPC);
+        SavedLiveIns, MOutLocs, MInLocs, vlocs, MF, *TPC);
   }
 
   delete MTracker;
@@ -3840,6 +3844,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
   SeenFragments.clear();
   SeenDbgPHIs.clear();
   DbgOpStore.clear();
+  DVMap.clear();
 
   return Changed;
 }
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
index 8770983481c2f..d9851ad13eab2 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
@@ -35,6 +35,44 @@ class DbgOpIDMap;
 
 using namespace llvm;
 
+using DebugVariableID = unsigned;
+using VarAndLoc = std::pair<DebugVariable, const DILocation *>;
+
+/// Mapping from DebugVariable to/from a unique identifying number. Each
+/// DebugVariable consists of three pointers, and after a small amount of
+/// work to identify overlapping fragments of variables we mostly only use
+/// DebugVariables as identities of variables. It's much more compile-time
+/// efficient to use an ID number instead, which this class provides.
+class DebugVariableMap {
+  DenseMap<DebugVariable, unsigned> VarToIdx;
+  SmallVector<VarAndLoc> IdxToVar;
+
+public:
+  DebugVariableID getDVID(const DebugVariable &Var) const {
+    auto It = VarToIdx.find(Var);
+    assert(It != VarToIdx.end());
+    return It->second;
+  }
+
+  DebugVariableID insertDVID(DebugVariable &Var, const DILocation *Loc) {
+    unsigned Size = VarToIdx.size();
+    auto ItPair = VarToIdx.insert({Var, Size});
+    if (ItPair.second) {
+      IdxToVar.push_back({Var, Loc});
+      return Size;
+    }
+
+    return ItPair.first->second;
+  }
+
+  const VarAndLoc &lookupDVID(DebugVariableID ID) const { return IdxToVar[ID]; }
+
+  void clear() {
+    VarToIdx.clear();
+    IdxToVar.clear();
+  }
+};
+
 /// Handle-class for a particular "location". This value-type uniquely
 /// symbolises a register or stack location, allowing manipulation of locations
 /// without concern for where that location is. Practically, this allows us to
@@ -985,7 +1023,7 @@ class MLocTracker {
   /// information in \pProperties, for variable Var. Don't insert it anywhere,
   /// just return the builder for it.
   MachineInstrBuilder emitLoc(const SmallVectorImpl<ResolvedDbgOp> &DbgOps,
-                              const DebugVariable &Var,
+                              const DebugVariable &Var, const DILocation *DILoc,
                               const DbgValueProperties &Properties);
 };
 
@@ -1003,38 +1041,45 @@ using OverlapMap =
 /// identified.
 class VLocTracker {
 public:
+  /// Ref to function-wide map of DebugVariable <=> ID-numbers.
+  DebugVariableMap &DVMap;
   /// Map DebugVariable to the latest Value it's defined to have.
   /// Needs to be a MapVector because we determine order-in-the-input-MIR from
-  /// the order in this container.
+  /// the order in this container. (FIXME: likely no longer true as the ordering
+  /// is now provided by DebugVariableMap).
   /// We only retain the last DbgValue in each block for each variable, to
   /// determine the blocks live-out variable value. The Vars container forms the
   /// transfer function for this block, as part of the dataflow analysis. The
   /// movement of values between locations inside of a block is handled at a
   /// much later stage, in the TransferTracker class.
-  MapVector<DebugVariable, DbgValue> Vars;
-  SmallDenseMap<DebugVariable, const DILocation *, 8> Scopes;
+  MapVector<DebugVariableID, DbgValue> Vars;
+  SmallDenseMap<DebugVariableID, const DILocation *, 8> Scopes;
   MachineBasicBlock *MBB = nullptr;
   const OverlapMap &OverlappingFragments;
   DbgValueProperties EmptyProperties;
 
 public:
-  VLocTracker(const OverlapMap &O, const DIExpression *EmptyExpr)
-      : OverlappingFragments(O), EmptyProperties(EmptyExpr, false, false) {}
+  VLocTracker(DebugVariableMap &DVMap, const OverlapMap &O,
+              const DIExpression *EmptyExpr)
+      : DVMap(DVMap), OverlappingFragments(O),
+        EmptyProperties(EmptyExpr, false, false) {}
 
   void defVar(const MachineInstr &MI, const DbgValueProperties &Properties,
               const SmallVectorImpl<DbgOpID> &DebugOps) {
     assert(MI.isDebugValueLike());
     DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
                       MI.getDebugLoc()->getInlinedAt());
+    // Either insert or fetch an ID number for this variable.
+    DebugVariableID VarID = DVMap.insertDVID(Var, MI.getDebugLoc().get());
     DbgValue Rec = (DebugOps.size() > 0)
                        ? DbgValue(DebugOps, Properties)
                        : DbgValue(Properties, DbgValue::Undef);
 
     // Attempt insertion; overwrite if it's already mapped.
-    auto Result = Vars.insert(std::make_pair(Var, Rec));
+    auto Result = Vars.insert(std::make_pair(VarID, Rec));
     if (!Result.second)
       Result.first->second = Rec;
-    Scopes[Var] = MI.getDebugLoc().get();
+    Scopes[VarID] = MI.getDebugLoc().get();
 
     considerOverlaps(Var, MI.getDebugLoc().get());
   }
@@ -1056,13 +1101,15 @@ class VLocTracker {
 
       DebugVariable Overlapped(Var.getVariable(), OptFragmentInfo,
                                Var.getInlinedAt());
+      // Produce an ID number for this overlapping fragment of a variable.
+      DebugVariableID OverlappedID = DVMap.insertDVID(Overlapped, Loc);
       DbgValue Rec = DbgValue(EmptyProperties, DbgValue::Undef);
 
       // Attempt insertion; overwrite if it's already mapped.
-      auto Result = Vars.insert(std::make_pair(Overlapped, Rec));
+      auto Result = Vars.insert(std::make_pair(OverlappedID, Rec));
       if (!Result.second)
         Result.first->second = Rec;
-      Scopes[Overlapped] = Loc;
+      Scopes[OverlappedID] = Loc;
     }
   }
 
@@ -1093,7 +1140,7 @@ class InstrRefBasedLDV : public LDVImpl {
   /// variables to their values.
   using LiveIdxT = DenseMap<const MachineBasicBlock *, DbgValue *>;
 
-  using VarAndLoc = std::pair<DebugVariable, DbgValue>;
+  using VarAndLoc = std::pair<DebugVariableID, DbgValue>;
 
   /// Type for a live-in value: the predecessor block, and its value.
   using InValueT = std::pair<MachineBasicBlock *, DbgValue *>;
@@ -1106,7 +1153,8 @@ class InstrRefBasedLDV : public LDVImpl {
   using ScopeToDILocT = DenseMap<const LexicalScope *, const DILocation *>;
 
   /// Mapping from lexical scopes to variables in that scope.
-  using ScopeToVarsT = DenseMap<const LexicalScope *, SmallSet<DebugVariable, 4>>;
+  using ScopeToVarsT =
+      DenseMap<const LexicalScope *, SmallSet<DebugVariableID, 4>>;
 
   /// Mapping from lexical scopes to blocks where variables in that scope are
   /// assigned. Such blocks aren't necessarily "in" the lexical scope, it's
@@ -1200,6 +1248,11 @@ class InstrRefBasedLDV : public LDVImpl {
 
   DbgOpIDMap DbgOpStore;
 
+  /// Mapping between DebugVariables and unique ID numbers. This is a more
+  /// efficient way to represent the identity of a variable, versus a plain
+  /// DebugVariable.
+  DebugVariableMap DVMap;
+
   /// True if we need to examine call instructions for stack clobbers. We
   /// normally assume that they don't clobber SP, but stack probes on Windows
   /// do.
@@ -1330,9 +1383,9 @@ class InstrRefBasedLDV : public LDVImpl {
   /// performance as it doesn't have to find the dominance frontier between
   /// different assignments.
   void placePHIsForSingleVarDefinition(
-          const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks,
-          MachineBasicBlock *MBB, SmallVectorImpl<VLocTracker> &AllTheVLocs,
-          const DebugVariable &Var, LiveInsT &Output);
+      const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks,
+      MachineBasicBlock *MBB, SmallVectorImpl<VLocTracker> &AllTheVLocs,
+      DebugVariableID Var, LiveInsT &Output);
 
   /// Calculate the iterated-dominance-frontier for a set of defs, using the
   /// existing LLVM facilities for this. Works for a single "value" or
@@ -1381,7 +1434,7 @@ class InstrRefBasedLDV : public LDVImpl {
   /// scope, but which do contain DBG_VALUEs, which VarLocBasedImpl tracks
   /// locations through.
   void buildVLocValueMap(const DILocation *DILoc,
-                         const SmallSet<DebugVariable, 4> &VarsWeCareAbout,
+                         const SmallSet<DebugVariableID, 4> &VarsWeCareAbout,
                          SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks,
                          LiveInsT &Output, FuncValueTable &MOutLocs,
                          FuncValueTable &MInLocs,
@@ -1414,10 +1467,8 @@ class InstrRefBasedLDV : public LDVImpl {
       const SmallVectorImpl<const MachineBasicBlock *> &BlockOrders);
 
   /// Take collections of DBG_VALUE instructions stored in TTracker, and
-  /// install them into their output blocks. Preserves a stable order of
-  /// DBG_VALUEs produced (which would otherwise cause nondeterminism) through
-  /// the AllVarsNumbering order.
-  bool emitTransfers(DenseMap<DebugVariable, unsigned> &AllVarsNumbering);
+  /// install them into their output blocks.
+  bool emitTransfers();
 
   /// Boilerplate computation of some initial sets, artifical blocks and
   /// RPOT block ordering.
@@ -1437,13 +1488,14 @@ class InstrRefBasedLDV : public LDVImpl {
   /// block information can be fully computed before exploration finishes,
   /// allowing us to emit it and free data structures earlier than otherwise.
   /// It's also good for locality.
-  bool depthFirstVLocAndEmit(
-      unsigned MaxNumBlocks, const ScopeToDILocT &ScopeToDILocation,
-      const ScopeToVarsT &ScopeToVars, ScopeToAssignBlocksT &ScopeToBlocks,
-      LiveInsT &Output, FuncValueTable &MOutLocs, FuncValueTable &MInLocs,
-      SmallVectorImpl<VLocTracker> &AllTheVLocs, MachineFunction &MF,
-      DenseMap<DebugVariable, unsigned> &AllVarsNumbering,
-      const TargetPassConfig &TPC);
+  bool depthFirstVLocAndEmit(unsigned MaxNumBlocks,
+                             const ScopeToDILocT &ScopeToDILocation,
+                             const ScopeToVarsT &ScopeToVars,
+                             ScopeToAssignBlocksT &ScopeToBlocks,
+                             LiveInsT &Output, FuncValueTable &MOutLocs,
+                             FuncValueTable &MInLocs,
+                             SmallVectorImpl<VLocTracker> &AllTheVLocs,
+                             MachineFunction &MF, const TargetPassConfig &TPC);
 
   bool ExtendRanges(MachineFunction &MF, MachineDominatorTree *DomTree,
                     TargetPassConfig *TPC, unsigned InputBBLimit,
@@ -1473,6 +1525,11 @@ class InstrRefBasedLDV : public LDVImpl {
   }
 
   std::optional<LocIdx> findLocationForMemOperand(const MachineInstr &MI);
+
+  // Utility for unit testing, don't use directly.
+  DebugVariableMap &getDVMap() {
+    return DVMap;
+  }
 };
 
 } // namespace LiveDebugValues
diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir
index b54c748ac9e84..67bfd85dcb379 100644
--- a/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir
+++ b/llvm/test/DebugInfo/MIR/X86/live-debug-values-fragments.mir
@@ -17,12 +17,12 @@
 # CHECK-LABEL: bb.3.bb3:
 # CHECK:      DBG_VALUE $ecx, $noreg, !{{[0-9]+}},
 # CHECK-SAME:                 !DIExpression(DW_OP_LLVM_fragment, 0, 32)
-# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}}, 
+# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}},
 # CHECK-SAME:                 !DIExpression({{[^)]+}}, DW_OP_LLVM_fragment, 0, 32)
 # CHECK-SAME:                 $ecx, $r8d
 # CHECK-NEXT: DBG_VALUE $ebx, $noreg, !{{[0-9]+}},
 # CHECK-SAME:                 !DIExpression(DW_OP_LLVM_fragment, 32, 32)
-# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}}, 
+# CHECK-NEXT: DBG_VALUE_LIST !{{[0-9]+}},
 # CHECK-SAME:                 !DIExpression({{[^)]+}}, DW_OP_LLVM_fragment, 32, 32)
 # CHECK-SAME:                 $ebx, $r10d
 # CHECK-NEXT: XOR32rr
diff --git a/llvm/unittests/CodeGen/InstrRefLDVTest.cpp b/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
index 306a97c3149cc..28cfb3046bd47 100644
--- a/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
+++ b/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
@@ -55,6 +55,7 @@ class InstrRefLDVTest : public testing::Test {
   DIBasicType *LongInt;
   DIExpression *EmptyExpr;
   LiveDebugValues::OverlapMap Overlaps;
+  LiveDebugValues::DebugVariableMap DVMap;
 
   DebugLoc OutermostLoc, InBlockLoc, NotNestedBlockLoc, InlinedLoc;
 
@@ -176,7 +177,7 @@ class InstrRefLDVTest : public testing::Test {
 
   void addVTracker() {
     ASSERT_TRUE(LDV);
-    VTracker = std::make_unique<VLocTracker>(Overlaps, EmptyExpr);
+    VTracker = std::make_unique<VLocTracker>(DVMap, Overlaps, EmptyExpr);
     LDV->VTracker = &*VTracker;
   }
 
@@ -215,7 +216,7 @@ class InstrRefLDVTest : public testing::Test {
   }
 
   void buildVLocValueMap(const DILocation *DILoc,
-                    const SmallSet<DebugVariable, 4> &VarsWeCareAbout,
+                    const SmallSet<DebugVariableID, 4> &VarsWeCareAbout,
                     SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks,
                     InstrRefBasedLDV::LiveInsT &Output, FuncValueTable &MOutLocs,
                     FuncValueTable &MInLocs,
@@ -2632,10 +2633,11 @@ TEST_F(InstrRefLDVTest, VLocSingleBlock) {
   MInLocs[0][0] = MOutLocs[0][0] = LiveInRsp;
 
   DebugVariable Var(FuncVariable, std::nullopt, nullptr);
+  DebugVariableID VarID = LDV->getDVMap().insertDVID(Var, OutermostLoc);
   DbgValueProperties EmptyProps(EmptyExpr, false, false);
 
-  SmallSet<DebugVariable, 4> AllVars;
-  AllVars.insert(Var);
+  SmallSet<DebugVariableID, 4> AllVars;
+  AllVars.insert(VarID);
 
   // Mild hack: rather than constructing machine instructions in each block
   // and creating lexical scopes across them, instead just tell
@@ -2645,7 +2647,7 @@ TEST_F(InstrRefLDVTest, VLocSingleBlock) {
   AssignBlocks.insert(MBB0);
 
   SmallVector<VLocTracker, 1> VLocs;
-  VLocs.resize(1, VLocTracker(Overlaps, EmptyExpr));
+  VLocs.resize(1, VLocTracker(LDV->getDVMap(), Overlaps, EmptyExpr));
 
   InstrRefBasedLDV::LiveInsT Output;
 
@@ -2657,7 +2659,7 @@ TEST_F(InstrRefLDVTest, VLocSingleBlock) {
 
   // If we put an assignment in the transfer function, that should... well,
   // do nothing, because we don't store the live-outs.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output.size(), 0ul);
@@ -2694,10 +2696,11 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
   initValueArray(MOutLocs, 4, 2);
 
   DebugVariable Var(FuncVariable, std::nullopt, nullptr);
+  DebugVariableID VarID = LDV->getDVMap().insertDVID(Var, OutermostLoc);
   DbgValueProperties EmptyProps(EmptyExpr, false, false);
 
-  SmallSet<DebugVariable, 4> AllVars;
-  AllVars.insert(Var);
+  SmallSet<DebugVariableID, 4> AllVars;
+  AllVars.insert(VarID);
 
   // Mild hack: rather than constructing machine instructions in each block
   // and creating lexical scopes across them, instead just tell
@@ -2710,7 +2713,7 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
   AssignBlocks.insert(MBB3);
 
   SmallVector<VLocTracker, 1> VLocs;
-  VLocs.resize(4, VLocTracker(Overlaps, EmptyExpr));
+  VLocs.resize(4, VLocTracker(LDV->getDVMap(), Overlaps, EmptyExpr));
 
   InstrRefBasedLDV::LiveInsT Output;
 
@@ -2736,7 +2739,7 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
 
   // An assignment in the end block should also not affect other blocks; or
   // produce any live-ins.
-  VLocs[3].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[3].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2748,7 +2751,7 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
   // Assignments in either of the side-of-diamond blocks should also not be
   // propagated anywhere.
   VLocs[3].Vars.clear();
-  VLocs[2].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[2].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2758,7 +2761,7 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
   VLocs[2].Vars.clear();
   ClearOutputs();
 
-  VLocs[1].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2770,7 +2773,7 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
 
   // However: putting an assignment in the first block should propagate variable
   // values through to all other blocks, as it dominates.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2790,7 +2793,7 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
   // should still be propagated, as buildVLocValueMap shouldn't care about
   // what's in the registers (except for PHIs).
   // values through to all other blocks, as it dominates.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRaxID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRaxID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2808,8 +2811,8 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
 
   // We should get a live-in to the merging block, if there are two assigns of
   // the same value in either side of the diamond.
-  VLocs[1].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[2].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[2].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2824,8 +2827,8 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
 
   // If we assign a value in the entry block, then 'undef' on a branch, we
   // shouldn't have a live-in in the merge block.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(EmptyProps, DbgValue::Undef)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(EmptyProps, DbgValue::Undef)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2843,8 +2846,8 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
   // Having different values joining into the merge block should mean we have
   // no live-in in that block. Block ones LiveInRax value doesn't appear as a
   // live-in anywhere, it's block internal.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(LiveInRaxID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(LiveInRaxID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2862,8 +2865,8 @@ TEST_F(InstrRefLDVTest, VLocDiamondBlocks) {
   // But on the other hand, if there's a location in the register file where
   // those two values can be joined, do so.
   MOutLocs[1][0] = LiveInRax;
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(LiveInRaxID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(LiveInRaxID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2915,14 +2918,15 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   initValueArray(MOutLocs, 3, 2);
 
   DebugVariable Var(FuncVariable, std::nullopt, nullptr);
+  DebugVariableID VarID = LDV->getDVMap().insertDVID(Var, OutermostLoc);
   DbgValueProperties EmptyProps(EmptyExpr, false, false);
   DIExpression *TwoOpExpr =
       DIExpression::get(Ctx, {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_LLVM_arg,
                               1, dwarf::DW_OP_plus});
   DbgValueProperties VariadicProps(TwoOpExpr, false, true);
 
-  SmallSet<DebugVariable, 4> AllVars;
-  AllVars.insert(Var);
+  SmallSet<DebugVariableID, 4> AllVars;
+  AllVars.insert(VarID);
 
   SmallPtrSet<MachineBasicBlock *, 4> AssignBlocks;
   AssignBlocks.insert(MBB0);
@@ -2930,7 +2934,7 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   AssignBlocks.insert(MBB2);
 
   SmallVector<VLocTracker, 3> VLocs;
-  VLocs.resize(3, VLocTracker(Overlaps, EmptyExpr));
+  VLocs.resize(3, VLocTracker(LDV->getDVMap(), Overlaps, EmptyExpr));
 
   InstrRefBasedLDV::LiveInsT Output;
 
@@ -2947,7 +2951,7 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   Output.resize(3);
 
   // Easy starter: a dominating assign should propagate to all blocks.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2963,7 +2967,7 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
 
   // A variadic assignment should behave the same.
   DbgOpID Locs0[] = {LiveInRspID, LiveInRaxID};
-  VLocs[0].Vars.insert({Var, DbgValue(Locs0, VariadicProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(Locs0, VariadicProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output, MOutLocs,
                     MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2979,8 +2983,8 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   VLocs[1].Vars.clear();
 
   // Put an undef assignment in the loop. Should get no live-in value.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(EmptyProps, DbgValue::Undef)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(EmptyProps, DbgValue::Undef)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -2991,8 +2995,8 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   VLocs[1].Vars.clear();
 
   // Assignment of the same value should naturally join.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3008,8 +3012,8 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
 
   // Assignment of different values shouldn't join with no machine PHI vals.
   // Will be live-in to exit block as it's dominated.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(LiveInRaxID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(LiveInRaxID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3025,8 +3029,8 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   // with unrelated assign in loop block again.
   MInLocs[1][0] = RspPHIInBlk1;
   MOutLocs[1][0] = RspDefInBlk1;
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(LiveInRaxID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(LiveInRaxID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3042,8 +3046,8 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   // find the appropriate PHI.
   MInLocs[1][0] = RspPHIInBlk1;
   MOutLocs[1][0] = RspDefInBlk1;
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(RspDefInBlk1ID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(RspDefInBlk1ID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3063,8 +3067,8 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   MOutLocs[1][0] = LiveInRsp;
   MInLocs[1][1] = RaxPHIInBlk1;
   MOutLocs[1][1] = RspDefInBlk1;
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(RspDefInBlk1ID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(RspDefInBlk1ID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3085,8 +3089,8 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   MOutLocs[1][0] = RspDefInBlk1;
   MInLocs[1][1] = RaxPHIInBlk1;
   MOutLocs[1][1] = RspDefInBlk1;
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(RspDefInBlk1ID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(RspDefInBlk1ID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3119,8 +3123,8 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   MOutLocs[1][0] = RspPHIInBlk1;
   MInLocs[1][1] = LiveInRax;
   MOutLocs[1][1] = LiveInRax;
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(RspPHIInBlk1ID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(RspPHIInBlk1ID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3138,8 +3142,8 @@ TEST_F(InstrRefLDVTest, VLocSimpleLoop) {
   // because there's a def in it.
   MInLocs[1][0] = LiveInRsp;
   MOutLocs[1][0] = LiveInRsp;
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3193,10 +3197,11 @@ TEST_F(InstrRefLDVTest, VLocNestedLoop) {
   initValueArray(MOutLocs, 5, 2);
 
   DebugVariable Var(FuncVariable, std::nullopt, nullptr);
+  DebugVariableID VarID = LDV->getDVMap().insertDVID(Var, OutermostLoc);
   DbgValueProperties EmptyProps(EmptyExpr, false, false);
 
-  SmallSet<DebugVariable, 4> AllVars;
-  AllVars.insert(Var);
+  SmallSet<DebugVariableID, 4> AllVars;
+  AllVars.insert(VarID);
 
   SmallPtrSet<MachineBasicBlock *, 5> AssignBlocks;
   AssignBlocks.insert(MBB0);
@@ -3206,7 +3211,7 @@ TEST_F(InstrRefLDVTest, VLocNestedLoop) {
   AssignBlocks.insert(MBB4);
 
   SmallVector<VLocTracker, 5> VLocs;
-  VLocs.resize(5, VLocTracker(Overlaps, EmptyExpr));
+  VLocs.resize(5, VLocTracker(LDV->getDVMap(), Overlaps, EmptyExpr));
 
   InstrRefBasedLDV::LiveInsT Output;
 
@@ -3223,7 +3228,7 @@ TEST_F(InstrRefLDVTest, VLocNestedLoop) {
   Output.resize(5);
 
   // A dominating assign should propagate to all blocks.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3244,8 +3249,8 @@ TEST_F(InstrRefLDVTest, VLocNestedLoop) {
 
   // Test that an assign in the inner loop causes unresolved PHIs at the heads
   // of both loops, and no output location. Dominated blocks do get values.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[2].Vars.insert({Var, DbgValue(LiveInRaxID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[2].Vars.insert({VarID, DbgValue(LiveInRaxID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3263,7 +3268,7 @@ TEST_F(InstrRefLDVTest, VLocNestedLoop) {
 
   // Same test, but with no assignment in block 0. We should still get values
   // in dominated blocks.
-  VLocs[2].Vars.insert({Var, DbgValue(LiveInRaxID, EmptyProps)});
+  VLocs[2].Vars.insert({VarID, DbgValue(LiveInRaxID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3280,8 +3285,8 @@ TEST_F(InstrRefLDVTest, VLocNestedLoop) {
 
   // Similarly, assignments in the outer loop gives location to dominated
   // blocks, but no PHI locations are found at the outer loop head.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[3].Vars.insert({Var, DbgValue(LiveInRaxID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[3].Vars.insert({VarID, DbgValue(LiveInRaxID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3295,8 +3300,8 @@ TEST_F(InstrRefLDVTest, VLocNestedLoop) {
   VLocs[0].Vars.clear();
   VLocs[3].Vars.clear();
 
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[1].Vars.insert({Var, DbgValue(LiveInRaxID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[1].Vars.insert({VarID, DbgValue(LiveInRaxID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3317,8 +3322,8 @@ TEST_F(InstrRefLDVTest, VLocNestedLoop) {
   // With an assignment of the same value in the inner loop, we should work out
   // that all PHIs can be eliminated and the same value is live-through the
   // whole function.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[2].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[2].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3348,8 +3353,8 @@ TEST_F(InstrRefLDVTest, VLocNestedLoop) {
   // one. Even though RspPHIInBlk2 isn't available later in the function, we
   // should still produce a live-in value. The fact it's unavailable is a
   // different concern.
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[2].Vars.insert({Var, DbgValue(LiveInRaxID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[2].Vars.insert({VarID, DbgValue(LiveInRaxID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);
@@ -3374,8 +3379,8 @@ TEST_F(InstrRefLDVTest, VLocNestedLoop) {
   MOutLocs[2][0] = RspDefInBlk2;
   MInLocs[3][0] = RspDefInBlk2;
   MOutLocs[3][0] = RspDefInBlk2;
-  VLocs[0].Vars.insert({Var, DbgValue(LiveInRspID, EmptyProps)});
-  VLocs[2].Vars.insert({Var, DbgValue(RspDefInBlk2ID, EmptyProps)});
+  VLocs[0].Vars.insert({VarID, DbgValue(LiveInRspID, EmptyProps)});
+  VLocs[2].Vars.insert({VarID, DbgValue(RspDefInBlk2ID, EmptyProps)});
   buildVLocValueMap(OutermostLoc, AllVars, AssignBlocks, Output,
                     MOutLocs, MInLocs, VLocs);
   EXPECT_EQ(Output[0].size(), 0ul);

From 497ea1d84951626dea5bf644fef2d99e145e21ac Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 18 Jul 2024 16:06:43 +0100
Subject: [PATCH 034/486] [DAG] tryToFoldExtendSelectLoad - reuse existing
 SDLoc. NFC.

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 30203f9119af7..9b2153c68ccae 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -12812,13 +12812,11 @@ static bool isCompatibleLoad(SDValue N, unsigned ExtOpcode) {
 /// This function is called by the DAGCombiner when visiting sext/zext/aext
 /// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
 static SDValue tryToFoldExtendSelectLoad(SDNode *N, const TargetLowering &TLI,
-                                         SelectionDAG &DAG,
+                                         SelectionDAG &DAG, const SDLoc &DL,
                                          CombineLevel Level) {
   unsigned Opcode = N->getOpcode();
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
-  SDLoc DL(N);
-
   assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND ||
           Opcode == ISD::ANY_EXTEND) &&
          "Expected EXTEND dag node in input!");
@@ -13775,7 +13773,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
     return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT));
   }
 
-  if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, Level))
+  if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
     return Res;
 
   return SDValue();
@@ -13860,8 +13858,7 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
   // fold (zext (zext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x)
   if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
       N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG)
-    return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(N), VT,
-                       N0.getOperand(0));
+    return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, N0.getOperand(0));
 
   // fold (zext (truncate x)) -> (zext x) or
   //      (zext (truncate x)) -> (truncate x)
@@ -14147,7 +14144,7 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
   if (SDValue V = widenAbs(N, DAG))
     return V;
 
-  if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, Level))
+  if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
     return Res;
 
   // CSE zext nneg with sext if the zext is not free.
@@ -14322,7 +14319,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
   if (SDValue NewCtPop = widenCtPop(N, DAG))
     return NewCtPop;
 
-  if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, Level))
+  if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
     return Res;
 
   return SDValue();

From 2bdcfbe62cb9a08df4b58a17d44be0a3082df053 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Thu, 18 Jul 2024 19:29:51 +0400
Subject: [PATCH 035/486] [clang] Fix crash in concept deprecation (#98622)

There is a gap between `getAs<AutoType>()` and
`getConstrainedAutoType()` that the original patch #92295 was not aware
of.

Fixes #98164
---
 clang/lib/Sema/SemaDecl.cpp           | 8 ++++----
 clang/lib/Sema/SemaType.cpp           | 9 ++++-----
 clang/test/CXX/drs/cwg24xx.cpp        | 9 +++++++++
 clang/test/SemaCXX/cxx-deprecated.cpp | 9 +++++++++
 4 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 6c3589bf87433..bb25a0b3a45ae 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -7436,10 +7436,10 @@ NamedDecl *Sema::ActOnVariableDeclarator(
     tryToFixVariablyModifiedVarType(TInfo, R, D.getIdentifierLoc(),
                                     /*DiagID=*/0);
 
-  if (const AutoType *AutoT = R->getAs<AutoType>())
-    CheckConstrainedAuto(
-        AutoT,
-        TInfo->getTypeLoc().getContainedAutoTypeLoc().getConceptNameLoc());
+  if (AutoTypeLoc TL = TInfo->getTypeLoc().getContainedAutoTypeLoc()) {
+    const AutoType *AT = TL.getTypePtr();
+    CheckConstrainedAuto(AT, TL.getConceptNameLoc());
+  }
 
   bool IsMemberSpecialization = false;
   bool IsVariableTemplateSpecialization = false;
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index baac1fe4f2407..6fa39cdccef2b 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -6363,11 +6363,10 @@ TypeResult Sema::ActOnTypeName(Declarator &D) {
     CheckExtraCXXDefaultArguments(D);
   }
 
-  if (const AutoType *AutoT = T->getAs<AutoType>())
-    CheckConstrainedAuto(
-        AutoT,
-        TInfo->getTypeLoc().getContainedAutoTypeLoc().getConceptNameLoc());
-
+  if (AutoTypeLoc TL = TInfo->getTypeLoc().getContainedAutoTypeLoc()) {
+    const AutoType *AT = TL.getTypePtr();
+    CheckConstrainedAuto(AT, TL.getConceptNameLoc());
+  }
   return CreateParsedType(T, TInfo);
 }
 
diff --git a/clang/test/CXX/drs/cwg24xx.cpp b/clang/test/CXX/drs/cwg24xx.cpp
index 16b8ec07fc50f..00b6bb5a865df 100644
--- a/clang/test/CXX/drs/cwg24xx.cpp
+++ b/clang/test/CXX/drs/cwg24xx.cpp
@@ -82,6 +82,15 @@ auto h() -> C auto {
   C auto foo = T();
   // expected-warning@-1 {{'C' is deprecated}}
   //   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
+  C auto *bar = T();
+  // expected-warning@-1 {{'C' is deprecated}}
+  //   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
+  C auto &baz = T();
+  // expected-warning@-1 {{'C' is deprecated}}
+  //   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
+  C auto &&quux = T();
+  // expected-warning@-1 {{'C' is deprecated}}
+  //   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
   return foo;
 }
 #endif
diff --git a/clang/test/SemaCXX/cxx-deprecated.cpp b/clang/test/SemaCXX/cxx-deprecated.cpp
index 81eb07608300d..d7de609d58cdd 100644
--- a/clang/test/SemaCXX/cxx-deprecated.cpp
+++ b/clang/test/SemaCXX/cxx-deprecated.cpp
@@ -36,4 +36,13 @@ template <C T>
 // expected-warning@-1 {{'C' is deprecated}}
 //   expected-note@#C {{'C' has been explicitly marked deprecated here}}
 void f();
+
+namespace GH98164 {
+template <int>
+auto b() = delete; // #b
+
+decltype(b<0>()) x;
+// expected-error@-1 {{call to deleted function 'b'}}
+//   expected-note@#b {{candidate function [with $0 = 0] has been explicitly deleted}}
+} // namespace GH98164
 } // namespace cxx20_concept

From a2d309912a2863dfe7286ffde67b968e8c720b07 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas@arm.com>
Date: Thu, 18 Jul 2024 16:34:16 +0100
Subject: [PATCH 036/486] [FMV][AArch64] Do not emit ifunc resolver on use.
 (#97761)

It was raised in https://github.com/llvm/llvm-project/issues/81494 that
we are not generating correct code when there is no TU-local caller.

The suggestion was to emit a resolver:
* Whenever there is a use in the TU.
* When the TU has a definition of the default version.

See the comment for more details:

https://github.com/llvm/llvm-project/issues/81494#issuecomment-1985963497

This got addressed with https://github.com/llvm/llvm-project/pull/84405.

Generating a resolver on use means that we may end up with multiple
resolvers across different translation units. Those resolvers may not be
the same because each translation unit may contain different version
declarations (user's fault). Therefore the order of linking the final
image determines which of these weak symbols gets selected, resulting in
non consisted behavior. I am proposing to stop emitting a resolver on
use and only do so in the translation unit which contains the default
definition. This way we guarantee the existence of a single resolver.
Now, when a versioned function is used we want to emit a declaration of
the function symbol omitting the multiversion mangling.

I have added a requirement to ACLE mandating that all the function
versions are declared in the translation unit which contains the default
definition: https://github.com/ARM-software/acle/pull/328
---
 clang/lib/CodeGen/CodeGenModule.cpp           |  97 ++-
 .../CodeGen/aarch64-fmv-resolver-emission.c   | 111 +++
 .../CodeGen/aarch64-mixed-target-attributes.c |   6 +-
 .../test/CodeGen/attr-target-clones-aarch64.c | 348 ++++----
 clang/test/CodeGen/attr-target-version.c      | 818 +++++++++---------
 .../aarch64-fmv-resolver-emission.cpp         | 111 +++
 .../CodeGenCXX/attr-target-clones-aarch64.cpp | 210 ++---
 clang/test/CodeGenCXX/attr-target-version.cpp | 247 +++---
 clang/test/CodeGenCXX/fmv-namespace.cpp       |  49 +-
 9 files changed, 1117 insertions(+), 880 deletions(-)
 create mode 100644 clang/test/CodeGen/aarch64-fmv-resolver-emission.c
 create mode 100644 clang/test/CodeGenCXX/aarch64-fmv-resolver-emission.cpp

diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 0c002b553e4c6..71192cb0e8c4a 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -3796,8 +3796,7 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
     // Forward declarations are emitted lazily on first use.
     if (!FD->doesThisDeclarationHaveABody()) {
       if (!FD->doesDeclarationForceExternallyVisibleDefinition() &&
-          (!FD->isMultiVersion() ||
-           !FD->getASTContext().getTargetInfo().getTriple().isAArch64()))
+          (!FD->isMultiVersion() || !getTarget().getTriple().isAArch64()))
         return;
 
       StringRef MangledName = getMangledName(GD);
@@ -4191,23 +4190,6 @@ llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM,
   return llvm::GlobalValue::WeakODRLinkage;
 }
 
-static FunctionDecl *createDefaultTargetVersionFrom(const FunctionDecl *FD) {
-  auto *DeclCtx = const_cast<DeclContext *>(FD->getDeclContext());
-  TypeSourceInfo *TInfo = FD->getTypeSourceInfo();
-  StorageClass SC = FD->getStorageClass();
-  DeclarationName Name = FD->getNameInfo().getName();
-
-  FunctionDecl *NewDecl =
-      FunctionDecl::Create(FD->getASTContext(), DeclCtx, FD->getBeginLoc(),
-                           FD->getEndLoc(), Name, TInfo->getType(), TInfo, SC);
-
-  NewDecl->setIsMultiVersion();
-  NewDecl->addAttr(TargetVersionAttr::CreateImplicit(
-      NewDecl->getASTContext(), "default", NewDecl->getSourceRange()));
-
-  return NewDecl;
-}
-
 void CodeGenModule::emitMultiVersionFunctions() {
   std::vector<GlobalDecl> MVFuncsToEmit;
   MultiVersionFuncs.swap(MVFuncsToEmit);
@@ -4234,29 +4216,30 @@ void CodeGenModule::emitMultiVersionFunctions() {
       return cast<llvm::Function>(Func);
     };
 
-    bool HasDefaultDecl = !FD->isTargetVersionMultiVersion();
-    bool ShouldEmitResolver =
-        !getContext().getTargetInfo().getTriple().isAArch64();
+    // For AArch64, a resolver is only emitted if a function marked with
+    // target_version("default")) or target_clones() is present and defined
+    // in this TU. For other architectures it is always emitted.
+    bool ShouldEmitResolver = !getTarget().getTriple().isAArch64();
     SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
 
     getContext().forEachMultiversionedFunctionVersion(
         FD, [&](const FunctionDecl *CurFD) {
           llvm::SmallVector<StringRef, 8> Feats;
+          bool IsDefined = CurFD->doesThisDeclarationHaveABody();
 
           if (const auto *TA = CurFD->getAttr<TargetAttr>()) {
             TA->getAddedFeatures(Feats);
             llvm::Function *Func = createFunction(CurFD);
             Options.emplace_back(Func, TA->getArchitecture(), Feats);
           } else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) {
-            bool HasDefaultDef = TVA->isDefaultVersion() &&
-                                 CurFD->doesThisDeclarationHaveABody();
-            HasDefaultDecl |= TVA->isDefaultVersion();
-            ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef);
+            if (TVA->isDefaultVersion() && IsDefined)
+              ShouldEmitResolver = true;
             TVA->getFeatures(Feats);
             llvm::Function *Func = createFunction(CurFD);
             Options.emplace_back(Func, /*Architecture*/ "", Feats);
           } else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) {
-            ShouldEmitResolver |= CurFD->doesThisDeclarationHaveABody();
+            if (IsDefined)
+              ShouldEmitResolver = true;
             for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) {
               if (!TC->isFirstOfVersion(I))
                 continue;
@@ -4282,13 +4265,6 @@ void CodeGenModule::emitMultiVersionFunctions() {
     if (!ShouldEmitResolver)
       continue;
 
-    if (!HasDefaultDecl) {
-      FunctionDecl *NewFD = createDefaultTargetVersionFrom(FD);
-      llvm::Function *Func = createFunction(NewFD);
-      llvm::SmallVector<StringRef, 1> Feats;
-      Options.emplace_back(Func, /*Architecture*/ "", Feats);
-    }
-
     llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD);
     if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(ResolverConstant)) {
       ResolverConstant = IFunc->getResolver();
@@ -4339,6 +4315,14 @@ void CodeGenModule::emitMultiVersionFunctions() {
     emitMultiVersionFunctions();
 }
 
+static void replaceDeclarationWith(llvm::GlobalValue *Old,
+                                   llvm::Constant *New) {
+  assert(cast<llvm::Function>(Old)->isDeclaration() && "Not a declaration");
+  New->takeName(Old);
+  Old->replaceAllUsesWith(New);
+  Old->eraseFromParent();
+}
+
 void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
   const auto *FD = cast<FunctionDecl>(GD.getDecl());
   assert(FD && "Not a FunctionDecl?");
@@ -4443,12 +4427,9 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
     // Fix up function declarations that were created for cpu_specific before
     // cpu_dispatch was known
     if (!isa<llvm::GlobalIFunc>(IFunc)) {
-      assert(cast<llvm::Function>(IFunc)->isDeclaration());
       auto *GI = llvm::GlobalIFunc::create(DeclTy, 0, Linkage, "", ResolverFunc,
                                            &getModule());
-      GI->takeName(IFunc);
-      IFunc->replaceAllUsesWith(GI);
-      IFunc->eraseFromParent();
+      replaceDeclarationWith(IFunc, GI);
       IFunc = GI;
     }
 
@@ -4478,7 +4459,8 @@ void CodeGenModule::AddDeferredMultiVersionResolverToEmit(GlobalDecl GD) {
 }
 
 /// If a dispatcher for the specified mangled name is not in the module, create
-/// and return an llvm Function with the specified type.
+/// and return it. The dispatcher is either an llvm Function with the specified
+/// type, or a global ifunc.
 llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
   const auto *FD = cast<FunctionDecl>(GD.getDecl());
   assert(FD && "Not a FunctionDecl?");
@@ -4506,8 +4488,15 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
     ResolverName += ".resolver";
   }
 
-  // If the resolver has already been created, just return it.
-  if (llvm::GlobalValue *ResolverGV = GetGlobalValue(ResolverName))
+  // If the resolver has already been created, just return it. This lookup may
+  // yield a function declaration instead of a resolver on AArch64. That is
+  // because we didn't know whether a resolver will be generated when we first
+  // encountered a use of the symbol named after this resolver. Therefore,
+  // targets which support ifuncs should not return here unless we actually
+  // found an ifunc.
+  llvm::GlobalValue *ResolverGV = GetGlobalValue(ResolverName);
+  if (ResolverGV &&
+      (isa<llvm::GlobalIFunc>(ResolverGV) || !getTarget().supportsIFunc()))
     return ResolverGV;
 
   const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
@@ -4533,7 +4522,8 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
                                   "", Resolver, &getModule());
     GIF->setName(ResolverName);
     SetCommonAttributes(FD, GIF);
-
+    if (ResolverGV)
+      replaceDeclarationWith(ResolverGV, GIF);
     return GIF;
   }
 
@@ -4542,6 +4532,8 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
   assert(isa<llvm::GlobalValue>(Resolver) &&
          "Resolver should be created for the first time");
   SetCommonAttributes(FD, cast<llvm::GlobalValue>(Resolver));
+  if (ResolverGV)
+    replaceDeclarationWith(ResolverGV, Resolver);
   return Resolver;
 }
 
@@ -4571,6 +4563,7 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
     ForDefinition_t IsForDefinition) {
   const Decl *D = GD.getDecl();
 
+  std::string NameWithoutMultiVersionMangling;
   // Any attempts to use a MultiVersion function should result in retrieving
   // the iFunc instead. Name Mangling will handle the rest of the changes.
   if (const FunctionDecl *FD = cast_or_null<FunctionDecl>(D)) {
@@ -4592,14 +4585,24 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
 
     if (FD->isMultiVersion()) {
       UpdateMultiVersionNames(GD, FD, MangledName);
-      if (FD->getASTContext().getTargetInfo().getTriple().isAArch64() &&
-          !FD->isUsed())
-        AddDeferredMultiVersionResolverToEmit(GD);
-      else if (!IsForDefinition)
-        return GetOrCreateMultiVersionResolver(GD);
+      if (!IsForDefinition) {
+        // On AArch64 we do not immediatelly emit an ifunc resolver when a
+        // function is used. Instead we defer the emission until we see a
+        // default definition. In the meantime we just reference the symbol
+        // without FMV mangling (it may or may not be replaced later).
+        if (getTarget().getTriple().isAArch64()) {
+          AddDeferredMultiVersionResolverToEmit(GD);
+          NameWithoutMultiVersionMangling = getMangledNameImpl(
+              *this, GD, FD, /*OmitMultiVersionMangling=*/true);
+        } else
+          return GetOrCreateMultiVersionResolver(GD);
+      }
     }
   }
 
+  if (!NameWithoutMultiVersionMangling.empty())
+    MangledName = NameWithoutMultiVersionMangling;
+
   // Lookup the entry, lazily creating it if necessary.
   llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
   if (Entry) {
diff --git a/clang/test/CodeGen/aarch64-fmv-resolver-emission.c b/clang/test/CodeGen/aarch64-fmv-resolver-emission.c
new file mode 100644
index 0000000000000..eeafb3d41860d
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-fmv-resolver-emission.c
@@ -0,0 +1,111 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -emit-llvm -o - %s | FileCheck %s
+
+// CHECK: @used_before_default_def = weak_odr ifunc void (), ptr @used_before_default_def.resolver
+// CHECK: @used_after_default_def = weak_odr ifunc void (), ptr @used_after_default_def.resolver
+// CHECK-NOT: @used_before_default_decl = weak_odr ifunc void (), ptr @used_before_default_decl.resolver
+// CHECK-NOT: @used_after_default_decl = weak_odr ifunc void (), ptr @used_after_default_decl.resolver
+// CHECK-NOT: @used_no_default = weak_odr ifunc void (), ptr @used_no_default.resolver
+// CHECK-NOT: @not_used_no_default = weak_odr ifunc void (), ptr @not_used_no_default.resolver
+// CHECK: @not_used_with_default = weak_odr ifunc void (), ptr @not_used_with_default.resolver
+
+
+// Test that an ifunc is generated and used when the default
+// version is defined after the first use of the function.
+//
+__attribute__((target_version("aes"))) void used_before_default_def(void) {}
+// CHECK-LABEL: define dso_local void @used_before_default_def._Maes(
+//
+void call_before_def(void) { used_before_default_def(); }
+// CHECK-LABEL: define dso_local void @call_before_def(
+// CHECK: call void @used_before_default_def()
+//
+__attribute__((target_version("default"))) void used_before_default_def(void) {}
+// CHECK-LABEL: define dso_local void @used_before_default_def.default(
+//
+// CHECK-NOT: declare void @used_before_default_def(
+
+
+// Test that an ifunc is generated and used when the default
+// version is defined before the first use of the function.
+//
+__attribute__((target_version("aes"))) void used_after_default_def(void) {}
+// CHECK-LABEL: define dso_local void @used_after_default_def._Maes(
+//
+__attribute__((target_version("default"))) void used_after_default_def(void) {}
+// CHECK-LABEL: define dso_local void @used_after_default_def.default(
+//
+void call_after_def(void) { used_after_default_def(); }
+// CHECK-LABEL: define dso_local void @call_after_def(
+// CHECK: call void @used_after_default_def()
+//
+// CHECK-NOT: declare void @used_after_default_def(
+
+
+// Test that an unmagled declaration is generated and used when the
+// default version is declared after the first use of the function.
+//
+__attribute__((target_version("aes"))) void used_before_default_decl(void) {}
+// CHECK-LABEL: define dso_local void @used_before_default_decl._Maes(
+//
+void call_before_decl(void) { used_before_default_decl(); }
+// CHECK-LABEL: define dso_local void @call_before_decl(
+// CHECK: call void @used_before_default_decl()
+//
+__attribute__((target_version("default"))) void used_before_default_decl(void);
+// CHECK: declare void @used_before_default_decl()
+
+
+// Test that an unmagled declaration is generated and used when the
+// default version is declared before the first use of the function.
+//
+__attribute__((target_version("aes"))) void used_after_default_decl(void) {}
+// CHECK-LABEL: define dso_local void @used_after_default_decl._Maes(
+//
+__attribute__((target_version("default"))) void used_after_default_decl(void);
+// CHECK: declare void @used_after_default_decl()
+//
+void call_after_decl(void) { used_after_default_decl(); }
+// CHECK-LABEL: define dso_local void @call_after_decl(
+// CHECK: call void @used_after_default_decl()
+
+
+// Test that an unmagled declaration is generated and used when
+// the default version is not present.
+//
+__attribute__((target_version("aes"))) void used_no_default(void) {}
+// CHECK-LABEL: define dso_local void @used_no_default._Maes(
+//
+void call_no_default(void) { used_no_default(); }
+// CHECK-LABEL: define dso_local void @call_no_default(
+// CHECK: call void @used_no_default()
+//
+// CHECK: declare void @used_no_default()
+
+
+// Test that neither an ifunc nor a declaration is generated if the default
+// definition is missing since the versioned function is not used.
+//
+__attribute__((target_version("aes"))) void not_used_no_default(void) {}
+// CHECK-LABEL: define dso_local void @not_used_no_default._Maes(
+//
+// CHECK-NOT: declare void @not_used_no_default(
+
+
+// Test that an ifunc is generated if the default version is defined but not used.
+//
+__attribute__((target_version("aes"))) void not_used_with_default(void) {}
+// CHECK-LABEL: define dso_local void @not_used_with_default._Maes(
+//
+__attribute__((target_version("default"))) void not_used_with_default(void) {}
+// CHECK-LABEL: define dso_local void @not_used_with_default.default(
+//
+// CHECK-NOT: declare void @not_used_with_default(
+
+
+// CHECK: define weak_odr ptr @used_before_default_def.resolver()
+// CHECK: define weak_odr ptr @used_after_default_def.resolver()
+// CHECK-NOT: define weak_odr ptr @used_before_default_decl.resolver(
+// CHECK-NOT: define weak_odr ptr @used_after_default_decl.resolver(
+// CHECK-NOT: define weak_odr ptr @used_no_default.resolver(
+// CHECK-NOT: define weak_odr ptr @not_used_no_default.resolver(
+// CHECK: define weak_odr ptr @not_used_with_default.resolver()
diff --git a/clang/test/CodeGen/aarch64-mixed-target-attributes.c b/clang/test/CodeGen/aarch64-mixed-target-attributes.c
index 3c047fec6ceed..d779abd395b5f 100644
--- a/clang/test/CodeGen/aarch64-mixed-target-attributes.c
+++ b/clang/test/CodeGen/aarch64-mixed-target-attributes.c
@@ -261,9 +261,9 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void
 // CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-v9.5a" }
 // CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm,-v9.5a" }
 // CHECK: attributes #[[ATTR5:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR6:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR7:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-v9.5a" }
-// CHECK: attributes #[[ATTR8:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-v9.5a" }
+// CHECK: attributes #[[ATTR6:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-v9.5a" }
+// CHECK: attributes #[[ATTR7:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-v9.5a" }
+// CHECK: attributes #[[ATTR8:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
 //.
 // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" }
 //.
diff --git a/clang/test/CodeGen/attr-target-clones-aarch64.c b/clang/test/CodeGen/attr-target-clones-aarch64.c
index 60f9c7f1fc24e..846b08298cc72 100644
--- a/clang/test/CodeGen/attr-target-clones-aarch64.c
+++ b/clang/test/CodeGen/attr-target-clones-aarch64.c
@@ -32,8 +32,8 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK: @ftc_dup1 = weak_odr ifunc i32 (), ptr @ftc_dup1.resolver
 // CHECK: @ftc_dup2 = weak_odr ifunc i32 (), ptr @ftc_dup2.resolver
 // CHECK: @ftc_dup3 = weak_odr ifunc i32 (), ptr @ftc_dup3.resolver
-// CHECK: @ftc_inline1 = weak_odr ifunc i32 (), ptr @ftc_inline1.resolver
 // CHECK: @ftc_inline2 = weak_odr ifunc i32 (), ptr @ftc_inline2.resolver
+// CHECK: @ftc_inline1 = weak_odr ifunc i32 (), ptr @ftc_inline1.resolver
 // CHECK: @ftc_inline3 = weak_odr ifunc i32 (), ptr @ftc_inline3.resolver
 //.
 // CHECK-MTE-BTI: @__aarch64_cpu_features = external dso_local global { i64 }
@@ -42,8 +42,8 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-MTE-BTI: @ftc_dup1 = weak_odr ifunc i32 (), ptr @ftc_dup1.resolver
 // CHECK-MTE-BTI: @ftc_dup2 = weak_odr ifunc i32 (), ptr @ftc_dup2.resolver
 // CHECK-MTE-BTI: @ftc_dup3 = weak_odr ifunc i32 (), ptr @ftc_dup3.resolver
-// CHECK-MTE-BTI: @ftc_inline1 = weak_odr ifunc i32 (), ptr @ftc_inline1.resolver
 // CHECK-MTE-BTI: @ftc_inline2 = weak_odr ifunc i32 (), ptr @ftc_inline2.resolver
+// CHECK-MTE-BTI: @ftc_inline1 = weak_odr ifunc i32 (), ptr @ftc_inline1.resolver
 // CHECK-MTE-BTI: @ftc_inline3 = weak_odr ifunc i32 (), ptr @ftc_inline3.resolver
 //.
 // CHECK: Function Attrs: noinline nounwind optnone
@@ -210,12 +210,6 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_inline2._Mfp16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 2
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: @ftc_direct(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
@@ -236,86 +230,6 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 [[ADD5]]
 //
 //
-// CHECK-LABEL: @ftc_inline1.resolver(
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 18014535948435456
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 18014535948435456
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @ftc_inline1._Msve2-aesMwfxt
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 140737492549632
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 140737492549632
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @ftc_inline1._MpredresMrcpc
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 513
-// CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 513
-// CHECK-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
-// CHECK-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
-// CHECK:       resolver_return3:
-// CHECK-NEXT:    ret ptr @ftc_inline1._MrngMsimd
-// CHECK:       resolver_else4:
-// CHECK-NEXT:    ret ptr @ftc_inline1.default
-//
-//
-// CHECK-LABEL: @ftc_inline2.resolver(
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 549757911040
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 549757911040
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @ftc_inline2._MfcmaMsve2-bitperm
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 65536
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 65536
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @ftc_inline2._Mfp16
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @ftc_inline2.default
-//
-//
-// CHECK-LABEL: @ftc_inline3.resolver(
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 70369817919488
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70369817919488
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @ftc_inline3._MsbMsve
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 1125899906842624
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 1125899906842624
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @ftc_inline3._Mbti
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @ftc_inline3.default
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_inline2._MfcmaMsve2-bitperm(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 2
-//
-//
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: @ftc.default(
 // CHECK-NEXT:  entry:
@@ -347,11 +261,45 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: @ftc_inline2._Mfp16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: @ftc_inline2._MfcmaMsve2-bitperm(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: @ftc_inline2.default(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
+// CHECK-LABEL: @ftc_inline2.resolver(
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 549757911040
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 549757911040
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @ftc_inline2._MfcmaMsve2-bitperm
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 65536
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 65536
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @ftc_inline2._Mfp16
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    ret ptr @ftc_inline2.default
+//
+//
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: @ftc_inline1._MrngMsimd(
 // CHECK-NEXT:  entry:
@@ -376,6 +324,36 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 1
 //
 //
+// CHECK-LABEL: @ftc_inline1.resolver(
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 18014535948435456
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 18014535948435456
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @ftc_inline1._Msve2-aesMwfxt
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 140737492549632
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 140737492549632
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @ftc_inline1._MpredresMrcpc
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 513
+// CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 513
+// CHECK-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
+// CHECK-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
+// CHECK:       resolver_return3:
+// CHECK-NEXT:    ret ptr @ftc_inline1._MrngMsimd
+// CHECK:       resolver_else4:
+// CHECK-NEXT:    ret ptr @ftc_inline1.default
+//
+//
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: @ftc_inline3._Mbti(
 // CHECK-NEXT:  entry:
@@ -394,6 +372,28 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 3
 //
 //
+// CHECK-LABEL: @ftc_inline3.resolver(
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 70369817919488
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70369817919488
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @ftc_inline3._MsbMsve
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 1125899906842624
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 1125899906842624
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @ftc_inline3._Mbti
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    ret ptr @ftc_inline3.default
+//
+//
 // CHECK-NOFMV: Function Attrs: noinline nounwind optnone
 // CHECK-NOFMV-LABEL: @ftc(
 // CHECK-NOFMV-NEXT:  entry:
@@ -624,12 +624,6 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
-// CHECK-MTE-BTI-LABEL: @ftc_inline2._Mfp16(
-// CHECK-MTE-BTI-NEXT:  entry:
-// CHECK-MTE-BTI-NEXT:    ret i32 2
-//
-//
-// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: @ftc_direct(
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 4
@@ -650,86 +644,6 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-MTE-BTI-NEXT:    ret i32 [[ADD5]]
 //
 //
-// CHECK-MTE-BTI-LABEL: @ftc_inline1.resolver(
-// CHECK-MTE-BTI-NEXT:  resolver_entry:
-// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 18014535948435456
-// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 18014535948435456
-// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK-MTE-BTI:       resolver_return:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1._Msve2-aesMwfxt
-// CHECK-MTE-BTI:       resolver_else:
-// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 140737492549632
-// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 140737492549632
-// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK-MTE-BTI:       resolver_return1:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1._MpredresMrcpc
-// CHECK-MTE-BTI:       resolver_else2:
-// CHECK-MTE-BTI-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 513
-// CHECK-MTE-BTI-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 513
-// CHECK-MTE-BTI-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
-// CHECK-MTE-BTI:       resolver_return3:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1._MrngMsimd
-// CHECK-MTE-BTI:       resolver_else4:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1.default
-//
-//
-// CHECK-MTE-BTI-LABEL: @ftc_inline2.resolver(
-// CHECK-MTE-BTI-NEXT:  resolver_entry:
-// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 549757911040
-// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 549757911040
-// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK-MTE-BTI:       resolver_return:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline2._MfcmaMsve2-bitperm
-// CHECK-MTE-BTI:       resolver_else:
-// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 65536
-// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 65536
-// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK-MTE-BTI:       resolver_return1:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline2._Mfp16
-// CHECK-MTE-BTI:       resolver_else2:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline2.default
-//
-//
-// CHECK-MTE-BTI-LABEL: @ftc_inline3.resolver(
-// CHECK-MTE-BTI-NEXT:  resolver_entry:
-// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 70369817919488
-// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70369817919488
-// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK-MTE-BTI:       resolver_return:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline3._MsbMsve
-// CHECK-MTE-BTI:       resolver_else:
-// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 1125899906842624
-// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 1125899906842624
-// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK-MTE-BTI:       resolver_return1:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline3._Mbti
-// CHECK-MTE-BTI:       resolver_else2:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline3.default
-//
-//
-// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
-// CHECK-MTE-BTI-LABEL: @ftc_inline2._MfcmaMsve2-bitperm(
-// CHECK-MTE-BTI-NEXT:  entry:
-// CHECK-MTE-BTI-NEXT:    ret i32 2
-//
-//
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: @ftc.default(
 // CHECK-MTE-BTI-NEXT:  entry:
@@ -761,11 +675,45 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
+// CHECK-MTE-BTI-LABEL: @ftc_inline2._Mfp16(
+// CHECK-MTE-BTI-NEXT:  entry:
+// CHECK-MTE-BTI-NEXT:    ret i32 2
+//
+//
+// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
+// CHECK-MTE-BTI-LABEL: @ftc_inline2._MfcmaMsve2-bitperm(
+// CHECK-MTE-BTI-NEXT:  entry:
+// CHECK-MTE-BTI-NEXT:    ret i32 2
+//
+//
+// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: @ftc_inline2.default(
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 2
 //
 //
+// CHECK-MTE-BTI-LABEL: @ftc_inline2.resolver(
+// CHECK-MTE-BTI-NEXT:  resolver_entry:
+// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 549757911040
+// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 549757911040
+// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK-MTE-BTI:       resolver_return:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline2._MfcmaMsve2-bitperm
+// CHECK-MTE-BTI:       resolver_else:
+// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 65536
+// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 65536
+// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK-MTE-BTI:       resolver_return1:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline2._Mfp16
+// CHECK-MTE-BTI:       resolver_else2:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline2.default
+//
+//
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: @ftc_inline1._MrngMsimd(
 // CHECK-MTE-BTI-NEXT:  entry:
@@ -790,6 +738,36 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-MTE-BTI-NEXT:    ret i32 1
 //
 //
+// CHECK-MTE-BTI-LABEL: @ftc_inline1.resolver(
+// CHECK-MTE-BTI-NEXT:  resolver_entry:
+// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 18014535948435456
+// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 18014535948435456
+// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK-MTE-BTI:       resolver_return:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1._Msve2-aesMwfxt
+// CHECK-MTE-BTI:       resolver_else:
+// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 140737492549632
+// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 140737492549632
+// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK-MTE-BTI:       resolver_return1:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1._MpredresMrcpc
+// CHECK-MTE-BTI:       resolver_else2:
+// CHECK-MTE-BTI-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 513
+// CHECK-MTE-BTI-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 513
+// CHECK-MTE-BTI-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
+// CHECK-MTE-BTI:       resolver_return3:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1._MrngMsimd
+// CHECK-MTE-BTI:       resolver_else4:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1.default
+//
+//
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: @ftc_inline3._Mbti(
 // CHECK-MTE-BTI-NEXT:  entry:
@@ -807,6 +785,28 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 3
 //
+//
+// CHECK-MTE-BTI-LABEL: @ftc_inline3.resolver(
+// CHECK-MTE-BTI-NEXT:  resolver_entry:
+// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 70369817919488
+// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70369817919488
+// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK-MTE-BTI:       resolver_return:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline3._MsbMsve
+// CHECK-MTE-BTI:       resolver_else:
+// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 1125899906842624
+// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 1125899906842624
+// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK-MTE-BTI:       resolver_return1:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline3._Mbti
+// CHECK-MTE-BTI:       resolver_else2:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline3.default
+//
 //.
 // CHECK: attributes #[[ATTR0:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon" }
 // CHECK: attributes #[[ATTR1:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2" }
diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c
index 4edfc5408fae7..fbe34a51b40b6 100644
--- a/clang/test/CodeGen/attr-target-version.c
+++ b/clang/test/CodeGen/attr-target-version.c
@@ -11,15 +11,15 @@ int __attribute__((target_version("fp+aes"))) fmv(void) { return 6; }
 int __attribute__((target_version("crc+ls64_v"))) fmv(void) { return 7; }
 int __attribute__((target_version("bti"))) fmv(void) { return 8; }
 int __attribute__((target_version("sme2"))) fmv(void) { return 9; }
-int __attribute__((target_version("default"))) fmv(void);
+int __attribute__((target_version("default"))) fmv(void) { return 0; }
 int __attribute__((target_version("ls64+simd"))) fmv_one(void) { return 1; }
 int __attribute__((target_version("dpb"))) fmv_one(void) { return 2; }
-int __attribute__((target_version("default"))) fmv_one(void);
+int __attribute__((target_version("default"))) fmv_one(void) { return 0; }
 int __attribute__((target_version("fp"))) fmv_two(void) { return 1; }
 int __attribute__((target_version("simd"))) fmv_two(void) { return 2; }
 int __attribute__((target_version("dgh"))) fmv_two(void) { return 3; }
 int __attribute__((target_version("fp16+simd"))) fmv_two(void) { return 4; }
-int __attribute__((target_version("default"))) fmv_two(void);
+int __attribute__((target_version("default"))) fmv_two(void) { return 0; }
 int foo() {
   return fmv()+fmv_one()+fmv_two();
 }
@@ -124,11 +124,11 @@ __attribute__((target_version("rdma"))) int default_def_with_version_decls(void)
 
 // The following is guarded because in NOFMV we get errors for calling undeclared functions.
 #ifdef __HAVE_FUNCTION_MULTI_VERSIONING
-// This should generate a default declaration, two target versions and the resolver.
+// This should generate a default declaration, two target versions but no resolver.
 __attribute__((target_version("jscvt"))) int used_def_without_default_decl(void) { return 1; }
 __attribute__((target_version("rdma"))) int used_def_without_default_decl(void) { return 2; }
 
-// This should generate a default declaration and the resolver.
+// This should generate a default declaration but no resolver.
 __attribute__((target_version("jscvt"))) int used_decl_without_default_decl(void);
 __attribute__((target_version("rdma"))) int used_decl_without_default_decl(void);
 
@@ -140,12 +140,10 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 // CHECK: @fmv = weak_odr ifunc i32 (), ptr @fmv.resolver
 // CHECK: @fmv_one = weak_odr ifunc i32 (), ptr @fmv_one.resolver
 // CHECK: @fmv_two = weak_odr ifunc i32 (), ptr @fmv_two.resolver
-// CHECK: @fmv_inline = weak_odr ifunc i32 (), ptr @fmv_inline.resolver
 // CHECK: @fmv_e = weak_odr ifunc i32 (), ptr @fmv_e.resolver
 // CHECK: @fmv_d = internal ifunc i32 (), ptr @fmv_d.resolver
 // CHECK: @fmv_c = weak_odr ifunc void (), ptr @fmv_c.resolver
-// CHECK: @used_def_without_default_decl = weak_odr ifunc i32 (), ptr @used_def_without_default_decl.resolver
-// CHECK: @used_decl_without_default_decl = weak_odr ifunc i32 (), ptr @used_decl_without_default_decl.resolver
+// CHECK: @fmv_inline = weak_odr ifunc i32 (), ptr @fmv_inline.resolver
 // CHECK: @unused_with_default_def = weak_odr ifunc i32 (), ptr @unused_with_default_def.resolver
 // CHECK: @unused_with_implicit_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_default_def.resolver
 // CHECK: @unused_with_implicit_forward_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_forward_default_def.resolver
@@ -215,6 +213,13 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv.default
+// CHECK-SAME: () #[[ATTR9:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd
 // CHECK-SAME: () #[[ATTR5]] {
 // CHECK-NEXT:  entry:
@@ -229,6 +234,13 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_one.default
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp
 // CHECK-SAME: () #[[ATTR5]] {
 // CHECK-NEXT:  entry:
@@ -244,21 +256,28 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mdgh
-// CHECK-SAME: () #[[ATTR11:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR9]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp16Msimd
-// CHECK-SAME: () #[[ATTR12:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR11:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_two.default
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@foo
-// CHECK-SAME: () #[[ATTR11]] {
+// CHECK-SAME: () #[[ATTR9]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[CALL:%.*]] = call i32 @fmv()
 // CHECK-NEXT:    [[CALL1:%.*]] = call i32 @fmv_one()
@@ -268,6 +287,183 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 // CHECK-NEXT:    ret i32 [[ADD3]]
 //
 //
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_e.default
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 20
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_c.default
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@goo
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @fmv_inline()
+// CHECK-NEXT:    [[CALL1:%.*]] = call i32 @fmv_e()
+// CHECK-NEXT:    [[CALL2:%.*]] = call i32 @fmv_d()
+// CHECK-NEXT:    call void @fmv_c()
+// CHECK-NEXT:    [[CALL3:%.*]] = call i32 @fmv_default()
+// CHECK-NEXT:    ret i32 [[CALL3]]
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_default
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 111
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@recur
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    call void @reca()
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@hoo
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[FP1:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[FP2:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    call void @f(ptr noundef @fmv)
+// CHECK-NEXT:    store ptr @fmv, ptr [[FP1]], align 8
+// CHECK-NEXT:    store ptr @fmv, ptr [[FP2]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[FP1]], align 8
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 [[TMP0]]()
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[FP2]], align 8
+// CHECK-NEXT:    [[CALL1:%.*]] = call i32 [[TMP1]]()
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]]
+// CHECK-NEXT:    ret i32 [[ADD]]
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_with_forward_default_decl._Mmops
+// CHECK-SAME: () #[[ATTR13:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_extern_forward_default_decl._Mdotprod
+// CHECK-SAME: () #[[ATTR14:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes
+// CHECK-SAME: () #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve
+// CHECK-SAME: () #[[ATTR15:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16
+// CHECK-SAME: () #[[ATTR11]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse
+// CHECK-SAME: () #[[ATTR16:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_without_default._Mrdm
+// CHECK-SAME: () #[[ATTR17:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mjscvt
+// CHECK-SAME: () #[[ATTR19:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mrdm
+// CHECK-SAME: () #[[ATTR17]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@caller
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @used_def_without_default_decl()
+// CHECK-NEXT:    [[CALL1:%.*]] = call i32 @used_decl_without_default_decl()
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]]
+// CHECK-NEXT:    ret i32 [[ADD]]
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@main
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 0, ptr [[RETVAL]], align 4
+// CHECK-NEXT:    call void @recur()
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @goo()
+// CHECK-NEXT:    ret i32 [[CALL]]
+//
+//
 // CHECK-LABEL: define {{[^@]+}}@fmv.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -406,57 +602,185 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 // CHECK-NEXT:    ret ptr @fmv_two.default
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_e.default
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 20
+// CHECK-LABEL: define {{[^@]+}}@fmv_e.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 2251799813685248
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2251799813685248
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @fmv_e._Mls64
+// CHECK:       resolver_else:
+// CHECK-NEXT:    ret ptr @fmv_e.default
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb
-// CHECK-SAME: () #[[ATTR13:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR20:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_d.default
-// CHECK-SAME: () #[[ATTR11]] {
+// CHECK-SAME: () #[[ATTR9]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
+// CHECK-LABEL: define {{[^@]+}}@fmv_d.resolver() {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 70368744177664
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70368744177664
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @fmv_d._Msb
+// CHECK:       resolver_else:
+// CHECK-NEXT:    ret ptr @fmv_d.default
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@fmv_c.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 281474976710656
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 281474976710656
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @fmv_c._Mssbs
+// CHECK:       resolver_else:
+// CHECK-NEXT:    ret ptr @fmv_c.default
+//
+//
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs
-// CHECK-SAME: () #[[ATTR11]] {
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1
+// CHECK-SAME: () #[[ATTR21:[0-9]+]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret void
+// CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_c.default
-// CHECK-SAME: () #[[ATTR11]] {
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme
+// CHECK-SAME: () #[[ATTR22:[0-9]+]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret void
+// CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@goo
-// CHECK-SAME: () #[[ATTR11]] {
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf32mmMi8mmMsha3
+// CHECK-SAME: () #[[ATTR23:[0-9]+]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CALL:%.*]] = call i32 @fmv_inline()
-// CHECK-NEXT:    [[CALL1:%.*]] = call i32 @fmv_e()
-// CHECK-NEXT:    [[CALL2:%.*]] = call i32 @fmv_d()
-// CHECK-NEXT:    call void @fmv_c()
-// CHECK-NEXT:    [[CALL3:%.*]] = call i32 @fmv_default()
-// CHECK-NEXT:    ret i32 [[CALL3]]
+// CHECK-NEXT:    ret i32 12
 //
 //
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16
+// CHECK-SAME: () #[[ATTR24:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 8
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MdpbMrcpc2
+// CHECK-SAME: () #[[ATTR25:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 6
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mdpb2Mjscvt
+// CHECK-SAME: () #[[ATTR26:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 7
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfrinttsMrcpc
+// CHECK-SAME: () #[[ATTR27:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 3
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16
+// CHECK-SAME: () #[[ATTR28:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 4
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2-aesMsve2-sha3
+// CHECK-SAME: () #[[ATTR29:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 5
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-bitpermMsve2-pmull128
+// CHECK-SAME: () #[[ATTR30:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 9
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag2Msve2-sm4
+// CHECK-SAME: () #[[ATTR31:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 10
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag3MmopsMrcpc3
+// CHECK-SAME: () #[[ATTR32:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 11
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MaesMdotprod
+// CHECK-SAME: () #[[ATTR14]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 13
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mfp16fmlMsimd
+// CHECK-SAME: () #[[ATTR4]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 14
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfpMsm4
+// CHECK-SAME: () #[[ATTR33:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 15
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MlseMrdm
+// CHECK-SAME: () #[[ATTR34:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 16
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline.default
+// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 3
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 4398048673856
 // CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4398048673856
 // CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
@@ -587,355 +911,6 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 // CHECK-NEXT:    ret ptr @fmv_inline.default
 //
 //
-// CHECK-LABEL: define {{[^@]+}}@fmv_e.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 2251799813685248
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2251799813685248
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @fmv_e._Mls64
-// CHECK:       resolver_else:
-// CHECK-NEXT:    ret ptr @fmv_e.default
-//
-//
-// CHECK-LABEL: define {{[^@]+}}@fmv_d.resolver() {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 70368744177664
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70368744177664
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @fmv_d._Msb
-// CHECK:       resolver_else:
-// CHECK-NEXT:    ret ptr @fmv_d.default
-//
-//
-// CHECK-LABEL: define {{[^@]+}}@fmv_c.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 281474976710656
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 281474976710656
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @fmv_c._Mssbs
-// CHECK:       resolver_else:
-// CHECK-NEXT:    ret ptr @fmv_c.default
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_default
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 111
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@recur
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    call void @reca()
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@hoo
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[FP1:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    [[FP2:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    call void @f(ptr noundef @fmv)
-// CHECK-NEXT:    store ptr @fmv, ptr [[FP1]], align 8
-// CHECK-NEXT:    store ptr @fmv, ptr [[FP2]], align 8
-// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[FP1]], align 8
-// CHECK-NEXT:    [[CALL:%.*]] = call i32 [[TMP0]]()
-// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[FP2]], align 8
-// CHECK-NEXT:    [[CALL1:%.*]] = call i32 [[TMP1]]()
-// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]]
-// CHECK-NEXT:    ret i32 [[ADD]]
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_forward_default_decl._Mmops
-// CHECK-SAME: () #[[ATTR14:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_extern_forward_default_decl._Mdotprod
-// CHECK-SAME: () #[[ATTR15:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes
-// CHECK-SAME: () #[[ATTR5]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve
-// CHECK-SAME: () #[[ATTR16:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16
-// CHECK-SAME: () #[[ATTR12]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse
-// CHECK-SAME: () #[[ATTR17:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_without_default._Mrdm
-// CHECK-SAME: () #[[ATTR18:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mjscvt
-// CHECK-SAME: () #[[ATTR21:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mrdm
-// CHECK-SAME: () #[[ATTR18]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 2
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@caller
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CALL:%.*]] = call i32 @used_def_without_default_decl()
-// CHECK-NEXT:    [[CALL1:%.*]] = call i32 @used_decl_without_default_decl()
-// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]]
-// CHECK-NEXT:    ret i32 [[ADD]]
-//
-//
-// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1048576
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @used_def_without_default_decl._Mjscvt
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 64
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @used_def_without_default_decl._Mrdm
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @used_def_without_default_decl.default
-//
-//
-// CHECK-LABEL: define {{[^@]+}}@used_decl_without_default_decl.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1048576
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @used_decl_without_default_decl._Mjscvt
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 64
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @used_decl_without_default_decl._Mrdm
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @used_decl_without_default_decl.default
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@main
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 0, ptr [[RETVAL]], align 4
-// CHECK-NEXT:    call void @recur()
-// CHECK-NEXT:    [[CALL:%.*]] = call i32 @goo()
-// CHECK-NEXT:    ret i32 [[CALL]]
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1
-// CHECK-SAME: () #[[ATTR22:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme
-// CHECK-SAME: () #[[ATTR23:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 2
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf32mmMi8mmMsha3
-// CHECK-SAME: () #[[ATTR24:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 12
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16
-// CHECK-SAME: () #[[ATTR25:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 8
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MdpbMrcpc2
-// CHECK-SAME: () #[[ATTR26:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 6
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mdpb2Mjscvt
-// CHECK-SAME: () #[[ATTR27:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 7
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfrinttsMrcpc
-// CHECK-SAME: () #[[ATTR28:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 3
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16
-// CHECK-SAME: () #[[ATTR29:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 4
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2-aesMsve2-sha3
-// CHECK-SAME: () #[[ATTR30:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 5
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-bitpermMsve2-pmull128
-// CHECK-SAME: () #[[ATTR31:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 9
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag2Msve2-sm4
-// CHECK-SAME: () #[[ATTR32:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 10
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag3MmopsMrcpc3
-// CHECK-SAME: () #[[ATTR33:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 11
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MaesMdotprod
-// CHECK-SAME: () #[[ATTR15]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 13
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mfp16fmlMsimd
-// CHECK-SAME: () #[[ATTR4]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 14
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfpMsm4
-// CHECK-SAME: () #[[ATTR34:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 15
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MlseMrdm
-// CHECK-SAME: () #[[ATTR35:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 16
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline.default
-// CHECK-SAME: () #[[ATTR11]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 3
-//
-//
 // CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -1013,6 +988,27 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 //
 //
 // CHECK-NOFMV: Function Attrs: noinline nounwind optnone
+// CHECK-NOFMV-LABEL: define {{[^@]+}}@fmv
+// CHECK-NOFMV-SAME: () #[[ATTR0]] {
+// CHECK-NOFMV-NEXT:  entry:
+// CHECK-NOFMV-NEXT:    ret i32 0
+//
+//
+// CHECK-NOFMV: Function Attrs: noinline nounwind optnone
+// CHECK-NOFMV-LABEL: define {{[^@]+}}@fmv_one
+// CHECK-NOFMV-SAME: () #[[ATTR0]] {
+// CHECK-NOFMV-NEXT:  entry:
+// CHECK-NOFMV-NEXT:    ret i32 0
+//
+//
+// CHECK-NOFMV: Function Attrs: noinline nounwind optnone
+// CHECK-NOFMV-LABEL: define {{[^@]+}}@fmv_two
+// CHECK-NOFMV-SAME: () #[[ATTR0]] {
+// CHECK-NOFMV-NEXT:  entry:
+// CHECK-NOFMV-NEXT:    ret i32 0
+//
+//
+// CHECK-NOFMV: Function Attrs: noinline nounwind optnone
 // CHECK-NOFMV-LABEL: define {{[^@]+}}@fmv_e
 // CHECK-NOFMV-SAME: () #[[ATTR0]] {
 // CHECK-NOFMV-NEXT:  entry:
@@ -1125,33 +1121,33 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 // CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" }
 // CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" }
 // CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR9:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
 // CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
-// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm,-v9.5a" }
-// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR20:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm,-v9.5a" }
-// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
-// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fp-armv8,+fullfp16,+neon,+rdm,+sme,-v9.5a" }
-// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fp-armv8,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-v9.5a" }
-// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
-// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+fp-armv8,+jsconv,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
-// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-v9.5a" }
-// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-v9.5a" }
-// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-v9.5a" }
-// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+sm4,-v9.5a" }
-// CHECK: attributes #[[ATTR35]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon,+rdm,-v9.5a" }
+// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR12:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
+// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm,-v9.5a" }
+// CHECK: attributes #[[ATTR18:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR19]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
+// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fp-armv8,+fullfp16,+neon,+rdm,+sme,-v9.5a" }
+// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fp-armv8,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-v9.5a" }
+// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
+// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+fp-armv8,+jsconv,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
+// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-v9.5a" }
+// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-v9.5a" }
+// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-v9.5a" }
+// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+sm4,-v9.5a" }
+// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon,+rdm,-v9.5a" }
+// CHECK: attributes #[[ATTR35:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm,-v9.5a" }
 //.
 // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" }
 // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" }
diff --git a/clang/test/CodeGenCXX/aarch64-fmv-resolver-emission.cpp b/clang/test/CodeGenCXX/aarch64-fmv-resolver-emission.cpp
new file mode 100644
index 0000000000000..79c07c0d9db11
--- /dev/null
+++ b/clang/test/CodeGenCXX/aarch64-fmv-resolver-emission.cpp
@@ -0,0 +1,111 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -emit-llvm -o - %s | FileCheck %s
+
+// CHECK: @_Z23used_before_default_defv = weak_odr ifunc void (), ptr @_Z23used_before_default_defv.resolver
+// CHECK: @_Z22used_after_default_defv = weak_odr ifunc void (), ptr @_Z22used_after_default_defv.resolver
+// CHECK-NOT: @_Z24used_before_default_declv = weak_odr ifunc void (), ptr @_Z24used_before_default_declv.resolver
+// CHECK-NOT: @_Z23used_after_default_declv = weak_odr ifunc void (), ptr @_Z23used_after_default_declv.resolver
+// CHECK-NOT: @_Z15used_no_defaultv = weak_odr ifunc void (), ptr @_Z15used_no_defaultv.resolver
+// CHECK-NOT: @_Z19not_used_no_defaultv = weak_odr ifunc void (), ptr @_Z19not_used_no_defaultv.resolver
+// CHECK: @_Z21not_used_with_defaultv = weak_odr ifunc void (), ptr @_Z21not_used_with_defaultv.resolver
+
+
+// Test that an ifunc is generated and used when the default
+// version is defined after the first use of the function.
+//
+__attribute__((target_version("aes"))) void used_before_default_def(void) {}
+// CHECK-LABEL: define dso_local void @_Z23used_before_default_defv._Maes(
+//
+void call_before_def(void) { used_before_default_def(); }
+// CHECK-LABEL: define dso_local void @_Z15call_before_defv(
+// CHECK: call void @_Z23used_before_default_defv()
+//
+__attribute__((target_version("default"))) void used_before_default_def(void) {}
+// CHECK-LABEL: define dso_local void @_Z23used_before_default_defv.default(
+//
+// CHECK-NOT: declare void @_Z23used_before_default_defv(
+
+
+// Test that an ifunc is generated and used when the default
+// version is defined before the first use of the function.
+//
+__attribute__((target_version("aes"))) void used_after_default_def(void) {}
+// CHECK-LABEL: define dso_local void @_Z22used_after_default_defv._Maes(
+//
+__attribute__((target_version("default"))) void used_after_default_def(void) {}
+// CHECK-LABEL: define dso_local void @_Z22used_after_default_defv.default(
+//
+void call_after_def(void) { used_after_default_def(); }
+// CHECK-LABEL: define dso_local void @_Z14call_after_defv(
+// CHECK: call void @_Z22used_after_default_defv()
+//
+// CHECK-NOT: declare void @_Z22used_after_default_defv(
+
+
+// Test that an unmagled declaration is generated and used when the
+// default version is declared after the first use of the function.
+//
+__attribute__((target_version("aes"))) void used_before_default_decl(void) {}
+// CHECK-LABEL: define dso_local void @_Z24used_before_default_declv._Maes(
+//
+void call_before_decl(void) { used_before_default_decl(); }
+// CHECK-LABEL: define dso_local void @_Z16call_before_declv(
+// CHECK: call void @_Z24used_before_default_declv()
+//
+__attribute__((target_version("default"))) void used_before_default_decl(void);
+// CHECK: declare void @_Z24used_before_default_declv()
+
+
+// Test that an unmagled declaration is generated and used when the
+// default version is declared before the first use of the function.
+//
+__attribute__((target_version("aes"))) void used_after_default_decl(void) {}
+// CHECK-LABEL: define dso_local void @_Z23used_after_default_declv._Maes(
+//
+__attribute__((target_version("default"))) void used_after_default_decl(void);
+// CHECK: declare void @_Z23used_after_default_declv()
+//
+void call_after_decl(void) { used_after_default_decl(); }
+// CHECK-LABEL: define dso_local void @_Z15call_after_declv(
+// CHECK: call void @_Z23used_after_default_declv()
+
+
+// Test that an unmagled declaration is generated and used when
+// the default version is not present.
+//
+__attribute__((target_version("aes"))) void used_no_default(void) {}
+// CHECK-LABEL: define dso_local void @_Z15used_no_defaultv._Maes(
+//
+void call_no_default(void) { used_no_default(); }
+// CHECK-LABEL: define dso_local void @_Z15call_no_defaultv(
+// CHECK: call void @_Z15used_no_defaultv()
+//
+// CHECK: declare void @_Z15used_no_defaultv()
+
+
+// Test that neither an ifunc nor a declaration is generated if the default
+// definition is missing since the versioned function is not used.
+//
+__attribute__((target_version("aes"))) void not_used_no_default(void) {}
+// CHECK-LABEL: define dso_local void @_Z19not_used_no_defaultv._Maes(
+//
+// CHECK-NOT: declare void @_Z19not_used_no_defaultv(
+
+
+// Test that an ifunc is generated if the default version is defined but not used.
+//
+__attribute__((target_version("aes"))) void not_used_with_default(void) {}
+// CHECK-LABEL: define dso_local void @_Z21not_used_with_defaultv._Maes(
+//
+__attribute__((target_version("default"))) void not_used_with_default(void) {}
+// CHECK-LABEL: define dso_local void @_Z21not_used_with_defaultv.default(
+//
+// CHECK-NOT: declare void @_Z21not_used_with_defaultv(
+
+
+// CHECK: define weak_odr ptr @_Z23used_before_default_defv.resolver()
+// CHECK: define weak_odr ptr @_Z22used_after_default_defv.resolver()
+// CHECK-NOT: define weak_odr ptr @_Z24used_before_default_declv.resolver(
+// CHECK-NOT: define weak_odr ptr @_Z23used_after_default_declv.resolver(
+// CHECK-NOT: define weak_odr ptr @_Z15used_no_defaultv.resolver(
+// CHECK-NOT: define weak_odr ptr @_Z19not_used_no_defaultv.resolver(
+// CHECK: define weak_odr ptr @_Z21not_used_with_defaultv.resolver()
diff --git a/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp b/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp
index 29ae6b6856500..6405621a9d647 100644
--- a/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp
+++ b/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp
@@ -1,4 +1,4 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals --include-generated-funcs
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 5
 // RUN: %clang_cc1 -std=c++11 -triple aarch64-linux-gnu -emit-llvm %s -o - | FileCheck %s
 
 int __attribute__((target_clones("ls64_v+fp16", "default"))) foo_ovl(int) { return 1; }
@@ -45,56 +45,60 @@ void run_foo_tml() {
 // CHECK: @_ZN7MyClassIssE7foo_tmlEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClassIssE7foo_tmlEv.resolver
 // CHECK: @_ZN7MyClassIisE7foo_tmlEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClassIisE7foo_tmlEv.resolver
 //.
-// CHECK-LABEL: @_Z7foo_ovli._Mfp16Mls64_v(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_Z7foo_ovli._Mfp16Mls64_v(
+// CHECK-SAME: i32 noundef [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[DOTADDR]], align 4
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_Z7foo_ovli.resolver(
-// CHECK-NEXT:  resolver_entry:
+// CHECK-LABEL: define weak_odr ptr @_Z7foo_ovli.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 4503599627436032
 // CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4503599627436032
 // CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
 // CHECK-NEXT:    ret ptr @_Z7foo_ovli._Mfp16Mls64_v
-// CHECK:       resolver_else:
+// CHECK:       [[RESOLVER_ELSE]]:
 // CHECK-NEXT:    ret ptr @_Z7foo_ovli.default
 //
 //
-// CHECK-LABEL: @_Z7foo_ovlv._Mls64Mls64_accdata(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_Z7foo_ovlv._Mls64Mls64_accdata(
+// CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK-LABEL: @_Z7foo_ovlv.resolver(
-// CHECK-NEXT:  resolver_entry:
+// CHECK-LABEL: define weak_odr ptr @_Z7foo_ovlv.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 11258999068426240
 // CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 11258999068426240
 // CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
 // CHECK-NEXT:    ret ptr @_Z7foo_ovlv._Mls64Mls64_accdata
-// CHECK:       resolver_else:
+// CHECK:       [[RESOLVER_ELSE]]:
 // CHECK-NEXT:    ret ptr @_Z7foo_ovlv.default
 //
 //
-// CHECK-LABEL: @_Z3barv(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_Z3barv(
+// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[CALL:%.*]] = call noundef i32 @_Z7foo_ovli(i32 noundef 1)
 // CHECK-NEXT:    [[CALL1:%.*]] = call noundef i32 @_Z7foo_ovlv()
 // CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]]
 // CHECK-NEXT:    ret i32 [[ADD]]
 //
 //
-// CHECK-LABEL: @_Z11run_foo_tmlv(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local void @_Z11run_foo_tmlv(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MC1:%.*]] = alloca [[STRUCT_MYCLASS:%.*]], align 1
 // CHECK-NEXT:    [[MC2:%.*]] = alloca [[STRUCT_MYCLASS_0:%.*]], align 1
 // CHECK-NEXT:    [[MC3:%.*]] = alloca [[STRUCT_MYCLASS_1:%.*]], align 1
@@ -106,131 +110,141 @@ void run_foo_tml() {
 // CHECK-NEXT:    ret void
 //
 //
-// CHECK-LABEL: @_ZN7MyClassIssE7foo_tmlEv.resolver(
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 36310271995674624
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36310271995674624
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @_ZN7MyClassIssE7foo_tmlEv._Msme-f64f64Mssbs
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 16777216
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16777216
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @_ZN7MyClassIssE7foo_tmlEv._Mfrintts
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @_ZN7MyClassIssE7foo_tmlEv.default
-//
-//
-// CHECK-LABEL: @_ZN7MyClassIisE7foo_tmlEv.resolver(
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 36310271995674624
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36310271995674624
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @_ZN7MyClassIisE7foo_tmlEv._Msme-f64f64Mssbs
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 16777216
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16777216
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @_ZN7MyClassIisE7foo_tmlEv._Mfrintts
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @_ZN7MyClassIisE7foo_tmlEv.default
-//
-//
-// CHECK-LABEL: @_ZN7MyClassIfsE7foo_tmlEv(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIfsE7foo_tmlEv(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR2]] comdat {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 3
 //
 //
-// CHECK-LABEL: @_ZN7MyClassIdfE7foo_tmlEv(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIdfE7foo_tmlEv(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR2]] comdat {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 4
 //
 //
-// CHECK-LABEL: @_Z7foo_ovli.default(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_Z7foo_ovli.default(
+// CHECK-SAME: i32 noundef [[TMP0:%.*]]) #[[ATTR2]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[DOTADDR]], align 4
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_Z7foo_ovlv.default(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_Z7foo_ovlv.default(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK-LABEL: @_ZN7MyClassIssE7foo_tmlEv._Mfrintts(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIssE7foo_tmlEv._Mfrintts(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR3:[0-9]+]] comdat {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_ZN7MyClassIssE7foo_tmlEv._Msme-f64f64Mssbs(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIssE7foo_tmlEv._Msme-f64f64Mssbs(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR4:[0-9]+]] comdat {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_ZN7MyClassIssE7foo_tmlEv.default(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIssE7foo_tmlEv.default(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR2]] comdat {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_ZN7MyClassIisE7foo_tmlEv._Mfrintts(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define weak_odr ptr @_ZN7MyClassIssE7foo_tmlEv.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 36310271995674624
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36310271995674624
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
+// CHECK-NEXT:    ret ptr @_ZN7MyClassIssE7foo_tmlEv._Msme-f64f64Mssbs
+// CHECK:       [[RESOLVER_ELSE]]:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 16777216
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16777216
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label %[[RESOLVER_RETURN1:.*]], label %[[RESOLVER_ELSE2:.*]]
+// CHECK:       [[RESOLVER_RETURN1]]:
+// CHECK-NEXT:    ret ptr @_ZN7MyClassIssE7foo_tmlEv._Mfrintts
+// CHECK:       [[RESOLVER_ELSE2]]:
+// CHECK-NEXT:    ret ptr @_ZN7MyClassIssE7foo_tmlEv.default
+//
+//
+// CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIisE7foo_tmlEv._Mfrintts(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR3]] comdat {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK-LABEL: @_ZN7MyClassIisE7foo_tmlEv._Msme-f64f64Mssbs(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIisE7foo_tmlEv._Msme-f64f64Mssbs(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR4]] comdat {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK-LABEL: @_ZN7MyClassIisE7foo_tmlEv.default(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIisE7foo_tmlEv.default(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR2]] comdat {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 2
 //
+//
+// CHECK-LABEL: define weak_odr ptr @_ZN7MyClassIisE7foo_tmlEv.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 36310271995674624
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36310271995674624
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
+// CHECK-NEXT:    ret ptr @_ZN7MyClassIisE7foo_tmlEv._Msme-f64f64Mssbs
+// CHECK:       [[RESOLVER_ELSE]]:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 16777216
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16777216
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label %[[RESOLVER_RETURN1:.*]], label %[[RESOLVER_ELSE2:.*]]
+// CHECK:       [[RESOLVER_RETURN1]]:
+// CHECK-NEXT:    ret ptr @_ZN7MyClassIisE7foo_tmlEv._Mfrintts
+// CHECK:       [[RESOLVER_ELSE2]]:
+// CHECK-NEXT:    ret ptr @_ZN7MyClassIisE7foo_tmlEv.default
+//
 //.
-// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" }
-// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ls64" }
-// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint" }
-// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" }
+// CHECK: attributes #[[ATTR0]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" }
+// CHECK: attributes #[[ATTR1]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ls64" }
+// CHECK: attributes #[[ATTR2]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK: attributes #[[ATTR3]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint" }
+// CHECK: attributes #[[ATTR4]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" }
 //.
 // CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
 // CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
diff --git a/clang/test/CodeGenCXX/attr-target-version.cpp b/clang/test/CodeGenCXX/attr-target-version.cpp
index fd19f4c5a3030..6661abead20c6 100644
--- a/clang/test/CodeGenCXX/attr-target-version.cpp
+++ b/clang/test/CodeGenCXX/attr-target-version.cpp
@@ -1,4 +1,4 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals --include-generated-funcs
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 5
 // RUN: %clang_cc1 -std=c++11 -triple aarch64-linux-gnu -emit-llvm %s -o - | FileCheck %s
 
 int __attribute__((target_version("sme-f64f64+bf16"))) foo(int) { return 1; }
@@ -59,152 +59,169 @@ int bar() {
   return m.goo(1) + foo(1) + foo();
 }
 
-
 //.
 // CHECK: @__aarch64_cpu_features = external dso_local global { i64 }
-// CHECK: @_ZN7MyClass3gooEi = weak_odr ifunc i32 (ptr, i32), ptr @_ZN7MyClass3gooEi.resolver
 // CHECK: @_Z3fooi = weak_odr ifunc i32 (i32), ptr @_Z3fooi.resolver
 // CHECK: @_Z3foov = weak_odr ifunc i32 (), ptr @_Z3foov.resolver
+// CHECK: @_ZN7MyClass3gooEi = weak_odr ifunc i32 (ptr, i32), ptr @_ZN7MyClass3gooEi.resolver
 // CHECK: @_ZN7MyClass23unused_with_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass23unused_with_default_defEv.resolver
 // CHECK: @_ZN7MyClass32unused_with_implicit_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass32unused_with_implicit_default_defEv.resolver
 // CHECK: @_ZN7MyClass40unused_with_implicit_forward_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.resolver
 //.
-// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_Z3fooi._Mbf16Msme-f64f64(
+// CHECK-SAME: i32 noundef [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[DOTADDR]], align 4
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_Z3fooi.default(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_Z3fooi.default(
+// CHECK-SAME: i32 noundef [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[DOTADDR]], align 4
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK-LABEL: @_Z3foov._Mebf16Msm4(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_Z3foov._Mebf16Msm4(
+// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    ret i32 3
 //
 //
-// CHECK-LABEL: @_Z3foov.default(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_Z3foov.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    ret i32 4
 //
 //
-// CHECK-LABEL: @_ZN7MyClass3gooEi.default(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass3gooEi.default(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]], i32 noundef [[TMP0:%.*]]) #[[ATTR1]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
-// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[DOTADDR]], align 4
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_ZN7MyClass3gooEi._Mcrc(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass3gooEi._Mcrc(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]], i32 noundef [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
-// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[DOTADDR]], align 4
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK-LABEL: @_ZN7MyClass3gooEi._Mdotprod(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass3gooEi._Mdotprod(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]], i32 noundef [[TMP0:%.*]]) #[[ATTR4:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
-// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[DOTADDR]], align 4
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 3
 //
 //
-// CHECK-LABEL: @_ZN7MyClass32unused_with_forward_default_declEv._Mmops(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass32unused_with_forward_default_declEv._Mmops(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR5:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK-LABEL: @_ZN7MyClass41unused_with_implicit_forward_default_declEv._Mdotprod(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass41unused_with_implicit_forward_default_declEv._Mdotprod(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR4]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK-LABEL: @_ZN7MyClass24unused_with_default_declEv._Maes(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass24unused_with_default_declEv._Maes(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR6:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv._Msve(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass23unused_with_default_defEv._Msve(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR7:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv.default(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass23unused_with_default_defEv.default(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR1]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_ZN7MyClass32unused_with_implicit_default_defEv._Mfp16(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass32unused_with_implicit_default_defEv._Mfp16(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR8:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK-LABEL: @_ZN7MyClass32unused_with_implicit_default_defEv.default(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass32unused_with_implicit_default_defEv.default(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR1]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_ZN7MyClass40unused_with_implicit_forward_default_defEv.default(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass40unused_with_implicit_forward_default_defEv.default(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR1]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK-LABEL: @_ZN7MyClass40unused_with_implicit_forward_default_defEv._Mlse(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass40unused_with_implicit_forward_default_defEv._Mlse(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR9:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv._Mrdm(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_ZN7MyClass22unused_without_defaultEv._Mrdm(
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR10:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK-LABEL: @_Z3barv(
-// CHECK-NEXT:  entry:
+// CHECK-LABEL: define dso_local noundef i32 @_Z3barv(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[M:%.*]] = alloca [[STRUCT_MYCLASS:%.*]], align 1
 // CHECK-NEXT:    [[CALL:%.*]] = call noundef i32 @_ZN7MyClass3gooEi(ptr noundef nonnull align 1 dereferenceable(1) [[M]], i32 noundef 1)
 // CHECK-NEXT:    [[CALL1:%.*]] = call noundef i32 @_Z3fooi(i32 noundef 1)
@@ -214,109 +231,109 @@ int bar() {
 // CHECK-NEXT:    ret i32 [[ADD3]]
 //
 //
-// CHECK-LABEL: @_ZN7MyClass3gooEi.resolver(
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1024
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1024
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @_ZN7MyClass3gooEi._Mcrc
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 16
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @_ZN7MyClass3gooEi._Mdotprod
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @_ZN7MyClass3gooEi.default
-//
-//
-// CHECK-LABEL: @_Z3fooi.resolver(
-// CHECK-NEXT:  resolver_entry:
+// CHECK-LABEL: define weak_odr ptr @_Z3fooi.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 36028797153181696
 // CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36028797153181696
 // CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
 // CHECK-NEXT:    ret ptr @_Z3fooi._Mbf16Msme-f64f64
-// CHECK:       resolver_else:
+// CHECK:       [[RESOLVER_ELSE]]:
 // CHECK-NEXT:    ret ptr @_Z3fooi.default
 //
 //
-// CHECK-LABEL: @_Z3foov.resolver(
-// CHECK-NEXT:  resolver_entry:
+// CHECK-LABEL: define weak_odr ptr @_Z3foov.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 268435488
 // CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 268435488
 // CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
 // CHECK-NEXT:    ret ptr @_Z3foov._Mebf16Msm4
-// CHECK:       resolver_else:
+// CHECK:       [[RESOLVER_ELSE]]:
 // CHECK-NEXT:    ret ptr @_Z3foov.default
 //
 //
-// CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv.resolver(
-// CHECK-NEXT:  resolver_entry:
+// CHECK-LABEL: define weak_odr ptr @_ZN7MyClass3gooEi.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1024
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1024
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
+// CHECK-NEXT:    ret ptr @_ZN7MyClass3gooEi._Mcrc
+// CHECK:       [[RESOLVER_ELSE]]:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 16
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label %[[RESOLVER_RETURN1:.*]], label %[[RESOLVER_ELSE2:.*]]
+// CHECK:       [[RESOLVER_RETURN1]]:
+// CHECK-NEXT:    ret ptr @_ZN7MyClass3gooEi._Mdotprod
+// CHECK:       [[RESOLVER_ELSE2]]:
+// CHECK-NEXT:    ret ptr @_ZN7MyClass3gooEi.default
+//
+//
+// CHECK-LABEL: define weak_odr ptr @_ZN7MyClass23unused_with_default_defEv.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1073741824
 // CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1073741824
 // CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
 // CHECK-NEXT:    ret ptr @_ZN7MyClass23unused_with_default_defEv._Msve
-// CHECK:       resolver_else:
+// CHECK:       [[RESOLVER_ELSE]]:
 // CHECK-NEXT:    ret ptr @_ZN7MyClass23unused_with_default_defEv.default
 //
 //
-// CHECK-LABEL: @_ZN7MyClass32unused_with_implicit_default_defEv.resolver(
-// CHECK-NEXT:  resolver_entry:
+// CHECK-LABEL: define weak_odr ptr @_ZN7MyClass32unused_with_implicit_default_defEv.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 65536
 // CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 65536
 // CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
 // CHECK-NEXT:    ret ptr @_ZN7MyClass32unused_with_implicit_default_defEv._Mfp16
-// CHECK:       resolver_else:
+// CHECK:       [[RESOLVER_ELSE]]:
 // CHECK-NEXT:    ret ptr @_ZN7MyClass32unused_with_implicit_default_defEv.default
 //
 //
-// CHECK-LABEL: @_ZN7MyClass40unused_with_implicit_forward_default_defEv.resolver(
-// CHECK-NEXT:  resolver_entry:
+// CHECK-LABEL: define weak_odr ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 128
 // CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 128
 // CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
 // CHECK-NEXT:    ret ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv._Mlse
-// CHECK:       resolver_else:
+// CHECK:       [[RESOLVER_ELSE]]:
 // CHECK-NEXT:    ret ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.default
 //
 //.
-// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" }
-// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" }
-// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" }
-// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" }
-// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" }
-// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" }
-// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" }
-// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" }
-// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" }
-// CHECK: attributes #[[ATTR10:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm" }
+// CHECK: attributes #[[ATTR0]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" }
+// CHECK: attributes #[[ATTR1]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK: attributes #[[ATTR2]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" }
+// CHECK: attributes #[[ATTR3]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" }
+// CHECK: attributes #[[ATTR4]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" }
+// CHECK: attributes #[[ATTR5]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" }
+// CHECK: attributes #[[ATTR6]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" }
+// CHECK: attributes #[[ATTR7]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" }
+// CHECK: attributes #[[ATTR8]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" }
+// CHECK: attributes #[[ATTR9]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" }
+// CHECK: attributes #[[ATTR10]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm" }
 // CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 //.
 // CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
diff --git a/clang/test/CodeGenCXX/fmv-namespace.cpp b/clang/test/CodeGenCXX/fmv-namespace.cpp
index abfff1a74f86a..1ac88e68a3a12 100644
--- a/clang/test/CodeGenCXX/fmv-namespace.cpp
+++ b/clang/test/CodeGenCXX/fmv-namespace.cpp
@@ -26,7 +26,6 @@ __attribute((target_version("mops"))) int bar() { return 1; }
 //.
 // CHECK: @__aarch64_cpu_features = external dso_local global { i64 }
 // CHECK: @_ZN4Name3fooEv = weak_odr ifunc i32 (), ptr @_ZN4Name3fooEv.resolver
-// CHECK: @_ZN9OtherName3fooEv = weak_odr ifunc i32 (), ptr @_ZN9OtherName3fooEv.resolver
 // CHECK: @_ZN3Foo3barEv = weak_odr ifunc i32 (), ptr @_ZN3Foo3barEv.resolver
 //.
 // CHECK-LABEL: define dso_local noundef i32 @_ZN4Name3fooEv._Msve(
@@ -42,20 +41,6 @@ __attribute((target_version("mops"))) int bar() { return 1; }
 // CHECK-NEXT:    ret i32 [[CALL]]
 //
 //
-// CHECK-LABEL: define weak_odr ptr @_ZN4Name3fooEv.resolver() comdat {
-// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1073741824
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1073741824
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
-// CHECK:       [[RESOLVER_RETURN]]:
-// CHECK-NEXT:    ret ptr @_ZN4Name3fooEv._Msve
-// CHECK:       [[RESOLVER_ELSE]]:
-// CHECK-NEXT:    ret ptr @_ZN4Name3fooEv.default
-//
-//
 // CHECK-LABEL: define dso_local noundef i32 @_ZN9OtherName3fooEv._Msve(
 // CHECK-SAME: ) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -69,20 +54,6 @@ __attribute((target_version("mops"))) int bar() { return 1; }
 // CHECK-NEXT:    ret i32 [[CALL]]
 //
 //
-// CHECK-LABEL: define weak_odr ptr @_ZN9OtherName3fooEv.resolver() comdat {
-// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1073741824
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1073741824
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
-// CHECK:       [[RESOLVER_RETURN]]:
-// CHECK-NEXT:    ret ptr @_ZN9OtherName3fooEv._Msve
-// CHECK:       [[RESOLVER_ELSE]]:
-// CHECK-NEXT:    ret ptr @_ZN9OtherName3fooEv.default
-//
-//
 // CHECK-LABEL: define dso_local noundef i32 @_ZN3Foo3barEv.default(
 // CHECK-SAME: ) #[[ATTR1]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -90,7 +61,7 @@ __attribute((target_version("mops"))) int bar() { return 1; }
 //
 //
 // CHECK-LABEL: define dso_local noundef i32 @_ZN3Foo3barEv._Mmops(
-// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
+// CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    ret i32 1
 //
@@ -101,6 +72,20 @@ __attribute((target_version("mops"))) int bar() { return 1; }
 // CHECK-NEXT:    ret i32 0
 //
 //
+// CHECK-LABEL: define weak_odr ptr @_ZN4Name3fooEv.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1073741824
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1073741824
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
+// CHECK-NEXT:    ret ptr @_ZN4Name3fooEv._Msve
+// CHECK:       [[RESOLVER_ELSE]]:
+// CHECK-NEXT:    ret ptr @_ZN4Name3fooEv.default
+//
+//
 // CHECK-LABEL: define weak_odr ptr @_ZN3Foo3barEv.resolver() comdat {
 // CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -117,8 +102,8 @@ __attribute((target_version("mops"))) int bar() { return 1; }
 //.
 // CHECK: attributes #[[ATTR0]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" }
 // CHECK: attributes #[[ATTR1]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-// CHECK: attributes #[[ATTR2]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" }
-// CHECK: attributes #[[ATTR3:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK: attributes #[[ATTR2:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" }
+// CHECK: attributes #[[ATTR3]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" }
 //.
 // CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
 // CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}

From 9af3628ce7400a96205a4c4468867c3c11dd4b2f Mon Sep 17 00:00:00 2001
From: Ulrich Weigand <ulrich.weigand@de.ibm.com>
Date: Thu, 18 Jul 2024 17:43:28 +0200
Subject: [PATCH 037/486] [SystemZ] Fix transparent_union calling convention

The SystemZ ABI code was missing code to handle the transparent_union
extension.  Arguments of such types are specified to be passed like
the first member of the union, instead of according to the usual
ABI calling convention for aggregates.

This did not make much difference in practice as the SystemZ ABI
already specifies that 1-, 2-, 4- or 8-byte aggregates are passed
in registers.  However, there *is* a difference if the first member
of the transparent union is a scalar integer type smaller than word
size - if passed as a scalar, it needs to be zero- or sign-extended
to word size, while if passed as aggregate, it is not.

Fixed by adding code to handle transparent_union similar to what
is done on other targets.
---
 clang/lib/CodeGen/Targets/SystemZ.cpp    |  5 ++++-
 clang/test/CodeGen/SystemZ/systemz-abi.c | 23 +++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/Targets/SystemZ.cpp b/clang/lib/CodeGen/Targets/SystemZ.cpp
index e6b63b6fe093f..4d61f51379346 100644
--- a/clang/lib/CodeGen/Targets/SystemZ.cpp
+++ b/clang/lib/CodeGen/Targets/SystemZ.cpp
@@ -412,13 +412,16 @@ ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const {
 }
 
 ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const {
+  // Handle transparent union types.
+  Ty = useFirstFieldIfTransparentUnion(Ty);
+
   // Handle the generic C++ ABI.
   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
     return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
 
   // Integers and enums are extended to full register width.
   if (isPromotableIntegerTypeForABI(Ty))
-    return ABIArgInfo::getExtend(Ty);
+    return ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty));
 
   // Handle vector types and vector-like structure types.  Note that
   // as opposed to float-like structure types, we do not allow any
diff --git a/clang/test/CodeGen/SystemZ/systemz-abi.c b/clang/test/CodeGen/SystemZ/systemz-abi.c
index 65a2bc9bbb680..3e39425e57f17 100644
--- a/clang/test/CodeGen/SystemZ/systemz-abi.c
+++ b/clang/test/CodeGen/SystemZ/systemz-abi.c
@@ -173,6 +173,29 @@ union union_double pass_union_double(union union_double arg) { return arg; }
 // CHECK-LABEL: define{{.*}} void @pass_union_double(ptr dead_on_unwind noalias writable sret(%union.union_double) align 8 %{{.*}}, i64 %{{.*}})
 
 
+// Verify that transparent unions are passed like their first member (but returned like a union)
+
+union tu_char { char a; } __attribute__((transparent_union));
+union tu_char pass_tu_char(union tu_char arg) { return arg; }
+// CHECK-LABEL: define{{.*}} void @pass_tu_char(ptr dead_on_unwind noalias writable sret(%union.tu_char) align 1 %{{.*}}, i8 signext %{{.*}})
+
+union tu_short { short a; } __attribute__((transparent_union));
+union tu_short pass_tu_short(union tu_short arg) { return arg; }
+// CHECK-LABEL: define{{.*}} void @pass_tu_short(ptr dead_on_unwind noalias writable sret(%union.tu_short) align 2 %{{.*}}, i16 signext %{{.*}})
+
+union tu_int { int a; } __attribute__((transparent_union));
+union tu_int pass_tu_int(union tu_int arg) { return arg; }
+// CHECK-LABEL: define{{.*}} void @pass_tu_int(ptr dead_on_unwind noalias writable sret(%union.tu_int) align 4 %{{.*}}, i32 signext %{{.*}})
+
+union tu_long { long a; } __attribute__((transparent_union));
+union tu_long pass_tu_long(union tu_long arg) { return arg; }
+// CHECK-LABEL: define{{.*}} void @pass_tu_long(ptr dead_on_unwind noalias writable sret(%union.tu_long) align 8 %{{.*}}, i64 %{{.*}})
+
+union tu_ptr { void *a; } __attribute__((transparent_union));
+union tu_ptr pass_tu_ptr(union tu_ptr arg) { return arg; }
+// CHECK-LABEL: define{{.*}} void @pass_tu_ptr(ptr dead_on_unwind noalias writable sret(%union.tu_ptr) align 8 %{{.*}}, ptr %{{.*}})
+
+
 // Accessing variable argument lists
 
 int va_int(__builtin_va_list l) { return __builtin_va_arg(l, int); }

From 0c4023ae3b64c54ff51947e9776aee0e963c5635 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 18 Jul 2024 08:47:06 -0700
Subject: [PATCH 038/486] [RISCV] Use Root instead of N throughout the worklist
 loop in combineBinOp_VLToVWBinOp_VL. (#99416)

We were only checking that the node from the worklist is a supported
root. We weren't checking the strategy or any of its operands unless it
was the original node. For any other node, we just rechecked the
original node's strategy and operands.

The effect of this is that we don't do all of the transformations at
once. Instead, when there were multiple possible nodes to transform we
would only do them as each node was visited by the main DAG combine
worklist.

The test shows a case where we widened an instruction without removing
all of the uses of the vsext. The sext is shared by one node that shares
another sext node with the root another node that doesn't share anything
with the root.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 10 ++---
 .../CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll  | 38 +++++++++++++++++++
 2 files changed, 43 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 21193ebe1eb94..e938454b8e642 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -14998,8 +14998,8 @@ static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N,
     if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
       return SDValue();
 
-    NodeExtensionHelper LHS(N, 0, DAG, Subtarget);
-    NodeExtensionHelper RHS(N, 1, DAG, Subtarget);
+    NodeExtensionHelper LHS(Root, 0, DAG, Subtarget);
+    NodeExtensionHelper RHS(Root, 1, DAG, Subtarget);
     auto AppendUsersIfNeeded = [&Worklist,
                                 &Inserted](const NodeExtensionHelper &Op) {
       if (Op.needToPromoteOtherUsers()) {
@@ -15016,18 +15016,18 @@ static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N,
       return SDValue();
 
     SmallVector<NodeExtensionHelper::CombineToTry> FoldingStrategies =
-        NodeExtensionHelper::getSupportedFoldings(N);
+        NodeExtensionHelper::getSupportedFoldings(Root);
 
     assert(!FoldingStrategies.empty() && "Nothing to be folded");
     bool Matched = false;
     for (int Attempt = 0;
-         (Attempt != 1 + NodeExtensionHelper::isCommutative(N)) && !Matched;
+         (Attempt != 1 + NodeExtensionHelper::isCommutative(Root)) && !Matched;
          ++Attempt) {
 
       for (NodeExtensionHelper::CombineToTry FoldingStrategy :
            FoldingStrategies) {
         std::optional<CombineResult> Res =
-            FoldingStrategy(N, LHS, RHS, DAG, Subtarget);
+            FoldingStrategy(Root, LHS, RHS, DAG, Subtarget);
         if (Res) {
           Matched = true;
           CombinesToApply.push_back(*Res);
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
index 9d63b8f31a3e8..feb0178569bc7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
@@ -882,3 +882,41 @@ define <2 x i64> @vwmul_vx_v2i64_i64(ptr %x, ptr %y) {
   %g = mul <2 x i64> %e, %f
   ret <2 x i64> %g
 }
+
+define <2 x i16> @vwmul_v2i16_multiuse(ptr %x, ptr %y, ptr %z, ptr %w) {
+; CHECK-LABEL: vwmul_v2i16_multiuse:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vle8.v v9, (a2)
+; CHECK-NEXT:    vle8.v v10, (a3)
+; CHECK-NEXT:    vle8.v v11, (a0)
+; CHECK-NEXT:    vsext.vf2 v12, v8
+; CHECK-NEXT:    vsext.vf2 v8, v9
+; CHECK-NEXT:    vsext.vf2 v9, v10
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vwmul.vv v13, v11, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmul.vv v9, v12, v9
+; CHECK-NEXT:    vdivu.vv v8, v12, v8
+; CHECK-NEXT:    vor.vv v9, v13, v9
+; CHECK-NEXT:    vor.vv v8, v9, v8
+; CHECK-NEXT:    ret
+  %a = load <2 x i8>, ptr %x
+  %b = load <2 x i8>, ptr %y
+  %c = load <2 x i8>, ptr %z
+  %d = load <2 x i8>, ptr %w
+
+  %as = sext <2 x i8> %a to <2 x i16>
+  %bs = sext <2 x i8> %b to <2 x i16>
+  %cs = sext <2 x i8> %c to <2 x i16>
+  %ds = sext <2 x i8> %d to <2 x i16>
+
+  %e = mul <2 x i16> %as, %ds
+  %f = mul <2 x i16> %bs, %ds ; shares 1 use with %e
+  %g = udiv <2 x i16> %bs, %cs ; shares 1 use with %f, and no uses with %e
+
+  %h = or <2 x i16> %e, %f
+  %i = or <2 x i16> %h, %g
+  ret <2 x i16> %i
+}

From 342bd4b89355c27203b5f1abd8c43de6b01aba14 Mon Sep 17 00:00:00 2001
From: Ben Langmuir <blangmuir@apple.com>
Date: Thu, 18 Jul 2024 08:54:43 -0700
Subject: [PATCH 039/486] [orc] Add the name of static archives to the name of
 their member objects (#99407)

Changes "MyObj.o" to "/path/to/libMyLib.a(MyObj.o)".

This allows us to differentiate between objects that have the same
basename but came from different archives. It also fixes a bug where if
two such objects were both linked and both have initializer sections
their initializer symbol would cause a duplicate symbol error.

rdar://131782514
---
 .../llvm/ExecutionEngine/Orc/ExecutionUtils.h |  1 +
 .../ExecutionEngine/Orc/ExecutionUtils.cpp    | 14 +++-
 .../MachO_archive_two_objects_same_name.s     | 64 +++++++++++++++++++
 3 files changed, 78 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/ExecutionEngine/JITLink/x86-64/MachO_archive_two_objects_same_name.s

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
index ed30a792e9e9c..f997faf1ebcb0 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
@@ -328,6 +328,7 @@ class StaticLibraryDefinitionGenerator : public DefinitionGenerator {
   std::unique_ptr<MemoryBuffer> ArchiveBuffer;
   std::unique_ptr<object::Archive> Archive;
   DenseMap<SymbolStringPtr, MemoryBufferRef> ObjectFilesMap;
+  BumpPtrAllocator ObjFileNameStorage;
 };
 
 /// A utility class to create COFF dllimport GOT symbols (__imp_*) and PLT
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index 8a5986c1b88b1..c1a193f6a2802 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -17,6 +17,7 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Object/MachOUniversal.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/StringSaver.h"
 #include "llvm/Target/TargetMachine.h"
 #include <string>
 
@@ -422,6 +423,7 @@ Error StaticLibraryDefinitionGenerator::buildObjectFilesMap() {
   DenseMap<uint64_t, MemoryBufferRef> MemoryBuffers;
   DenseSet<uint64_t> Visited;
   DenseSet<uint64_t> Excluded;
+  StringSaver FileNames(ObjFileNameStorage);
   for (auto &S : Archive->symbols()) {
     StringRef SymName = S.getName();
     auto Member = S.getMember();
@@ -438,7 +440,17 @@ Error StaticLibraryDefinitionGenerator::buildObjectFilesMap() {
         Excluded.insert(DataOffset);
         continue;
       }
-      MemoryBuffers[DataOffset] = (*Child)->getMemoryBufferRef();
+
+      // Give members of the archive a name that contains the archive path so
+      // that they can be differentiated from a member with the same name in a
+      // different archive. This also ensure initializer symbols names will be
+      // unique within a JITDylib.
+      StringRef FullName = FileNames.save(Archive->getFileName() + "(" +
+                                          (*Child)->getFileName() + ")");
+      MemoryBufferRef MemBuffer((*Child)->getMemoryBufferRef().getBuffer(),
+                                FullName);
+
+      MemoryBuffers[DataOffset] = MemBuffer;
     }
     if (!Excluded.count(DataOffset))
       ObjectFilesMap[L.getExecutionSession().intern(SymName)] =
diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/MachO_archive_two_objects_same_name.s b/llvm/test/ExecutionEngine/JITLink/x86-64/MachO_archive_two_objects_same_name.s
new file mode 100644
index 0000000000000..f2bd64f02ca80
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/x86-64/MachO_archive_two_objects_same_name.s
@@ -0,0 +1,64 @@
+# Check that the generated __inits symbol name does not clash between objects
+# with the same base name in two different static archives. Otherwise we get a
+# duplicate symbol error.
+
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: split-file %s %t
+
+# RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
+# RUN:   -o %t/dir1/myobj.o %t/dir1/myobj.s
+# RUN: llvm-ar crs %t/libmyobj1.a %t/dir1/myobj.o
+
+# RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
+# RUN:   -o %t/dir2/myobj.o %t/dir2/myobj.s
+# RUN: llvm-ar crs %t/libmyobj2.a %t/dir2/myobj.o
+
+# RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
+# RUN:   -o %t/main.o %t/main.s
+
+# RUN: llvm-jitlink -noexec %t/main.o -lmyobj1 -lmyobj2 -L%t
+
+#--- dir1/myobj.s
+        .section        __TEXT,__text,regular,pure_instructions
+        .build_version macos, 15, 0     sdk_version 15, 0
+        .globl  _myobj1
+        .p2align        4, 0x90
+_myobj1:                                     ## @f
+        retq
+
+        .section        __DATA,__mod_init_func,mod_init_funcs
+        .p2align        3, 0x0
+        .quad   _myobj1
+
+        .subsections_via_symbols
+
+#--- dir2/myobj.s
+        .section        __TEXT,__text,regular,pure_instructions
+        .build_version macos, 15, 0     sdk_version 15, 0
+        .globl  _myobj2
+        .p2align        4, 0x90
+_myobj2:                                     ## @f
+        retq
+
+        .section        __DATA,__mod_init_func,mod_init_funcs
+        .p2align        3, 0x0
+        .quad   _myobj2
+
+        .subsections_via_symbols
+
+#--- main.s
+
+        .section  __TEXT,__text,regular,pure_instructions
+
+        .globl  _main
+        .p2align  4, 0x90
+_main:
+        pushq   %rbp
+        movq    %rsp, %rbp
+        callq   _myobj1
+        callq   _myobj2
+        xorl    %eax, %eax
+        popq    %rbp
+        retq
+
+        .subsections_via_symbols

From 10627d20044cb13d3fa60a3bce31d37edb3a591f Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 18 Jul 2024 09:02:17 -0700
Subject: [PATCH 040/486] Revert "[RISCV] Use Root instead of N throughout the
 worklist loop in combineBinOp_VLToVWBinOp_VL. (#99416)"

This reverts commit 0c4023ae3b64c54ff51947e9776aee0e963c5635.

I messed up re-generating the test after the change.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 10 ++---
 .../CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll  | 38 -------------------
 2 files changed, 5 insertions(+), 43 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e938454b8e642..21193ebe1eb94 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -14998,8 +14998,8 @@ static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N,
     if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
       return SDValue();
 
-    NodeExtensionHelper LHS(Root, 0, DAG, Subtarget);
-    NodeExtensionHelper RHS(Root, 1, DAG, Subtarget);
+    NodeExtensionHelper LHS(N, 0, DAG, Subtarget);
+    NodeExtensionHelper RHS(N, 1, DAG, Subtarget);
     auto AppendUsersIfNeeded = [&Worklist,
                                 &Inserted](const NodeExtensionHelper &Op) {
       if (Op.needToPromoteOtherUsers()) {
@@ -15016,18 +15016,18 @@ static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N,
       return SDValue();
 
     SmallVector<NodeExtensionHelper::CombineToTry> FoldingStrategies =
-        NodeExtensionHelper::getSupportedFoldings(Root);
+        NodeExtensionHelper::getSupportedFoldings(N);
 
     assert(!FoldingStrategies.empty() && "Nothing to be folded");
     bool Matched = false;
     for (int Attempt = 0;
-         (Attempt != 1 + NodeExtensionHelper::isCommutative(Root)) && !Matched;
+         (Attempt != 1 + NodeExtensionHelper::isCommutative(N)) && !Matched;
          ++Attempt) {
 
       for (NodeExtensionHelper::CombineToTry FoldingStrategy :
            FoldingStrategies) {
         std::optional<CombineResult> Res =
-            FoldingStrategy(Root, LHS, RHS, DAG, Subtarget);
+            FoldingStrategy(N, LHS, RHS, DAG, Subtarget);
         if (Res) {
           Matched = true;
           CombinesToApply.push_back(*Res);
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
index feb0178569bc7..9d63b8f31a3e8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
@@ -882,41 +882,3 @@ define <2 x i64> @vwmul_vx_v2i64_i64(ptr %x, ptr %y) {
   %g = mul <2 x i64> %e, %f
   ret <2 x i64> %g
 }
-
-define <2 x i16> @vwmul_v2i16_multiuse(ptr %x, ptr %y, ptr %z, ptr %w) {
-; CHECK-LABEL: vwmul_v2i16_multiuse:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a1)
-; CHECK-NEXT:    vle8.v v9, (a2)
-; CHECK-NEXT:    vle8.v v10, (a3)
-; CHECK-NEXT:    vle8.v v11, (a0)
-; CHECK-NEXT:    vsext.vf2 v12, v8
-; CHECK-NEXT:    vsext.vf2 v8, v9
-; CHECK-NEXT:    vsext.vf2 v9, v10
-; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
-; CHECK-NEXT:    vwmul.vv v13, v11, v10
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vmul.vv v9, v12, v9
-; CHECK-NEXT:    vdivu.vv v8, v12, v8
-; CHECK-NEXT:    vor.vv v9, v13, v9
-; CHECK-NEXT:    vor.vv v8, v9, v8
-; CHECK-NEXT:    ret
-  %a = load <2 x i8>, ptr %x
-  %b = load <2 x i8>, ptr %y
-  %c = load <2 x i8>, ptr %z
-  %d = load <2 x i8>, ptr %w
-
-  %as = sext <2 x i8> %a to <2 x i16>
-  %bs = sext <2 x i8> %b to <2 x i16>
-  %cs = sext <2 x i8> %c to <2 x i16>
-  %ds = sext <2 x i8> %d to <2 x i16>
-
-  %e = mul <2 x i16> %as, %ds
-  %f = mul <2 x i16> %bs, %ds ; shares 1 use with %e
-  %g = udiv <2 x i16> %bs, %cs ; shares 1 use with %f, and no uses with %e
-
-  %h = or <2 x i16> %e, %f
-  %i = or <2 x i16> %h, %g
-  ret <2 x i16> %i
-}

From 0ce11a1a763d46e4afe678f3f94a1932c1dcfe5d Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 18 Jul 2024 18:04:19 +0200
Subject: [PATCH 041/486] [libc++] Add a release note about C++03 being frozen
 after LLVM 21 (#95894)

Co-authored-by: Louis Dionne <ldionne.2@gmail.com>
---
 libcxx/docs/ReleaseNotes/19.rst | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 36cb23dfde6c9..624550f998858 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -169,8 +169,13 @@ LLVM 20
 
 LLVM 21
 ~~~~~~~
-TODO
 
+- The status of the C++03 implementation will be frozen after the LLVM 21 release. This means that starting in LLVM 22, non-critical bug fixes may not be back-ported
+  to C++03, including LWG issues. C++03 is a legacy platform, where most projects are no longer actively maintained. To
+  reduce the amount of fixes required to keep such legacy projects compiling with up-to-date toolchains, libc++ will aim to freeze the status of the headers in C++03 mode to avoid unintended breaking changes.
+  See https://discourse.llvm.org/t/rfc-freezing-c-03-headers-in-libc for more details.
+
+  If you are using C++03 in your project, you should consider moving to a newer version of the Standard to get the most out of libc++.
 
 ABI Affecting Changes
 ---------------------

From 2bf91db0c743f041c9f83609399f75654c07445a Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 18 Jul 2024 18:05:06 +0200
Subject: [PATCH 042/486] [libc++] Use char_traits::copy while inserting when
 possible (#97201)

This reduces the number of asm lines from 707 to 519 for this snippet:
```c++
auto test(std::string& str, const char* begin, const char* end) {
  str.insert(str.begin(), begin, end);
}
```
While that's not a performance metric, I've never seen a use of `memcpy`
result in a performance regression for any realistic usage.
---
 libcxx/include/string | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/libcxx/include/string b/libcxx/include/string
index 90394e9edbe83..54e9d8990c220 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -1870,6 +1870,23 @@ private:
   template <class _Iterator, class _Sentinel>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
 
+  // Copy [__first, __last) into [__dest, __dest + (__last - __first)). Assumes that the ranges don't overlap.
+  template <class _ForwardIter, class _Sent>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static value_type*
+  __copy_non_overlapping_range(_ForwardIter __first, _Sent __last, value_type* __dest) {
+#ifndef _LIBCPP_CXX03_LANG
+    if constexpr (__libcpp_is_contiguous_iterator<_ForwardIter>::value &&
+                  is_same<value_type, __iter_value_type<_ForwardIter>>::value && is_same<_ForwardIter, _Sent>::value) {
+      traits_type::copy(__dest, std::__to_address(__first), __last - __first);
+      return __dest + (__last - __first);
+    }
+#endif
+
+    for (; __first != __last; ++__first)
+      traits_type::assign(*__dest++, *__first);
+    return __dest;
+  }
+
   template <class _ForwardIterator, class _Sentinel>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 iterator
   __insert_from_safe_copy(size_type __n, size_type __ip, _ForwardIterator __first, _Sentinel __last) {
@@ -1889,8 +1906,7 @@ private:
     __sz += __n;
     __set_size(__sz);
     traits_type::assign(__p[__sz], value_type());
-    for (__p += __ip; __first != __last; ++__p, ++__first)
-      traits_type::assign(*__p, *__first);
+    __copy_non_overlapping_range(__first, __last, __p + __ip);
 
     return begin() + __ip;
   }
@@ -2405,9 +2421,8 @@ basic_string<_CharT, _Traits, _Allocator>::__init_with_size(_InputIterator __fir
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   try {
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    for (; __first != __last; ++__first, (void)++__p)
-      traits_type::assign(*__p, *__first);
-    traits_type::assign(*__p, value_type());
+    auto __end = __copy_non_overlapping_range(__first, __last, std::__to_address(__p));
+    traits_type::assign(*__end, value_type());
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     if (__is_long())
@@ -2873,10 +2888,8 @@ basic_string<_CharT, _Traits, _Allocator>::append(_ForwardIterator __first, _For
       if (__cap - __sz < __n)
         __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
       __annotate_increase(__n);
-      pointer __p = __get_pointer() + __sz;
-      for (; __first != __last; ++__p, (void)++__first)
-        traits_type::assign(*__p, *__first);
-      traits_type::assign(*__p, value_type());
+      auto __end = __copy_non_overlapping_range(__first, __last, std::__to_address(__get_pointer() + __sz));
+      traits_type::assign(*__end, value_type());
       __set_size(__sz + __n);
     } else {
       const basic_string __temp(__first, __last, __alloc());

From 77ac07444d32668d5826ef27c24180fb10425213 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 18 Jul 2024 09:10:52 -0700
Subject: [PATCH 043/486] Re-commit "[RISCV] Use Root instead of N throughout
 the worklist loop in combineBinOp_VLToVWBinOp_VL. (#99416)"

With correct test update.

Original message:

We were only checking that the node from the worklist is a supported
root. We weren't checking the strategy or any of its operands unless it
was the original node. For any other node, we just rechecked the
original node's strategy and operands.

The effect of this is that we don't do all of the transformations at
once. Instead, when there were multiple possible nodes to transform we
would only do them as each node was visited by the main DAG combine
worklist.

The test shows a case where we widened an instruction without removing
all of the uses of the vsext. The sext is shared by one node that shares
another sext node with the root another node that doesn't share anything
with the root.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 10 ++---
 .../CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll  | 37 +++++++++++++++++++
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 21193ebe1eb94..e938454b8e642 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -14998,8 +14998,8 @@ static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N,
     if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
       return SDValue();
 
-    NodeExtensionHelper LHS(N, 0, DAG, Subtarget);
-    NodeExtensionHelper RHS(N, 1, DAG, Subtarget);
+    NodeExtensionHelper LHS(Root, 0, DAG, Subtarget);
+    NodeExtensionHelper RHS(Root, 1, DAG, Subtarget);
     auto AppendUsersIfNeeded = [&Worklist,
                                 &Inserted](const NodeExtensionHelper &Op) {
       if (Op.needToPromoteOtherUsers()) {
@@ -15016,18 +15016,18 @@ static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N,
       return SDValue();
 
     SmallVector<NodeExtensionHelper::CombineToTry> FoldingStrategies =
-        NodeExtensionHelper::getSupportedFoldings(N);
+        NodeExtensionHelper::getSupportedFoldings(Root);
 
     assert(!FoldingStrategies.empty() && "Nothing to be folded");
     bool Matched = false;
     for (int Attempt = 0;
-         (Attempt != 1 + NodeExtensionHelper::isCommutative(N)) && !Matched;
+         (Attempt != 1 + NodeExtensionHelper::isCommutative(Root)) && !Matched;
          ++Attempt) {
 
       for (NodeExtensionHelper::CombineToTry FoldingStrategy :
            FoldingStrategies) {
         std::optional<CombineResult> Res =
-            FoldingStrategy(N, LHS, RHS, DAG, Subtarget);
+            FoldingStrategy(Root, LHS, RHS, DAG, Subtarget);
         if (Res) {
           Matched = true;
           CombinesToApply.push_back(*Res);
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
index 9d63b8f31a3e8..97c7f101c2582 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
@@ -882,3 +882,40 @@ define <2 x i64> @vwmul_vx_v2i64_i64(ptr %x, ptr %y) {
   %g = mul <2 x i64> %e, %f
   ret <2 x i64> %g
 }
+
+define <2 x i16> @vwmul_v2i16_multiuse(ptr %x, ptr %y, ptr %z, ptr %w) {
+; CHECK-LABEL: vwmul_v2i16_multiuse:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vle8.v v9, (a1)
+; CHECK-NEXT:    vle8.v v10, (a2)
+; CHECK-NEXT:    vle8.v v11, (a3)
+; CHECK-NEXT:    vsext.vf2 v12, v8
+; CHECK-NEXT:    vsext.vf2 v8, v9
+; CHECK-NEXT:    vsext.vf2 v9, v10
+; CHECK-NEXT:    vsext.vf2 v10, v11
+; CHECK-NEXT:    vmul.vv v11, v12, v10
+; CHECK-NEXT:    vmul.vv v10, v8, v10
+; CHECK-NEXT:    vdivu.vv v8, v8, v9
+; CHECK-NEXT:    vor.vv v9, v11, v10
+; CHECK-NEXT:    vor.vv v8, v9, v8
+; CHECK-NEXT:    ret
+  %a = load <2 x i8>, ptr %x
+  %b = load <2 x i8>, ptr %y
+  %c = load <2 x i8>, ptr %z
+  %d = load <2 x i8>, ptr %w
+
+  %as = sext <2 x i8> %a to <2 x i16>
+  %bs = sext <2 x i8> %b to <2 x i16>
+  %cs = sext <2 x i8> %c to <2 x i16>
+  %ds = sext <2 x i8> %d to <2 x i16>
+
+  %e = mul <2 x i16> %as, %ds
+  %f = mul <2 x i16> %bs, %ds ; shares 1 use with %e
+  %g = udiv <2 x i16> %bs, %cs ; shares 1 use with %f, and no uses with %e
+
+  %h = or <2 x i16> %e, %f
+  %i = or <2 x i16> %h, %g
+  ret <2 x i16> %i
+}

From c0c157a51832a2c7bbd09a449e33cc94d7747abf Mon Sep 17 00:00:00 2001
From: Sayhaan Siddiqui <49014204+sayhaan@users.noreply.github.com>
Date: Thu, 18 Jul 2024 09:24:46 -0700
Subject: [PATCH 044/486] [BOLT][DWARF][NFC] Remove DWO ranges base (#99284)

Removes getters and setters for DWO ranges base due to it not being
used.
---
 bolt/include/bolt/Rewrite/DWARFRewriter.h | 9 ---------
 bolt/lib/Rewrite/DWARFRewriter.cpp        | 1 -
 2 files changed, 10 deletions(-)

diff --git a/bolt/include/bolt/Rewrite/DWARFRewriter.h b/bolt/include/bolt/Rewrite/DWARFRewriter.h
index abd18b56113b6..b798c5b76fc28 100644
--- a/bolt/include/bolt/Rewrite/DWARFRewriter.h
+++ b/bolt/include/bolt/Rewrite/DWARFRewriter.h
@@ -95,9 +95,6 @@ class DWARFRewriter {
 
   std::mutex LocListDebugInfoPatchesMutex;
 
-  /// Dwo id specific its RangesBase.
-  std::unordered_map<uint64_t, uint64_t> DwoRangesBase;
-
   std::unordered_map<DWARFUnit *, uint64_t> LineTablePatchMap;
   std::unordered_map<const DWARFUnit *, uint64_t> TypeUnitRelocMap;
 
@@ -191,12 +188,6 @@ class DWARFRewriter {
   /// Update stmt_list for CUs based on the new .debug_line \p Layout.
   void updateLineTableOffsets(const MCAssembler &Asm);
 
-  uint64_t getDwoRangesBase(uint64_t DWOId) { return DwoRangesBase[DWOId]; }
-
-  void setDwoRangesBase(uint64_t DWOId, uint64_t RangesBase) {
-    DwoRangesBase[DWOId] = RangesBase;
-  }
-
   using OverriddenSectionsMap = std::unordered_map<DWARFSectionKind, StringRef>;
   /// Output .dwo files.
   void writeDWOFiles(DWARFUnit &, const OverriddenSectionsMap &,
diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp
index 042c39a574561..4ba6344925856 100644
--- a/bolt/lib/Rewrite/DWARFRewriter.cpp
+++ b/bolt/lib/Rewrite/DWARFRewriter.cpp
@@ -719,7 +719,6 @@ void DWARFRewriter::updateDebugInfo() {
       } else {
         TempRangesSectionWriter = LegacyRangesWritersByCU[*DWOId].get();
         RangesBase = RangesSectionWriter->getSectionOffset();
-        setDwoRangesBase(*DWOId, *RangesBase);
       }
 
       updateUnitDebugInfo(*(*SplitCU), DWODIEBuilder, DebugLocDWoWriter,

From 8c8e0ddae96882247717b8ae1739abcf09726eab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Thu, 18 Jul 2024 18:23:45 +0200
Subject: [PATCH 045/486] [clang][Interp][test] Use fixed triple in cxx11 test

This uses 'long', which has a different size on Windows. The test
I copied this from also uses x86_64-linux.

This should fix the bot:
https://lab.llvm.org/buildbot/#/builders/81/builds/853
---
 clang/test/AST/Interp/cxx11.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/test/AST/Interp/cxx11.cpp b/clang/test/AST/Interp/cxx11.cpp
index c0b88f0e567e0..92ab9b605f30d 100644
--- a/clang/test/AST/Interp/cxx11.cpp
+++ b/clang/test/AST/Interp/cxx11.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=both,expected -std=c++11 %s
-// RUN: %clang_cc1 -verify=both,ref -std=c++11 %s
+// RUN: %clang_cc1 -triple x86_64-linux -fexperimental-new-constant-interpreter -verify=both,expected -std=c++11 %s
+// RUN: %clang_cc1 -triple x86_64-linux -verify=both,ref -std=c++11 %s
 
 namespace IntOrEnum {
   const int k = 0;

From fe04aafe6c27f32ad4ba38e552d06d14431cb2de Mon Sep 17 00:00:00 2001
From: Uday Bondhugula <uday@polymagelabs.com>
Date: Thu, 18 Jul 2024 22:06:44 +0530
Subject: [PATCH 046/486] [MLIR][Affine] NFC. Expose affine loop tiling
 validity utility (#99459)

Move the utility to check for the validity of tiling affine loop nests
to affine loop utils and expose for users outside the loop tiling pass
or downstream users.
---
 .../Dialect/Affine/Analysis/LoopAnalysis.h    | 10 +++
 .../Dialect/Affine/Analysis/LoopAnalysis.cpp  | 58 ++++++++++++++++
 .../Dialect/Affine/Transforms/LoopTiling.cpp  | 67 +------------------
 .../Dialect/Affine/loop-tiling-validity.mlir  |  2 +-
 4 files changed, 71 insertions(+), 66 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Affine/Analysis/LoopAnalysis.h b/mlir/include/mlir/Dialect/Affine/Analysis/LoopAnalysis.h
index 7b92b930fb5f5..ed3c21d952a01 100644
--- a/mlir/include/mlir/Dialect/Affine/Analysis/LoopAnalysis.h
+++ b/mlir/include/mlir/Dialect/Affine/Analysis/LoopAnalysis.h
@@ -109,6 +109,16 @@ bool isVectorizableLoopBody(AffineForOp loop, int *memRefDim,
 // the support.
 bool isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts);
 
+/// Checks whether hyper-rectangular loop tiling of the nest represented by
+/// `loops` is valid. The validity condition is from Irigoin and Triolet,
+/// which states that two tiles cannot depend on each other. We simplify such
+/// condition to just checking whether there is any negative dependence
+/// direction, since we have the prior knowledge that the tiling results will be
+/// hyper-rectangles, which are scheduled in the lexicographically increasing
+/// order on the vector of loop indices. This function will return failure when
+/// any dependence component is negative along any of `loops`.
+bool isTilingValid(ArrayRef<AffineForOp> loops);
+
 } // namespace affine
 } // namespace mlir
 
diff --git a/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp b/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
index 82ba8fc5ccbc1..411b5efb36cab 100644
--- a/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
@@ -23,6 +23,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Debug.h"
 #include <numeric>
 #include <optional>
 #include <type_traits>
@@ -30,6 +31,8 @@
 using namespace mlir;
 using namespace mlir::affine;
 
+#define DEBUG_TYPE "affine-loop-analysis"
+
 /// Returns the trip count of the loop as an affine expression if the latter is
 /// expressible as an affine expression, and nullptr otherwise. The trip count
 /// expression is simplified before returning. This method only utilizes map
@@ -390,3 +393,58 @@ bool mlir::affine::isOpwiseShiftValid(AffineForOp forOp,
   }
   return true;
 }
+
+bool mlir::affine::isTilingValid(ArrayRef<AffineForOp> loops) {
+  assert(!loops.empty() && "no original loops provided");
+
+  // We first find out all dependences we intend to check.
+  SmallVector<Operation *, 8> loadAndStoreOps;
+  loops[0]->walk([&](Operation *op) {
+    if (isa<AffineReadOpInterface, AffineWriteOpInterface>(op))
+      loadAndStoreOps.push_back(op);
+  });
+
+  unsigned numOps = loadAndStoreOps.size();
+  unsigned numLoops = loops.size();
+  for (unsigned d = 1; d <= numLoops + 1; ++d) {
+    for (unsigned i = 0; i < numOps; ++i) {
+      Operation *srcOp = loadAndStoreOps[i];
+      MemRefAccess srcAccess(srcOp);
+      for (unsigned j = 0; j < numOps; ++j) {
+        Operation *dstOp = loadAndStoreOps[j];
+        MemRefAccess dstAccess(dstOp);
+
+        SmallVector<DependenceComponent, 2> depComps;
+        DependenceResult result = checkMemrefAccessDependence(
+            srcAccess, dstAccess, d, /*dependenceConstraints=*/nullptr,
+            &depComps);
+
+        // Skip if there is no dependence in this case.
+        if (!hasDependence(result))
+          continue;
+
+        // Check whether there is any negative direction vector in the
+        // dependence components found above, which means that dependence is
+        // violated by the default hyper-rect tiling method.
+        LLVM_DEBUG(llvm::dbgs() << "Checking whether tiling legality violated "
+                                   "for dependence at depth: "
+                                << Twine(d) << " between:\n";);
+        LLVM_DEBUG(srcAccess.opInst->dump());
+        LLVM_DEBUG(dstAccess.opInst->dump());
+        for (const DependenceComponent &depComp : depComps) {
+          if (depComp.lb.has_value() && depComp.ub.has_value() &&
+              *depComp.lb < *depComp.ub && *depComp.ub < 0) {
+            LLVM_DEBUG(llvm::dbgs()
+                       << "Dependence component lb = " << Twine(*depComp.lb)
+                       << " ub = " << Twine(*depComp.ub)
+                       << " is negative  at depth: " << Twine(d)
+                       << " and thus violates the legality rule.\n");
+            return false;
+          }
+        }
+      }
+    }
+  }
+
+  return true;
+}
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
index 2650a06d198ea..c8400dfe8cd5c 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
@@ -93,69 +93,6 @@ static void adjustToDivisorsOfTripCounts(ArrayRef<AffineForOp> band,
   }
 }
 
-/// Checks whether hyper-rectangular loop tiling of the nest represented by
-/// `origLoops` is valid. The validity condition is from Irigoin and Triolet,
-/// which states that two tiles cannot depend on each other. We simplify such
-/// condition to just checking whether there is any negative dependence
-/// direction, since we have the prior knowledge that the tiling results will be
-/// hyper-rectangles, which are scheduled in the lexicographically increasing
-/// order on the vector of loop indices. This function will return failure when
-/// any dependence component is negative along any of `origLoops`.
-static bool checkTilingLegality(MutableArrayRef<AffineForOp> origLoops) {
-  assert(!origLoops.empty() && "no original loops provided");
-
-  // We first find out all dependences we intend to check.
-  SmallVector<Operation *, 8> loadAndStoreOps;
-  origLoops[0]->walk([&](Operation *op) {
-    if (isa<AffineReadOpInterface, AffineWriteOpInterface>(op))
-      loadAndStoreOps.push_back(op);
-  });
-
-  unsigned numOps = loadAndStoreOps.size();
-  unsigned numLoops = origLoops.size();
-  for (unsigned d = 1; d <= numLoops + 1; ++d) {
-    for (unsigned i = 0; i < numOps; ++i) {
-      Operation *srcOp = loadAndStoreOps[i];
-      MemRefAccess srcAccess(srcOp);
-      for (unsigned j = 0; j < numOps; ++j) {
-        Operation *dstOp = loadAndStoreOps[j];
-        MemRefAccess dstAccess(dstOp);
-
-        SmallVector<DependenceComponent, 2> depComps;
-        DependenceResult result = checkMemrefAccessDependence(
-            srcAccess, dstAccess, d, /*dependenceConstraints=*/nullptr,
-            &depComps);
-
-        // Skip if there is no dependence in this case.
-        if (!hasDependence(result))
-          continue;
-
-        // Check whether there is any negative direction vector in the
-        // dependence components found above, which means that dependence is
-        // violated by the default hyper-rect tiling method.
-        LLVM_DEBUG(llvm::dbgs() << "Checking whether tiling legality violated "
-                                   "for dependence at depth: "
-                                << Twine(d) << " between:\n";);
-        LLVM_DEBUG(srcAccess.opInst->dump(););
-        LLVM_DEBUG(dstAccess.opInst->dump(););
-        for (const DependenceComponent &depComp : depComps) {
-          if (depComp.lb.has_value() && depComp.ub.has_value() &&
-              *depComp.lb < *depComp.ub && *depComp.ub < 0) {
-            LLVM_DEBUG(llvm::dbgs()
-                       << "Dependence component lb = " << Twine(*depComp.lb)
-                       << " ub = " << Twine(*depComp.ub)
-                       << " is negative  at depth: " << Twine(d)
-                       << " and thus violates the legality rule.\n");
-            return false;
-          }
-        }
-      }
-    }
-  }
-
-  return true;
-}
-
 // Returns tile sizes to use. Checks CL options; if none are specified, sets it
 // based on a simple model that looks at the memory footprint and determines
 // tile sizes assuming identity accesses / 1:1 tile size proportional footprint
@@ -242,8 +179,8 @@ void LoopTiling::runOnOperation() {
 
   // Tile each band.
   for (auto &band : bands) {
-    if (!checkTilingLegality(band)) {
-      band.front().emitRemark("tiling code is illegal due to dependences");
+    if (!isTilingValid(band)) {
+      band.front().emitRemark("tiling nest is invalid due to dependences");
       continue;
     }
 
diff --git a/mlir/test/Dialect/Affine/loop-tiling-validity.mlir b/mlir/test/Dialect/Affine/loop-tiling-validity.mlir
index d1b80520ca7fe..e2c3832f695cc 100644
--- a/mlir/test/Dialect/Affine/loop-tiling-validity.mlir
+++ b/mlir/test/Dialect/Affine/loop-tiling-validity.mlir
@@ -34,7 +34,7 @@ func.func @illegal_loop_with_diag_dependence() {
   %A = memref.alloc() : memref<64x64xf32>
 
   affine.for %i = 0 to 64 {
-    // expected-remark@above {{tiling code is illegal due to dependences}}
+    // expected-remark@above {{tiling nest is invalid due to dependences}}
     affine.for %j = 0 to 64 {
       %0 = affine.load %A[%j, %i] : memref<64x64xf32>
       %1 = affine.load %A[%i, %j - 1] : memref<64x64xf32>

From 06ab30b57450694818dbb649dec2a687f44df7f4 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Thu, 18 Jul 2024 09:40:37 -0700
Subject: [PATCH 047/486] [AMDGPU] Constant folding of llvm.amdgcn.trig.preop
 (#98562)

If the parameters(the input and segment select) coming in to
amdgcn.trig.preop intrinsic are compile time constants, we pre-compute
the output of amdgcn.trig.preop on the CPU and replaces the uses with
the computed constant.

This work extends the patch https://reviews.llvm.org/D120150 to make it
a complete coverage.

For the segment select, only src1[4:0] are used. A segment select is
invalid if we are selecting the 53-bit segment beyond the [1200:0] range
of the 2/PI table. 0 is returned when a segment select is not valid.
---
 .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp     |  72 +++++
 .../InstCombine/AMDGPU/amdgcn-intrinsics.ll   | 291 ++++++++++++++----
 2 files changed, 302 insertions(+), 61 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 93bca4402ed23..9197404309663 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1102,6 +1102,78 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
 
     break;
   }
+  case Intrinsic::amdgcn_trig_preop: {
+    // The intrinsic is declared with name mangling, but currently the
+    // instruction only exists for f64
+    if (!II.getType()->isDoubleTy())
+      break;
+
+    Value *Src = II.getArgOperand(0);
+    Value *Segment = II.getArgOperand(1);
+    if (isa<PoisonValue>(Src) || isa<PoisonValue>(Segment))
+      return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
+
+    if (isa<UndefValue>(Src)) {
+      auto *QNaN = ConstantFP::get(
+          II.getType(), APFloat::getQNaN(II.getType()->getFltSemantics()));
+      return IC.replaceInstUsesWith(II, QNaN);
+    }
+
+    const ConstantFP *Csrc = dyn_cast<ConstantFP>(Src);
+    if (!Csrc)
+      break;
+
+    if (II.isStrictFP())
+      break;
+
+    const APFloat &Fsrc = Csrc->getValueAPF();
+    if (Fsrc.isNaN()) {
+      auto *Quieted = ConstantFP::get(II.getType(), Fsrc.makeQuiet());
+      return IC.replaceInstUsesWith(II, Quieted);
+    }
+
+    const ConstantInt *Cseg = dyn_cast<ConstantInt>(Segment);
+    if (!Cseg)
+      break;
+
+    unsigned Exponent = (Fsrc.bitcastToAPInt().getZExtValue() >> 52) & 0x7ff;
+    unsigned SegmentVal = Cseg->getValue().trunc(5).getZExtValue();
+    unsigned Shift = SegmentVal * 53;
+    if (Exponent > 1077)
+      Shift += Exponent - 1077;
+
+    // 2.0/PI table.
+    static const uint32_t TwoByPi[] = {
+        0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, 0xdb629599, 0x3c439041,
+        0xfe5163ab, 0xdebbc561, 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c,
+        0xfe1deb1c, 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41,
+        0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, 0xde05980f,
+        0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, 0x4f463f66, 0x9e5fea2d,
+        0x7527bac7, 0xebe5f17b, 0x3d0739f7, 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08,
+        0x56033046};
+
+    // Return 0 for outbound segment (hardware behavior).
+    unsigned Idx = Shift >> 5;
+    if (Idx + 2 >= std::size(TwoByPi)) {
+      APFloat Zero = APFloat::getZero(II.getType()->getFltSemantics());
+      return IC.replaceInstUsesWith(II, ConstantFP::get(II.getType(), Zero));
+    }
+
+    unsigned BShift = Shift & 0x1f;
+    uint64_t Thi = Make_64(TwoByPi[Idx], TwoByPi[Idx + 1]);
+    uint64_t Tlo = Make_64(TwoByPi[Idx + 2], 0);
+    if (BShift)
+      Thi = (Thi << BShift) | (Tlo >> (64 - BShift));
+    Thi = Thi >> 11;
+    APFloat Result = APFloat((double)Thi);
+
+    int Scale = -53 - Shift;
+    if (Exponent >= 1968)
+      Scale += 128;
+
+    Result = scalbn(Result, Scale, RoundingMode::NearestTiesToEven);
+    return IC.replaceInstUsesWith(II, ConstantFP::get(Src->getType(), Result));
+  }
   case Intrinsic::amdgcn_fmul_legacy: {
     Value *Op0 = II.getArgOperand(0);
     Value *Op1 = II.getArgOperand(1);
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index 59118a172a2bc..9cb79b2644865 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -5608,8 +5608,7 @@ declare float @llvm.amdgcn.trig.preop.f32(float, i32)
 
 define double @trig_preop_constfold_variable_undef_arg(i32 %arg) {
 ; CHECK-LABEL: @trig_preop_constfold_variable_undef_arg(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double undef, i32 [[ARG:%.*]])
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    ret double 0x7FF8000000000000
 ;
   %val = call double @llvm.amdgcn.trig.preop.f64(double undef, i32 %arg)
   ret double %val
@@ -5617,8 +5616,7 @@ define double @trig_preop_constfold_variable_undef_arg(i32 %arg) {
 
 define double @trig_preop_constfold_variable_poison_arg(i32 %arg) {
 ; CHECK-LABEL: @trig_preop_constfold_variable_poison_arg(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double poison, i32 [[ARG:%.*]])
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    ret double poison
 ;
   %val = call double @llvm.amdgcn.trig.preop.f64(double poison, i32 %arg)
   ret double %val
@@ -5635,8 +5633,7 @@ define double @trig_preop_constfold_variable_arg_undef(double %arg) {
 
 define double @trig_preop_constfold_variable_arg_poison(double %arg) {
 ; CHECK-LABEL: @trig_preop_constfold_variable_arg_poison(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double [[ARG:%.*]], i32 poison)
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    ret double poison
 ;
   %val = call double @llvm.amdgcn.trig.preop.f64(double %arg, i32 poison)
   ret double %val
@@ -5653,8 +5650,7 @@ define double @trig_preop_constfold_variable_int(i32 %arg) {
 
 define double @trig_preop_qnan(i32 %arg) {
 ; CHECK-LABEL: @trig_preop_qnan(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0x7FF8000000000000, i32 [[ARG:%.*]])
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    ret double 0x7FF8000000000000
 ;
   %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7FF8000000000000, i32 %arg)
   ret double %val
@@ -5662,8 +5658,7 @@ define double @trig_preop_qnan(i32 %arg) {
 
 define double @trig_preop_snan(i32 %arg) {
 ; CHECK-LABEL: @trig_preop_snan(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0x7FF0000000000001, i32 [[ARG:%.*]])
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    ret double 0x7FF8000000000001
 ;
   %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7FF0000000000001, i32 %arg)
   ret double %val
@@ -5671,8 +5666,7 @@ define double @trig_preop_snan(i32 %arg) {
 
 define double @trig_preop_inf_0() {
 ; CHECK-LABEL: @trig_preop_inf_0(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0x7FF0000000000000, i32 0)
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    ret double 0xB43DD63F5F2F8BD
 ;
   %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7FF0000000000000, i32 0)
   ret double %val
@@ -5680,8 +5674,7 @@ define double @trig_preop_inf_0() {
 
 define double @trig_preop_ninf_0() {
 ; CHECK-LABEL: @trig_preop_ninf_0(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0xFFF0000000000000, i32 0)
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    ret double 0xB43DD63F5F2F8BD
 ;
   %val = call double @llvm.amdgcn.trig.preop.f64(double 0xFFF0000000000000, i32 0)
   ret double %val
@@ -5707,10 +5700,36 @@ define double @trig_preop_variable_args(double %arg0, i32 %arg1) {
 
 define double @trig_preop_constfold() {
 ; CHECK-LABEL: @trig_preop_constfold(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 3.454350e+02, i32 5)
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    ret double 0x394A6EE06DB14ACC
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 3.454350e+02, i32 2)
+  ret double %val
+}
+
+; src1[4:0] <= 21 for segment to be inbound with this exponent of src0.
+define double @trig_preop_constfold_outbound_segment() {
+; CHECK-LABEL: @trig_preop_constfold_outbound_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
 ;
-  %val = call double @llvm.amdgcn.trig.preop.f64(double 3.454350e+02, i32 5)
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 3.454350e+02, i32 22)
+  ret double %val
+}
+
+; Only use src1[4:0], so segment is actually 31 for -1.
+define double @trig_preop_constfold_neg1_segment() {
+; CHECK-LABEL: @trig_preop_constfold_neg1_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 3.454350e+02, i32 -1)
+  ret double %val
+}
+
+; Only use src1[4:0], so segment is actually 0 for -32.
+define double @trig_preop_constfold_neg32_segment() {
+; CHECK-LABEL: @trig_preop_constfold_neg32_segment(
+; CHECK-NEXT:    ret double 0x3FE45F306DC9C882
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 3.454350e+02, i32 -32)
   ret double %val
 }
 
@@ -5723,84 +5742,234 @@ define double @trig_preop_constfold_strictfp() strictfp {
   ret double %val
 }
 
-define double @trig_preop_constfold_0.0__0() {
-; CHECK-LABEL: @trig_preop_constfold_0.0__0(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0.000000e+00, i32 0)
-; CHECK-NEXT:    ret double [[VAL]]
+define double @trig_preop_constfold_exponent0_mantissa0__segment0() {
+; CHECK-LABEL: @trig_preop_constfold_exponent0_mantissa0__segment0(
+; CHECK-NEXT:    ret double 0x3FE45F306DC9C882
 ;
   %val = call double @llvm.amdgcn.trig.preop.f64(double 0.0, i32 0)
   ret double %val
 }
 
-define double @trig_preop_constfold_0.0__1() {
-; CHECK-LABEL: @trig_preop_constfold_0.0__1(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0.000000e+00, i32 1)
-; CHECK-NEXT:    ret double [[VAL]]
+define double @trig_preop_constfold_exponent0_mantissa1__segment0() {
+; CHECK-LABEL: @trig_preop_constfold_exponent0_mantissa1__segment0(
+; CHECK-NEXT:    ret double 0x3FE45F306DC9C882
 ;
-  %val = call double @llvm.amdgcn.trig.preop.f64(double 0.0, i32 1)
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x000FFFFFFFFFFFFF, i32 0)
   ret double %val
 }
 
-define double @trig_preop_constfold_0.0__neg1() {
-; CHECK-LABEL: @trig_preop_constfold_0.0__neg1(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0.000000e+00, i32 -1)
-; CHECK-NEXT:    ret double [[VAL]]
+define double @trig_preop_constfold_exponent0_mantissaX__segment0() {
+; CHECK-LABEL: @trig_preop_constfold_exponent0_mantissaX__segment0(
+; CHECK-NEXT:    ret double 0x3FE45F306DC9C882
 ;
-  %val = call double @llvm.amdgcn.trig.preop.f64(double 0.0, i32 -1)
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x0004A7F09D5F47D4, i32 0)
   ret double %val
 }
 
-define double @trig_preop_constfold_0.0__9999999() {
-; CHECK-LABEL: @trig_preop_constfold_0.0__9999999(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0.000000e+00, i32 9999999)
-; CHECK-NEXT:    ret double [[VAL]]
+define double @trig_preop_constfold_exponent0_mantissa0__segment2() {
+; CHECK-LABEL: @trig_preop_constfold_exponent0_mantissa0__segment2(
+; CHECK-NEXT:    ret double 0x394A6EE06DB14ACC
 ;
-  %val = call double @llvm.amdgcn.trig.preop.f64(double 0.0, i32 9999999)
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0.0, i32 2)
   ret double %val
 }
 
-define double @trig_preop_constfold_0.0__neg999999() {
-; CHECK-LABEL: @trig_preop_constfold_0.0__neg999999(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0.000000e+00, i32 -999999)
-; CHECK-NEXT:    ret double [[VAL]]
+define double @trig_preop_constfold_exponent0_mantissa1__segment2() {
+; CHECK-LABEL: @trig_preop_constfold_exponent0_mantissa1__segment2(
+; CHECK-NEXT:    ret double 0x394A6EE06DB14ACC
 ;
-  %val = call double @llvm.amdgcn.trig.preop.f64(double 0.0, i32 -999999)
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x000FFFFFFFFFFFFF, i32 2)
   ret double %val
 }
 
-define double @trig_preop_constfold_0x0020000000000000_0() {
-; CHECK-LABEL: @trig_preop_constfold_0x0020000000000000_0(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0x10000000000000, i32 0)
-; CHECK-NEXT:    ret double [[VAL]]
+define double @trig_preop_constfold_exponent0_mantissaX__segment2() {
+; CHECK-LABEL: @trig_preop_constfold_exponent0_mantissaX__segment2(
+; CHECK-NEXT:    ret double 0x394A6EE06DB14ACC
 ;
-  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x0010000000000000, i32 0)
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x00094A6EE06DB14A, i32 2)
   ret double %val
 }
 
-define double @trig_preop_constfold_0x001fffffffffffff_0() {
-; CHECK-LABEL: @trig_preop_constfold_0x001fffffffffffff_0(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0xFFFFFFFFFFFFF, i32 0)
-; CHECK-NEXT:    ret double [[VAL]]
+; src1[4:0] <= 21 for segment to be inbound with this exponent of src0.
+define double @trig_preop_constfold_exponent0_mantissa0__outbound_segment() {
+; CHECK-LABEL: @trig_preop_constfold_exponent0_mantissa0__outbound_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
 ;
-  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x000fffffffffffff, i32 0)
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0.0, i32 22)
   ret double %val
 }
 
-define double @trig_preop_constfold_0x8020000000000000_0() {
-; CHECK-LABEL: @trig_preop_constfold_0x8020000000000000_0(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0x8020000000000000, i32 0)
-; CHECK-NEXT:    ret double [[VAL]]
+; src1[4:0] <= 21 for segment to be inbound with this exponent of src0.
+define double @trig_preop_constfold_exponent0_mantissa1__outbound_segment() {
+; CHECK-LABEL: @trig_preop_constfold_exponent0_mantissa1__outbound_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
 ;
-  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x8020000000000000, i32 0)
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x000FFFFFFFFFFFFF, i32 22)
   ret double %val
 }
 
-define double @trig_preop_constfold_0x801fffffffffffff_0() {
-; CHECK-LABEL: @trig_preop_constfold_0x801fffffffffffff_0(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.trig.preop.f64(double 0x801FFFFFFFFFFFFF, i32 0)
-; CHECK-NEXT:    ret double [[VAL]]
+; src1[4:0] <= 21 for segment to be inbound with this exponent of src0.
+define double @trig_preop_constfold_exponent0_mantissaX__outbound_segment() {
+; CHECK-LABEL: @trig_preop_constfold_exponent0_mantissaX__outbound_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x000A6EE06DB14ACC, i32 22)
+  ret double %val
+}
+
+; 1607 = 1077 + 10 * 53
+define double @trig_preop_constfold_exponent1607_mantissa0__segment0() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1607_mantissa0__segment0(
+; CHECK-NEXT:    ret double 0x1EC8135A2FBF209C
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x6470000000000000, i32 0)
+  ret double %val
+}
+
+; 1607 = 1077 + 10 * 53
+define double @trig_preop_constfold_exponent1607_mantissa1__segment1() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1607_mantissa1__segment1(
+; CHECK-NEXT:    ret double 0x1EC8135A2FBF209C
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x647FFFFFFFFFFFFF, i32 0)
+  ret double %val
+}
+
+; 1607 = 1077 + 10 * 53
+define double @trig_preop_constfold_exponent1607_mantissaX__segment1() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1607_mantissaX__segment1(
+; CHECK-NEXT:    ret double 0x1EC8135A2FBF209C
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x6471B791D6398353, i32 0)
+  ret double %val
+}
+
+; 1607 = 1077 + 10 * 53
+define double @trig_preop_constfold_exponent1607_mantissa0__segment2() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1607_mantissa0__segment2(
+; CHECK-NEXT:    ret double 0x181272117E2EF7E4
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x6470000000000000, i32 2)
+  ret double %val
+}
+
+; 1607 = 1077 + 10 * 53
+define double @trig_preop_constfold_exponent1607_mantissa1__segment2() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1607_mantissa1__segment2(
+; CHECK-NEXT:    ret double 0x181272117E2EF7E4
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x647FFFFFFFFFFFFF, i32 2)
+  ret double %val
+}
+
+; 1607 = 1077 + 10 * 53
+define double @trig_preop_constfold_exponent1607_mantissaX__segment2() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1607_mantissaX__segment2(
+; CHECK-NEXT:    ret double 0x181272117E2EF7E4
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x647272117E2EF7E4, i32 2)
+  ret double %val
+}
+
+; src1[4:0] <= 11 for segment to be inbound with this exponent of src0.
+define double @trig_preop_constfold_exponent1607_mantissa0__outbound_segment() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1607_mantissa0__outbound_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x6470000000000000, i32 12)
+  ret double %val
+}
+
+; src1[4:0] <= 11 for segment to be inbound with this exponent of src0.
+define double @trig_preop_constfold_exponent1607_mantissa1__outbound_segment() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1607_mantissa1__outbound_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x647FFFFFFFFFFFFF, i32 12)
+  ret double %val
+}
+
+; src1[4:0] <= 11 for segment to be inbound with this exponent of src0.
+define double @trig_preop_constfold_exponent1607_mantissaX__outbound_segment() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1607_mantissaX__outbound_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x647181272117E2EF, i32 12)
+  ret double %val
+}
+
+define double @trig_preop_constfold_exponent1968_mantissa0__segment0() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1968_mantissa0__segment0(
+; CHECK-NEXT:    ret double 0x10374F463F669E5F
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7B00000000000000, i32 0)
+  ret double %val
+}
+
+define double @trig_preop_constfold_exponent1968_mantissa1__segment0() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1968_mantissa1__segment0(
+; CHECK-NEXT:    ret double 0x10374F463F669E5F
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7B0FFFFFFFFFFFFF, i32 0)
+  ret double %val
+}
+
+define double @trig_preop_constfold_exponent1968_mantissax__segment0() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1968_mantissax__segment0(
+; CHECK-NEXT:    ret double 0x10374F463F669E5F
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7B074F463F669E5F, i32 0)
+  ret double %val
+}
+
+define double @trig_preop_constfold_exponent1968_mantissa0__segment2() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1968_mantissa0__segment2(
+; CHECK-NEXT:    ret double 0x98F2F8BD9E839CE
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7B00000000000000, i32 2)
+  ret double %val
+}
+
+define double @trig_preop_constfold_exponent1968_mantissa1__segment2() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1968_mantissa1__segment2(
+; CHECK-NEXT:    ret double 0x98F2F8BD9E839CE
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7B0FFFFFFFFFFFFF, i32 2)
+  ret double %val
+}
+
+define double @trig_preop_constfold_exponent1968_mantissaX__segment2() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1968_mantissaX__segment2(
+; CHECK-NEXT:    ret double 0x98F2F8BD9E839CE
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7B0A2F8BD9E839CE, i32 2)
+  ret double %val
+}
+
+; src1[4:0] <= 4 for segment to be inbound with this exponent of src0.
+define double @trig_preop_constfold_exponent1968_mantissa0__outbound_segment() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1968_mantissa0__outbound_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7B00000000000000, i32 5)
+  ret double %val
+}
+
+; src1[4:0] <= 4 for segment to be inbound with this exponent of src0.
+define double @trig_preop_constfold_exponent1968_mantissa1__outbound_segment() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1968_mantissa1__outbound_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7B0FFFFFFFFFFFFF, i32 5)
+  ret double %val
+}
+
+; src1[4:0] <= 4 for segment to be inbound with this exponent of src0.
+define double @trig_preop_constfold_exponent1968_mantissaX__outbound_segment() {
+; CHECK-LABEL: @trig_preop_constfold_exponent1968_mantissaX__outbound_segment(
+; CHECK-NEXT:    ret double 0.000000e+00
 ;
-  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x801fffffffffffff, i32 0)
+  %val = call double @llvm.amdgcn.trig.preop.f64(double 0x7B0A98F2F8BD9E83, i32 5)
   ret double %val
 }
 

From 371777695fe1b5407753ef2232d1b73014d3e501 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 18 Jul 2024 17:43:08 +0100
Subject: [PATCH 048/486] [LV] Assert uniform recipes don't get predicated for
 when vectorizing.

Add assertion ensuring invariant on construction, split off as suggested
from https://github.com/llvm/llvm-project/pull/98892.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7ca798a8b2d89..748db418fee8c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8578,6 +8578,12 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I,
     BlockInMask = getBlockInMask(I->getParent());
   }
 
+  // Note that there is some custom logic to mark some intrinsics as uniform
+  // manually above for scalable vectors, which this assert needs to account for
+  // as well.
+  assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
+          (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
+         "Should not predicate a uniform recipe");
   auto *Recipe = new VPReplicateRecipe(I, mapToVPValues(I->operands()),
                                        IsUniform, BlockInMask);
   return Recipe;

From d06b55e7934635049d55efff2dc9e745f911240c Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm@amd.com>
Date: Thu, 18 Jul 2024 18:46:24 +0200
Subject: [PATCH 049/486] [Flang][Runtime] Improve runtime implementation of
 the RENAME intrinsic (#99445)

The RENAME implementation in the Fortran runtime had a few glitches that
had to be addressed:

- Wrong usage of RTDECL (fixed)
- Issue fatal error when trying to use RENAME on a target device (fixed)
---
 flang/runtime/misc-intrinsic.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/flang/runtime/misc-intrinsic.cpp b/flang/runtime/misc-intrinsic.cpp
index 2f7fcd2e2341f..f7d893829fc0d 100644
--- a/flang/runtime/misc-intrinsic.cpp
+++ b/flang/runtime/misc-intrinsic.cpp
@@ -56,10 +56,10 @@ static RT_API_ATTRS void TransferImpl(Descriptor &result,
 extern "C" {
 RT_EXT_API_GROUP_BEGIN
 
-void RTDECL(Rename)(const Descriptor &path1, const Descriptor &path2,
+void RTDEF(Rename)(const Descriptor &path1, const Descriptor &path2,
     const Descriptor *status, const char *sourceFile, int line) {
   Terminator terminator{sourceFile, line};
-
+#if !defined(RT_DEVICE_COMPILATION)
   char *pathSrc{EnsureNullTerminated(
       path1.OffsetElement(), path1.ElementBytes(), terminator)};
   char *pathDst{EnsureNullTerminated(
@@ -84,6 +84,9 @@ void RTDECL(Rename)(const Descriptor &path1, const Descriptor &path2,
   if (pathDst != path2.OffsetElement()) {
     FreeMemory(pathDst);
   }
+#else // !defined(RT_DEVICE_COMPILATION)
+  terminator.Crash("RENAME intrinsic is only supported on host devices");
+#endif // !defined(RT_DEVICE_COMPILATION)
 }
 
 void RTDEF(Transfer)(Descriptor &result, const Descriptor &source,

From 3d69bbc35158822c9e1371b5c37a24213a8a81fc Mon Sep 17 00:00:00 2001
From: Dominik Steenken <dost@de.ibm.com>
Date: Thu, 18 Jul 2024 18:57:25 +0200
Subject: [PATCH 050/486] Allow MAY(R)? to accept the high components of
 register pairs (#98606)

The HFP instructions `MAY` and `MAYR`, unlike any other floating point
instructions, allow the specification of a 128bit register pair by
either the lower-numbered or the higher-numbered component register. In
order to support this, but change as little about codegen as possible,
the existing `MAY(R)?` definition is made `CodeGenOnly`, while a copy is
provided for the assembler and disassembler, which simply accepts a
64bit floating point register in place of the 128bit one. This copy is
stripped of its pattern to prevent codegen from using it.
The corresponding assembly tests that checked the register specification
rule that this commit removes from `MAY(R)?` have also been removed.
---
 llvm/lib/Target/SystemZ/SystemZInstrHFP.td | 14 ++++++++++++--
 llvm/test/MC/SystemZ/insn-bad.s            |  8 --------
 llvm/test/MC/SystemZ/insn-good.s           |  4 ++++
 3 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/SystemZ/SystemZInstrHFP.td b/llvm/lib/Target/SystemZ/SystemZInstrHFP.td
index d2e05b63c6c63..ea194a38090db 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrHFP.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrHFP.td
@@ -209,13 +209,23 @@ def MYH  : BinaryRXF<"myh",  0xED3D, null_frag, FP64,  FP64, z_load, 8>;
 def MYL  : BinaryRXF<"myl",  0xED39, null_frag, FP64,  FP64, z_load, 8>;
 
 // Fused multiply-add (unnormalized).
-def MAYR  : TernaryRRD<"mayr",  0xB33A, null_frag, FP128, FP64>;
 def MAYHR : TernaryRRD<"mayhr", 0xB33C, null_frag, FP64,  FP64>;
 def MAYLR : TernaryRRD<"maylr", 0xB338, null_frag, FP64,  FP64>;
-def MAY   : TernaryRXF<"may",   0xED3A, null_frag, FP128, FP64, z_load, 8>;
 def MAYH  : TernaryRXF<"mayh",  0xED3C, null_frag, FP64,  FP64, z_load, 8>;
 def MAYL  : TernaryRXF<"mayl",  0xED38, null_frag, FP64,  FP64, z_load, 8>;
 
+// MAY and MAYR allow the user to specify the floating point register pair
+// making up the FP128 register by either the lower-numbered register or the
+// higher-numbered register, in contrast to all other floating point
+// instructions.
+// For this reason, the defs below accept `FP64,FP64` instead of `FP128,FP64`.
+// This is ok since these instructions are not used in code generation.
+// If and when code generation is enabled, the code gen variants should be
+// split out from this and use the proper register classes, while these should
+// remain for the Assembler and Disassembler to remain compliant with the POP.
+def MAY   : TernaryRXF<"may",   0xED3A, null_frag, FP64, FP64, z_load, 8>;
+def MAYR  : TernaryRRD<"mayr",  0xB33A, null_frag, FP64, FP64>;
+
 // Division.
 def DER : BinaryRR <"der", 0x3D,   null_frag, FP32,  FP32>;
 def DDR : BinaryRR <"ddr", 0x2D,   null_frag, FP64,  FP64>;
diff --git a/llvm/test/MC/SystemZ/insn-bad.s b/llvm/test/MC/SystemZ/insn-bad.s
index 6f94731fa0871..f81278610c73a 100644
--- a/llvm/test/MC/SystemZ/insn-bad.s
+++ b/llvm/test/MC/SystemZ/insn-bad.s
@@ -4176,12 +4176,9 @@
 #CHECK: may	%f0, %f0, -1
 #CHECK: error: invalid operand
 #CHECK: may	%f0, %f0, 4096
-#CHECK: error: invalid register pair
-#CHECK: may	%f2, %f0, 0
 
 	may	%f0, %f0, -1
 	may	%f0, %f0, 4096
-	may	%f2, %f0, 0
 
 #CHECK: error: invalid operand
 #CHECK: mayh	%f0, %f0, -1
@@ -4199,11 +4196,6 @@
 	mayl	%f0, %f0, -1
 	mayl	%f0, %f0, 4096
 
-#CHECK: error: invalid register pair
-#CHECK: mayr	%f2, %f0, %f0
-
-	mayr	%f2, %f0, %f0
-
 #CHECK: error: invalid operand
 #CHECK: mc	-1, 0
 #CHECK: error: invalid operand
diff --git a/llvm/test/MC/SystemZ/insn-good.s b/llvm/test/MC/SystemZ/insn-good.s
index 9fcb8a42cd73c..553c1b281eb4d 100644
--- a/llvm/test/MC/SystemZ/insn-good.s
+++ b/llvm/test/MC/SystemZ/insn-good.s
@@ -11526,6 +11526,7 @@
 #CHECK: may	%f0, %f15, 0            # encoding: [0xed,0xf0,0x00,0x00,0x00,0x3a]
 #CHECK: may	%f13, %f0, 0            # encoding: [0xed,0x00,0x00,0x00,0xd0,0x3a]
 #CHECK: may	%f13, %f15, 0           # encoding: [0xed,0xf0,0x00,0x00,0xd0,0x3a]
+#CHECK: may	%f2, %f0, 0             # encoding: [0xed,0x00,0x00,0x00,0x20,0x3a]
 
 	may	%f0, %f0, 0
 	may	%f0, %f0, 4095
@@ -11536,6 +11537,7 @@
 	may	%f0, %f15, 0
 	may	%f13, %f0, 0
 	may	%f13, %f15, 0
+	may	%f2, %f0, 0
 
 #CHECK: mayh	%f0, %f0, 0             # encoding: [0xed,0x00,0x00,0x00,0x00,0x3c]
 #CHECK: mayh	%f0, %f0, 4095          # encoding: [0xed,0x00,0x0f,0xff,0x00,0x3c]
@@ -11611,6 +11613,7 @@
 #CHECK: mayr	%f13, %f0, %f0          # encoding: [0xb3,0x3a,0xd0,0x00]
 #CHECK: mayr	%f5, %f8, %f9           # encoding: [0xb3,0x3a,0x50,0x89]
 #CHECK: mayr	%f13, %f15, %f15        # encoding: [0xb3,0x3a,0xd0,0xff]
+#CHECK: mayr	%f2, %f0, %f0           # encoding: [0xb3,0x3a,0x20,0x00]
 
 	mayr	%f0, %f0, %f0
 	mayr	%f0, %f0, %f15
@@ -11618,6 +11621,7 @@
 	mayr	%f13, %f0, %f0
 	mayr	%f5, %f8, %f9
 	mayr	%f13, %f15, %f15
+	mayr	%f2, %f0, %f0
 
 #CHECK: mc	0, 0                    # encoding: [0xaf,0x00,0x00,0x00]
 #CHECK: mc	4095, 0                 # encoding: [0xaf,0x00,0x0f,0xff]

From 574dbe3e9cdaf94b390015a53b76f87bdaf68aae Mon Sep 17 00:00:00 2001
From: Christopher Di Bella <cjdb@google.com>
Date: Thu, 18 Jul 2024 10:03:55 -0700
Subject: [PATCH 051/486] suppresses unused variable warning (#99526)

---
 llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
index b9cf36a07846c..0a6ce6a135817 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
@@ -3361,7 +3361,7 @@ void InstrRefBasedLDV::buildVLocValueMap(
         continue;
       if (BlockLiveIn->Kind == DbgValue::VPHI)
         BlockLiveIn->Kind = DbgValue::Def;
-      auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
+      [[maybe_unused]] auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
       assert(BlockLiveIn->Properties.DIExpr->getFragmentInfo() ==
                  Var.getFragment() &&
              "Fragment info missing during value prop");

From 74e51e3efe1d4c79c1b7914c3ead19832e8cc1fb Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen@sifive.com>
Date: Thu, 18 Jul 2024 10:00:45 -0700
Subject: [PATCH 052/486] Move the test to the correct folder. A test specified
 for a target should remain in its designated folder.

---
 llvm/test/Transforms/SLPVectorizer/{ => X86}/revec-fix-99411.ll | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename llvm/test/Transforms/SLPVectorizer/{ => X86}/revec-fix-99411.ll (100%)

diff --git a/llvm/test/Transforms/SLPVectorizer/revec-fix-99411.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-fix-99411.ll
similarity index 100%
rename from llvm/test/Transforms/SLPVectorizer/revec-fix-99411.ll
rename to llvm/test/Transforms/SLPVectorizer/X86/revec-fix-99411.ll

From 13a8f8d51962b59949496c460ea0b8ad22ae908a Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Fri, 19 Jul 2024 01:08:51 +0800
Subject: [PATCH 053/486] [InferAttrs] Set attributes for `remainder` (#99521)

Fixes one of the issues in
https://github.com/llvm/llvm-project/issues/99497.
---
 llvm/lib/Transforms/Utils/BuildLibCalls.cpp         | 3 +++
 llvm/test/Transforms/InferFunctionAttrs/annotate.ll | 9 +++++++++
 2 files changed, 12 insertions(+)

diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index e97506b4bbd95..1ced3657184d4 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -1195,6 +1195,9 @@ bool llvm::inferNonMandatoryLibFuncAttrs(Function &F,
   case LibFunc_pow:
   case LibFunc_powf:
   case LibFunc_powl:
+  case LibFunc_remainder:
+  case LibFunc_remainderf:
+  case LibFunc_remainderl:
   case LibFunc_rint:
   case LibFunc_rintf:
   case LibFunc_rintl:
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
index 456155d7e4437..0944402a91fd0 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -800,6 +800,15 @@ declare ptr @vec_realloc(ptr, i64)
 ; CHECK: declare noundef ptr @realpath(ptr nocapture noundef readonly, ptr noundef) [[NOFREE_NOUNWIND]]
 declare ptr @realpath(ptr, ptr)
 
+; CHECK: declare double @remainder(double, double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare double @remainder(double, double)
+
+; CHECK: declare float @remainderf(float, float) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare float @remainderf(float, float)
+
+; CHECK: declare x86_fp80 @remainderl(x86_fp80, x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare x86_fp80 @remainderl(x86_fp80, x86_fp80)
+
 ; CHECK: declare noundef i32 @remove(ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
 declare i32 @remove(ptr)
 

From 1c55586e9a475a09b7d769e7fc9a254e7150c972 Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Thu, 18 Jul 2024 09:07:28 -0700
Subject: [PATCH 054/486] [clang] Fix typo in comments

---
 clang/lib/Lex/Preprocessor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 44b69a58f3411..63e27e62cffc8 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -988,7 +988,7 @@ void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) {
 }
 
 /// Lex a header-name token (including one formed from header-name-tokens if
-/// \p AllowConcatenation is \c true).
+/// \p AllowMacroExpansion is \c true).
 ///
 /// \param FilenameTok Filled in with the next token. On success, this will
 ///        be either a header_name token. On failure, it will be whatever other

From ce8c43fe274f3f090cad2342af6032176efb846f Mon Sep 17 00:00:00 2001
From: Lei Wang <wlei@fb.com>
Date: Thu, 18 Jul 2024 10:16:44 -0700
Subject: [PATCH 055/486] Fix assertion of null pointer samples in inline
 replay mode (#99378)

Fix https://github.com/llvm/llvm-project/issues/97108. In inline replay
mode, `CalleeSamples` may be null and the order doesn't matter.
---
 llvm/lib/Transforms/IPO/SampleProfile.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 5cc2911a1a80e..6af284d513efc 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -439,7 +439,10 @@ struct CandidateComparer {
 
     const FunctionSamples *LCS = LHS.CalleeSamples;
     const FunctionSamples *RCS = RHS.CalleeSamples;
-    assert(LCS && RCS && "Expect non-null FunctionSamples");
+    // In inline replay mode, CalleeSamples may be null and the order doesn't
+    // matter.
+    if (!LCS || !RCS)
+      return LCS;
 
     // Tie breaker using number of samples try to favor smaller functions first
     if (LCS->getBodySamples().size() != RCS->getBodySamples().size())

From b2dcf62c514d3c9c143c85bd029d22098b92c38d Mon Sep 17 00:00:00 2001
From: Meredith Julian <35236176+mjulian31@users.noreply.github.com>
Date: Thu, 18 Jul 2024 10:29:24 -0700
Subject: [PATCH 056/486] [NVPTX] fix emission for i1 load and extload (#99392)

Currently, an illegal 2-byte load from a 1-byte global variable is being
generated. This change instead generates a 1-byte load and zero-extends
it to i16 register. This was always the intended behavior of the
function.

In addition, an i1 ext load of any kind needs to be promoted. A missing
setLoadExtAction for ISD::EXTLOAD was causing an "Unhandled source type"
unreachable due to an illegal i1 ext load during ISelDAGtoDAG (see below
bug).

Bug https://github.com/llvm/llvm-project/issues/98033.
---
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp |  8 +++--
 llvm/test/CodeGen/NVPTX/i1-ext-load.ll      | 34 +++++++++++++++++++++
 llvm/test/CodeGen/NVPTX/i1-load-lower.ll    | 31 +++++++++++++++++++
 3 files changed, 70 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/i1-ext-load.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/i1-load-lower.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index a2181b478c269..bc23998455a68 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -629,6 +629,7 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   for (MVT VT : MVT::integer_valuetypes()) {
     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
     setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
+    setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote);
     setTruncStoreAction(VT, MVT::i1, Expand);
   }
 
@@ -2920,9 +2921,10 @@ SDValue NVPTXTargetLowering::LowerLOADi1(SDValue Op, SelectionDAG &DAG) const {
   assert(LD->getExtensionType() == ISD::NON_EXTLOAD);
   assert(Node->getValueType(0) == MVT::i1 &&
          "Custom lowering for i1 load only");
-  SDValue newLD = DAG.getLoad(MVT::i16, dl, LD->getChain(), LD->getBasePtr(),
-                              LD->getPointerInfo(), LD->getAlign(),
-                              LD->getMemOperand()->getFlags());
+  SDValue newLD = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i16, LD->getChain(),
+                                 LD->getBasePtr(), LD->getPointerInfo(),
+                                 MVT::i8, LD->getAlign(),
+                                 LD->getMemOperand()->getFlags());
   SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD);
   // The legalizer (the caller) is expecting two values from the legalized
   // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()
diff --git a/llvm/test/CodeGen/NVPTX/i1-ext-load.ll b/llvm/test/CodeGen/NVPTX/i1-ext-load.ll
new file mode 100644
index 0000000000000..b775e40470047
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/i1-ext-load.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --function foo --extra_scrub --default-march nvptx64 --filter-out ".*//.*" --filter-out "[\{\}\(\)]" --version 5
+
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_50 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_50 | %ptxas-verify %}
+
+target triple = "nvptx-nvidia-cuda"
+
+define void @foo(ptr noalias readonly %ptr, ptr noalias %retval) {
+; CHECK-LABEL: foo(
+; CHECK:    .reg .b16 %rs<2>;
+; CHECK:    .reg .b32 %r<4>;
+; CHECK:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK:    ld.param.u64 %rd1, [foo_param_0];
+; CHECK:    ld.param.u64 %rd2, [foo_param_1];
+; CHECK:    cvta.to.global.u64 %rd3, %rd2;
+; CHECK:    cvta.to.global.u64 %rd4, %rd1;
+; CHECK:    ld.global.nc.u8 %rs1, [%rd4];
+; CHECK:    cvt.u32.u8 %r1, %rs1;
+; CHECK:    add.s32 %r2, %r1, 1;
+; CHECK:    and.b32 %r3, %r2, 1;
+; CHECK:    st.global.u32 [%rd3], %r3;
+; CHECK:    ret;
+  %ld = load i1, ptr %ptr, align 1
+  %zext = zext i1 %ld to i32
+  %add = add i32 %zext, 1
+  %and = and i32 %add, 1
+  store i32 %and, ptr %retval
+  ret void
+}
+
+!nvvm.annotations = !{!0}
+
+!0 = !{ptr @foo, !"kernel", i32 1}
diff --git a/llvm/test/CodeGen/NVPTX/i1-load-lower.ll b/llvm/test/CodeGen/NVPTX/i1-load-lower.ll
new file mode 100644
index 0000000000000..d1f99b5724de8
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/i1-load-lower.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --function foo --extra_scrub --default-march nvptx64 --filter-out ".*//.*" --filter-out "[\(\)\{\}]" --version 5
+
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
+
+target triple = "nvptx-nvidia-cuda"
+
+@i1g = addrspace(1) global i1 false, align 2
+
+define void @foo() {
+; CHECK-LABEL: foo(
+; CHECK:    .reg .pred %p<2>;
+; CHECK:    .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK:    ld.global.u8 %rs1, [i1g];
+; CHECK:    and.b16 %rs2, %rs1, 1;
+; CHECK:    setp.eq.b16 %p1, %rs2, 1;
+; CHECK:    @%p1 bra $L__BB0_2;
+; CHECK:    mov.u16 %rs3, 1;
+; CHECK:    st.global.u8 [i1g], %rs3;
+; CHECK:    ret;
+  %tmp = load i1, ptr addrspace(1) @i1g, align 2
+  br i1 %tmp, label %if.end, label %if.then
+
+if.then:
+  store i1 true, ptr addrspace(1) @i1g, align 2
+  br label %if.end
+
+if.end:
+  ret void
+}

From 04bcd74df73af6fed16bfd0d6784fc0aec582bc0 Mon Sep 17 00:00:00 2001
From: Utkarsh Saxena <usx@google.com>
Date: Thu, 18 Jul 2024 19:49:01 +0200
Subject: [PATCH 057/486] Revert "Add source file name for template
 instantiations in -ftime-trace" (#99534)

Reverts llvm/llvm-project#98320

Breaks windows tests:

```
Step 8 (test-build-unified-tree-check-clang-unit) failure: test (failure)
******************** TEST 'Clang-Unit :: Support/./ClangSupportTests.exe/1/3' FAILED ********************
Script(shard):
--
GTEST_OUTPUT=json:C:\buildbot\as-builder-3\llvm-clang-x86_64-win-fast\build\tools\clang\unittests\Support\.\ClangSupportTests.exe-Clang-Unit-4296-1-3.json GTEST_SHUFFLE=0 GTEST_TOTAL_SHARDS=3 GTEST_SHARD_INDEX=1 C:\buildbot\as-builder-3\llvm-clang-x86_64-win-fast\build\tools\clang\unittests\Support\.\ClangSupportTests.exe
--

Script:
--
C:\buildbot\as-builder-3\llvm-clang-x86_64-win-fast\build\tools\clang\unittests\Support\.\ClangSupportTests.exe --gtest_filter=TimeProfilerTest.TemplateInstantiations
--
C:\buildbot\as-builder-3\llvm-clang-x86_64-win-fast\llvm-project\clang\unittests\Support\TimeProfilerTest.cpp(278): error: Expected equality of these values:
  R"(
Frontend
| ParseFunctionDefinition (fooB)
| ParseFunctionDefinition (fooMTA)
| ParseFunctionDefinition (fooA)
| ParseDeclarationOrFunctionDefinition (test.cc:3:5)
| | ParseFunctionDefinition (user)
| PerformPendingInstantiations
| | InstantiateFunction (fooA<int>, ./a.h:7)
| | | InstantiateFunction (fooB<int>, ./b.h:3)
| | | InstantiateFunction (fooMTA<int>, ./a.h:4)
)"
    Which is: "\nFrontend\n| ParseFunctionDefinition (fooB)\n| ParseFunctionDefinition (fooMTA)\n| ParseFunctionDefinition (fooA)\n| ParseDeclarationOrFunctionDefinition (test.cc:3:5)\n| | ParseFunctionDefinition (user)\n| PerformPendingInstantiations\n| | InstantiateFunction (fooA<int>, ./a.h:7)\n| | | InstantiateFunction (fooB<int>, ./b.h:3)\n| | | InstantiateFunction (fooMTA<int>, ./a.h:4)\n"
  buildTraceGraph(Json)
    Which is: "\nFrontend\n| ParseFunctionDefinition (fooB)\n| ParseFunctionDefinition (fooMTA)\n| ParseFunctionDefinition (fooA)\n| ParseDeclarationOrFunctionDefinition (test.cc:3:5)\n| | ParseFunctionDefinition (user)\n| PerformPendingInstantiations\n| | InstantiateFunction (fooA<int>, .\\a.h:7)\n| | | InstantiateFunction (fooB<int>, .\\b.h:3)\n| | | InstantiateFunction (fooMTA<int>, .\\a.h:4)\n"
With diff:
@@ -7,5 +7,5 @@
 | | ParseFunctionDefinition (user)
 | PerformPendingInstantiations
-| | InstantiateFunction (fooA<int>, ./a.h:7)
-| | | InstantiateFunction (fooB<int>, ./b.h:3)
-| | | InstantiateFunction (fooMTA<int>, ./a.h:4)\n
+| | InstantiateFunction (fooA<int>, .\\a.h:7)
+| | | InstantiateFunction (fooB<int>, .\\b.h:3)
+| | | InstantiateFunction (fooMTA<int>, .\\a.h:4)\n



C:\buildbot\as-builder-3\llvm-clang-x86_64-win-fast\llvm-project\clang\unittests\Support\TimeProfilerTest.cpp:278
Expected equality of these values:
  R"(
Frontend
| ParseFunctionDefinition (fooB)
| ParseFunctionDefinition (fooMTA)
| ParseFunctionDefinition (fooA)
| ParseDeclarationOrFunctionDefinition (test.cc:3:5)
| | ParseFunctionDefinition (user)
| PerformPendingInstantiations
| | InstantiateFunction (fooA<int>, ./a.h:7)

```
---
 a-abfdec1d.o.tmp                              |   0
 clang/docs/ReleaseNotes.rst                   |   3 -
 clang/include/clang/Driver/Options.td         |   4 -
 .../include/clang/Frontend/FrontendOptions.h  |   8 +-
 clang/lib/Driver/ToolChains/Clang.cpp         |   1 -
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  11 +-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |  11 +-
 clang/test/Driver/ftime-trace-sections.cpp    |   2 +-
 clang/test/Driver/ftime-trace.cpp             |  39 +++---
 clang/tools/driver/cc1_main.cpp               |   3 +-
 clang/unittests/Support/TimeProfilerTest.cpp  | 121 ++++--------------
 llvm/include/llvm/Support/TimeProfiler.h      |  23 +---
 llvm/lib/Support/TimeProfiler.cpp             |  61 ++-------
 13 files changed, 64 insertions(+), 223 deletions(-)
 delete mode 100644 a-abfdec1d.o.tmp

diff --git a/a-abfdec1d.o.tmp b/a-abfdec1d.o.tmp
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 971df672b6ca1..e0e86af257a19 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -736,9 +736,6 @@ Improvements to Clang's time-trace
 - Clang now specifies that using ``auto`` in a lambda parameter is a C++14 extension when
   appropriate. (`#46059: <https://github.com/llvm/llvm-project/issues/46059>`_).
 
-- Clang now adds source file infomation for template instantiations as ``event["args"]["filename"]``. This
-  added behind an option ``-ftime-trace-verbose``. This is expected to increase the size of trace by 2-3 times.
-
 Improvements to Coverage Mapping
 --------------------------------
 
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index d3068c1b30a7a..1675e435d210c 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3988,10 +3988,6 @@ def ftime_trace_granularity_EQ : Joined<["-"], "ftime-trace-granularity=">, Grou
   HelpText<"Minimum time granularity (in microseconds) traced by time profiler">,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
   MarshallingInfoInt<FrontendOpts<"TimeTraceGranularity">, "500u">;
-def ftime_trace_verbose : Joined<["-"], "ftime-trace-verbose">, Group<f_Group>,
-  HelpText<"Make time trace capture verbose event details (e.g. source filenames). This can increase the size of the output by 2-3 times">,
-  Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
-  MarshallingInfoFlag<FrontendOpts<"TimeTraceVerbose">>;
 def ftime_trace_EQ : Joined<["-"], "ftime-trace=">, Group<f_Group>,
   HelpText<"Similar to -ftime-trace. Specify the JSON file or a directory which will contain the JSON file">,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 8241925c98476..5e5034fe01eb5 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -580,11 +580,6 @@ class FrontendOptions {
   /// Minimum time granularity (in microseconds) traced by time profiler.
   unsigned TimeTraceGranularity;
 
-  /// Make time trace capture verbose event details (e.g. source filenames).
-  /// This can increase the size of the output by 2-3 times.
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned TimeTraceVerbose : 1;
-
   /// Path which stores the output files for -ftime-trace
   std::string TimeTracePath;
 
@@ -606,8 +601,7 @@ class FrontendOptions {
         EmitSymbolGraph(false), EmitExtensionSymbolGraphs(false),
         EmitSymbolGraphSymbolLabelsForTesting(false),
         EmitPrettySymbolGraphs(false), GenReducedBMI(false),
-        UseClangIRPipeline(false), TimeTraceGranularity(500),
-        TimeTraceVerbose(false) {}
+        UseClangIRPipeline(false), TimeTraceGranularity(500) {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file
   /// extension. For example, "c" would return Language::C.
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 6b33301d36401..1fd6fba210042 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6754,7 +6754,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (const char *Name = C.getTimeTraceFile(&JA)) {
     CmdArgs.push_back(Args.MakeArgString("-ftime-trace=" + Twine(Name)));
     Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_granularity_EQ);
-    Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_verbose);
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_ftrapv_handler_EQ)) {
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 725b62db5e80a..a7bc6749c5852 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -3426,16 +3426,11 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
     return true;
 
   llvm::TimeTraceScope TimeScope("InstantiateClass", [&]() {
-    llvm::TimeTraceMetadata M;
-    llvm::raw_string_ostream OS(M.Detail);
+    std::string Name;
+    llvm::raw_string_ostream OS(Name);
     Instantiation->getNameForDiagnostic(OS, getPrintingPolicy(),
                                         /*Qualified=*/true);
-    if (llvm::isTimeTraceVerbose()) {
-      auto Loc = SourceMgr.getExpansionLoc(Instantiation->getLocation());
-      M.File = SourceMgr.getFilename(Loc);
-      M.Line = SourceMgr.getExpansionLineNumber(Loc);
-    }
-    return M;
+    return Name;
   });
 
   Pattern = PatternDef;
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 4e619f4b491a6..01432301633ed 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -4966,16 +4966,11 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   }
 
   llvm::TimeTraceScope TimeScope("InstantiateFunction", [&]() {
-    llvm::TimeTraceMetadata M;
-    llvm::raw_string_ostream OS(M.Detail);
+    std::string Name;
+    llvm::raw_string_ostream OS(Name);
     Function->getNameForDiagnostic(OS, getPrintingPolicy(),
                                    /*Qualified=*/true);
-    if (llvm::isTimeTraceVerbose()) {
-      auto Loc = SourceMgr.getExpansionLoc(Function->getLocation());
-      M.File = SourceMgr.getFilename(Loc);
-      M.Line = SourceMgr.getExpansionLineNumber(Loc);
-    }
-    return M;
+    return Name;
   });
 
   // If we're performing recursive template instantiation, create our own
diff --git a/clang/test/Driver/ftime-trace-sections.cpp b/clang/test/Driver/ftime-trace-sections.cpp
index da7109b9d81a6..0c16052bc0c3a 100644
--- a/clang/test/Driver/ftime-trace-sections.cpp
+++ b/clang/test/Driver/ftime-trace-sections.cpp
@@ -1,5 +1,5 @@
 // RUN: rm -rf %t && mkdir %t && cd %t
-// RUN: %clangxx -S -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
+// RUN: %clangxx -S -ftime-trace -ftime-trace-granularity=0 -o out %s
 // RUN: %python %S/ftime-trace-sections.py < out.json
 
 template <typename T>
diff --git a/clang/test/Driver/ftime-trace.cpp b/clang/test/Driver/ftime-trace.cpp
index 60c5885704b58..5fe63de915a71 100644
--- a/clang/test/Driver/ftime-trace.cpp
+++ b/clang/test/Driver/ftime-trace.cpp
@@ -1,18 +1,18 @@
 // RUN: rm -rf %t && mkdir -p %t && cd %t
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace -ftime-trace-granularity=0 -o out %s
 // RUN: cat out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=new-name.json -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=new-name.json -ftime-trace-granularity=0 -o out %s
 // RUN: cat new-name.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
 // RUN: mkdir dir1 dir2
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir1 -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir1 -ftime-trace-granularity=0 -o out %s
 // RUN: cat dir1/out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir2/ -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir2/ -ftime-trace-granularity=0 -o out %s
 // RUN: cat dir2/out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
@@ -34,33 +34,32 @@
 // RUN: mkdir d e f && cp %s d/a.cpp && touch d/b.c
 
 /// TODO: Support -fno-integrated-as.
-// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -fintegrated-as d/a.cpp -o e/a.o 2>&1 | FileCheck %s --check-prefix=COMPILE1
-// COMPILE1: -cc1{{.*}} "-ftime-trace=e/a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -fintegrated-as d/a.cpp -o e/a.o 2>&1 | FileCheck %s --check-prefix=COMPILE1
+// COMPILE1: -cc1{{.*}} "-ftime-trace=e/a.json" "-ftime-trace-granularity=0"
 
-// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=COMPILE2
-// COMPILE2: -cc1{{.*}} "-ftime-trace=f/a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
-// COMPILE2: -cc1{{.*}} "-ftime-trace=f/b.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=COMPILE2
+// COMPILE2: -cc1{{.*}} "-ftime-trace=f/a.json" "-ftime-trace-granularity=0"
+// COMPILE2: -cc1{{.*}} "-ftime-trace=f/b.json" "-ftime-trace-granularity=0"
 
 /// -o specifies the link output. Create ${output}-${basename}.json.
-// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o e/x 2>&1 | FileCheck %s --check-prefix=LINK1
-// LINK1: -cc1{{.*}} "-ftime-trace=e/x-a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
-// LINK1: -cc1{{.*}} "-ftime-trace=e/x-b.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -o e/x 2>&1 | FileCheck %s --check-prefix=LINK1
+// LINK1: -cc1{{.*}} "-ftime-trace=e/x-a.json" "-ftime-trace-granularity=0"
+// LINK1: -cc1{{.*}} "-ftime-trace=e/x-b.json" "-ftime-trace-granularity=0"
 
 /// -dumpdir is f/g, not ending with a path separator. We create f/g${basename}.json.
-// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o e/x -dumpdir f/g 2>&1 | FileCheck %s --check-prefix=LINK2
-// LINK2: -cc1{{.*}} "-ftime-trace=f/ga.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
-// LINK2: -cc1{{.*}} "-ftime-trace=f/gb.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -o e/x -dumpdir f/g 2>&1 | FileCheck %s --check-prefix=LINK2
+// LINK2: -cc1{{.*}} "-ftime-trace=f/ga.json" "-ftime-trace-granularity=0"
+// LINK2: -cc1{{.*}} "-ftime-trace=f/gb.json" "-ftime-trace-granularity=0"
 
-// RUN: %clang -### -ftime-trace=e -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o f/x -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=LINK3
-// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}a-{{[^.]*}}.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
-// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}b-{{[^.]*}}.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// RUN: %clang -### -ftime-trace=e -ftime-trace-granularity=0 d/a.cpp d/b.c -o f/x -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=LINK3
+// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}a-{{[^.]*}}.json" "-ftime-trace-granularity=0"
+// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}b-{{[^.]*}}.json" "-ftime-trace-granularity=0"
 
-// RUN: %clang -### -ftime-trace -ftime-trace=e -ftime-trace-granularity=1 -ftime-trace-verbose -xassembler d/a.cpp 2>&1 | \
+// RUN: %clang -### -ftime-trace -ftime-trace=e -ftime-trace-granularity=1 -xassembler d/a.cpp 2>&1 | \
 // RUN:   FileCheck %s --check-prefix=UNUSED
 // UNUSED:      warning: argument unused during compilation: '-ftime-trace'
 // UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace=e'
 // UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace-granularity=1'
-// UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace-verbose'
 // UNUSED-NOT:  warning:
 
 template <typename T>
diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp
index f5e5fad36573e..c2ccb47a15bc8 100644
--- a/clang/tools/driver/cc1_main.cpp
+++ b/clang/tools/driver/cc1_main.cpp
@@ -241,8 +241,7 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
 
   if (!Clang->getFrontendOpts().TimeTracePath.empty()) {
     llvm::timeTraceProfilerInitialize(
-        Clang->getFrontendOpts().TimeTraceGranularity, Argv0,
-        Clang->getFrontendOpts().TimeTraceVerbose);
+        Clang->getFrontendOpts().TimeTraceGranularity, Argv0);
   }
   // --print-supported-cpus takes priority over the actual compilation.
   if (Clang->getFrontendOpts().PrintSupportedCPUs)
diff --git a/clang/unittests/Support/TimeProfilerTest.cpp b/clang/unittests/Support/TimeProfilerTest.cpp
index 96e137508ed94..5f3950ff033f1 100644
--- a/clang/unittests/Support/TimeProfilerTest.cpp
+++ b/clang/unittests/Support/TimeProfilerTest.cpp
@@ -10,14 +10,11 @@
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Lex/PreprocessorOptions.h"
 
-#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/TimeProfiler.h"
-#include "llvm/Support/VirtualFileSystem.h"
 #include <stack>
 
 #include "gtest/gtest.h"
-#include <tuple>
 
 using namespace clang;
 using namespace llvm;
@@ -26,8 +23,7 @@ namespace {
 
 // Should be called before testing.
 void setupProfiler() {
-  timeTraceProfilerInitialize(/*TimeTraceGranularity=*/0, "test",
-                              /*TimeTraceVerbose=*/true);
+  timeTraceProfilerInitialize(/*TimeTraceGranularity=*/0, "test");
 }
 
 // Should be called after `compileFromString()`.
@@ -42,24 +38,14 @@ std::string teardownProfiler() {
 
 // Returns true if code compiles successfully.
 // We only parse AST here. This is enough for constexpr evaluation.
-bool compileFromString(StringRef Code, StringRef Standard, StringRef File,
-                       llvm::StringMap<std::string> Headers = {}) {
+bool compileFromString(StringRef Code, StringRef Standard, StringRef FileName) {
   CompilerInstance Compiler;
   Compiler.createDiagnostics();
 
-  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> FS(
-      new llvm::vfs::InMemoryFileSystem());
-  FS->addFile(File, 0, MemoryBuffer::getMemBuffer(Code));
-  for (const auto &Header : Headers) {
-    FS->addFile(Header.getKey(), 0,
-                MemoryBuffer::getMemBuffer(Header.getValue()));
-  }
-  llvm::IntrusiveRefCntPtr<FileManager> Files(
-      new FileManager(FileSystemOptions(), FS));
-  Compiler.setFileManager(Files.get());
-
   auto Invocation = std::make_shared<CompilerInvocation>();
-  std::vector<const char *> Args = {Standard.data(), File.data()};
+  Invocation->getPreprocessorOpts().addRemappedFile(
+      FileName, MemoryBuffer::getMemBuffer(Code).release());
+  const char *Args[] = {Standard.data(), FileName.data()};
   CompilerInvocation::CreateFromArgs(*Invocation, Args,
                                      Compiler.getDiagnostics());
   Compiler.setInvocation(std::move(Invocation));
@@ -74,27 +60,13 @@ bool compileFromString(StringRef Code, StringRef Standard, StringRef File,
   return Compiler.ExecuteAction(Action);
 }
 
-std::string GetMetadata(json::Object *Event) {
-  std::string Metadata;
-  llvm::raw_string_ostream OS(Metadata);
-  if (json::Object *Args = Event->getObject("args")) {
-    if (auto Detail = Args->getString("detail"))
-      OS << Detail->str();
-    if (auto File = Args->getString("file"))
-      OS << ", " << File->str();
-    if (auto Line = Args->getInteger("line"))
-      OS << ":" << *Line;
-  }
-  return Metadata;
-}
-
 // Returns pretty-printed trace graph.
 std::string buildTraceGraph(StringRef Json) {
   struct EventRecord {
     int64_t TimestampBegin;
     int64_t TimestampEnd;
-    std::string Name;
-    std::string Metadata;
+    StringRef Name;
+    StringRef Detail;
   };
   std::vector<EventRecord> Events;
 
@@ -109,13 +81,10 @@ std::string buildTraceGraph(StringRef Json) {
     int64_t TimestampBegin = TraceEventObj->getInteger("ts").value_or(0);
     int64_t TimestampEnd =
         TimestampBegin + TraceEventObj->getInteger("dur").value_or(0);
-    std::string Name = TraceEventObj->getString("name").value_or("").str();
-    std::string Metadata = GetMetadata(TraceEventObj);
-
-    // Source events are asynchronous events and may not perfectly nest the
-    // synchronous events. Skip testing them.
-    if (Name == "Source")
-      continue;
+    StringRef Name = TraceEventObj->getString("name").value_or("");
+    StringRef Detail = "";
+    if (json::Object *Args = TraceEventObj->getObject("args"))
+      Detail = Args->getString("detail").value_or("");
 
     // This is a "summary" event, like "Total PerformPendingInstantiations",
     // skip it
@@ -123,7 +92,7 @@ std::string buildTraceGraph(StringRef Json) {
       continue;
 
     Events.emplace_back(
-        EventRecord{TimestampBegin, TimestampEnd, Name, Metadata});
+        EventRecord{TimestampBegin, TimestampEnd, Name, Detail});
   }
 
   // There can be nested events that are very fast, for example:
@@ -163,9 +132,9 @@ std::string buildTraceGraph(StringRef Json) {
       Stream << "| ";
     }
     Stream.write(Event.Name.data(), Event.Name.size());
-    if (!Event.Metadata.empty()) {
+    if (!Event.Detail.empty()) {
       Stream << " (";
-      Stream.write(Event.Metadata.data(), Event.Metadata.size());
+      Stream.write(Event.Detail.data(), Event.Detail.size());
       Stream << ")";
     }
     Stream << "\n";
@@ -176,7 +145,7 @@ std::string buildTraceGraph(StringRef Json) {
 } // namespace
 
 TEST(TimeProfilerTest, ConstantEvaluationCxx20) {
-  std::string Code = R"(
+  constexpr StringRef Code = R"(
 void print(double value);
 
 namespace slow_namespace {
@@ -206,7 +175,8 @@ constexpr int slow_init_list[] = {1, 1, 2, 3, 5, 8, 13, 21}; // 25th line
   setupProfiler();
   ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc"));
   std::string Json = teardownProfiler();
-  ASSERT_EQ(R"(
+  std::string TraceGraph = buildTraceGraph(Json);
+  ASSERT_TRUE(TraceGraph == R"(
 Frontend
 | ParseDeclarationOrFunctionDefinition (test.cc:2:1)
 | ParseDeclarationOrFunctionDefinition (test.cc:6:1)
@@ -232,54 +202,14 @@ Frontend
 | ParseDeclarationOrFunctionDefinition (test.cc:25:1)
 | | EvaluateAsInitializer (slow_init_list)
 | PerformPendingInstantiations
-)",
-            buildTraceGraph(Json));
-}
-
-TEST(TimeProfilerTest, TemplateInstantiations) {
-  std::string B_H = R"(
-    template <typename T>
-    T fooB(T t) {
-      return T();
-    }
+)");
 
-    #define MacroTemp(x) template <typename T> void foo##x(T) { T(); }
-  )";
-
-  std::string A_H = R"(
-    #include "b.h"
-
-    MacroTemp(MTA)
-
-    template <typename T>
-    void fooA(T t) { fooB(t); fooMTA(t); }
-  )";
-  std::string Code = R"(
-    #include "a.h"
-    void user() { fooA(0); }
-  )";
-
-  setupProfiler();
-  ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc",
-                                /*Headers=*/{{"a.h", A_H}, {"b.h", B_H}}));
-  std::string Json = teardownProfiler();
-  ASSERT_EQ(R"(
-Frontend
-| ParseFunctionDefinition (fooB)
-| ParseFunctionDefinition (fooMTA)
-| ParseFunctionDefinition (fooA)
-| ParseDeclarationOrFunctionDefinition (test.cc:3:5)
-| | ParseFunctionDefinition (user)
-| PerformPendingInstantiations
-| | InstantiateFunction (fooA<int>, ./a.h:7)
-| | | InstantiateFunction (fooB<int>, ./b.h:3)
-| | | InstantiateFunction (fooMTA<int>, ./a.h:4)
-)",
-            buildTraceGraph(Json));
+  // NOTE: If this test is failing, run this test with
+  // `llvm::errs() << TraceGraph;` and change the assert above.
 }
 
 TEST(TimeProfilerTest, ConstantEvaluationC99) {
-  std::string Code = R"(
+  constexpr StringRef Code = R"(
 struct {
   short quantval[4]; // 3rd line
 } value;
@@ -288,12 +218,15 @@ struct {
   setupProfiler();
   ASSERT_TRUE(compileFromString(Code, "-std=c99", "test.c"));
   std::string Json = teardownProfiler();
-  ASSERT_EQ(R"(
+  std::string TraceGraph = buildTraceGraph(Json);
+  ASSERT_TRUE(TraceGraph == R"(
 Frontend
 | ParseDeclarationOrFunctionDefinition (test.c:2:1)
 | | isIntegerConstantExpr (<test.c:3:18>)
 | | EvaluateKnownConstIntCheckOverflow (<test.c:3:18>)
 | PerformPendingInstantiations
-)",
-            buildTraceGraph(Json));
+)");
+
+  // NOTE: If this test is failing, run this test with
+  // `llvm::errs() << TraceGraph;` and change the assert above.
 }
diff --git a/llvm/include/llvm/Support/TimeProfiler.h b/llvm/include/llvm/Support/TimeProfiler.h
index 6eb92930b36fd..31f7df10916db 100644
--- a/llvm/include/llvm/Support/TimeProfiler.h
+++ b/llvm/include/llvm/Support/TimeProfiler.h
@@ -83,28 +83,16 @@ namespace llvm {
 
 class raw_pwrite_stream;
 
-struct TimeTraceMetadata {
-  std::string Detail;
-  // Source file and line number information for the event.
-  std::string File;
-  int Line;
-
-  bool isEmpty() const { return Detail.empty() && File.empty(); }
-};
-
 struct TimeTraceProfiler;
 TimeTraceProfiler *getTimeTraceProfilerInstance();
 
-bool isTimeTraceVerbose();
-
 struct TimeTraceProfilerEntry;
 
 /// Initialize the time trace profiler.
 /// This sets up the global \p TimeTraceProfilerInstance
 /// variable to be the profiler instance.
 void timeTraceProfilerInitialize(unsigned TimeTraceGranularity,
-                                 StringRef ProcName,
-                                 bool TimeTraceVerbose = false);
+                                 StringRef ProcName);
 
 /// Cleanup the time trace profiler, if it was initialized.
 void timeTraceProfilerCleanup();
@@ -140,10 +128,6 @@ TimeTraceProfilerEntry *
 timeTraceProfilerBegin(StringRef Name,
                        llvm::function_ref<std::string()> Detail);
 
-TimeTraceProfilerEntry *
-timeTraceProfilerBegin(StringRef Name,
-                       llvm::function_ref<TimeTraceMetadata()> MetaData);
-
 /// Manually begin a time section, with the given \p Name and \p Detail.
 /// This starts Async Events having \p Name as a category which is shown
 /// separately from other traces. See
@@ -180,11 +164,6 @@ class TimeTraceScope {
     if (getTimeTraceProfilerInstance() != nullptr)
       Entry = timeTraceProfilerBegin(Name, Detail);
   }
-  TimeTraceScope(StringRef Name,
-                 llvm::function_ref<TimeTraceMetadata()> Metadata) {
-    if (getTimeTraceProfilerInstance() != nullptr)
-      Entry = timeTraceProfilerBegin(Name, Metadata);
-  }
   ~TimeTraceScope() {
     if (getTimeTraceProfilerInstance() != nullptr)
       timeTraceProfilerEnd(Entry);
diff --git a/llvm/lib/Support/TimeProfiler.cpp b/llvm/lib/Support/TimeProfiler.cpp
index c2014028ddadc..9612db7d30f98 100644
--- a/llvm/lib/Support/TimeProfiler.cpp
+++ b/llvm/lib/Support/TimeProfiler.cpp
@@ -73,20 +73,12 @@ struct llvm::TimeTraceProfilerEntry {
   const TimePointType Start;
   TimePointType End;
   const std::string Name;
-  TimeTraceMetadata Metadata;
-
+  const std::string Detail;
   const bool AsyncEvent = false;
   TimeTraceProfilerEntry(TimePointType &&S, TimePointType &&E, std::string &&N,
                          std::string &&Dt, bool Ae)
-      : Start(std::move(S)), End(std::move(E)), Name(std::move(N)), Metadata(),
-        AsyncEvent(Ae) {
-    Metadata.Detail = std::move(Dt);
-  }
-
-  TimeTraceProfilerEntry(TimePointType &&S, TimePointType &&E, std::string &&N,
-                         TimeTraceMetadata &&Mt, bool Ae)
       : Start(std::move(S)), End(std::move(E)), Name(std::move(N)),
-        Metadata(std::move(Mt)), AsyncEvent(Ae) {}
+        Detail(std::move(Dt)), AsyncEvent(Ae) {}
 
   // Calculate timings for FlameGraph. Cast time points to microsecond precision
   // rather than casting duration. This avoids truncation issues causing inner
@@ -105,12 +97,10 @@ struct llvm::TimeTraceProfilerEntry {
 };
 
 struct llvm::TimeTraceProfiler {
-  TimeTraceProfiler(unsigned TimeTraceGranularity = 0, StringRef ProcName = "",
-                    bool TimeTraceVerbose = false)
+  TimeTraceProfiler(unsigned TimeTraceGranularity = 0, StringRef ProcName = "")
       : BeginningOfTime(system_clock::now()), StartTime(ClockType::now()),
         ProcName(ProcName), Pid(sys::Process::getProcessId()),
-        Tid(llvm::get_threadid()), TimeTraceGranularity(TimeTraceGranularity),
-        TimeTraceVerbose(TimeTraceVerbose) {
+        Tid(llvm::get_threadid()), TimeTraceGranularity(TimeTraceGranularity) {
     llvm::get_thread_name(ThreadName);
   }
 
@@ -123,15 +113,6 @@ struct llvm::TimeTraceProfiler {
     return Stack.back().get();
   }
 
-  TimeTraceProfilerEntry *
-  begin(std::string Name, llvm::function_ref<TimeTraceMetadata()> Metadata,
-        bool AsyncEvent = false) {
-    Stack.emplace_back(std::make_unique<TimeTraceProfilerEntry>(
-        ClockType::now(), TimePointType(), std::move(Name), Metadata(),
-        AsyncEvent));
-    return Stack.back().get();
-  }
-
   void end() {
     assert(!Stack.empty() && "Must call begin() first");
     end(*Stack.back());
@@ -203,15 +184,8 @@ struct llvm::TimeTraceProfiler {
           J.attribute("dur", DurUs);
         }
         J.attribute("name", E.Name);
-        if (!E.Metadata.isEmpty()) {
-          J.attributeObject("args", [&] {
-            if (!E.Metadata.Detail.empty())
-              J.attribute("detail", E.Metadata.Detail);
-            if (!E.Metadata.File.empty())
-              J.attribute("file", E.Metadata.File);
-            if (E.Metadata.Line > 0)
-              J.attribute("line", E.Metadata.Line);
-          });
+        if (!E.Detail.empty()) {
+          J.attributeObject("args", [&] { J.attribute("detail", E.Detail); });
         }
       });
 
@@ -333,25 +307,14 @@ struct llvm::TimeTraceProfiler {
 
   // Minimum time granularity (in microseconds)
   const unsigned TimeTraceGranularity;
-
-  // Make time trace capture verbose event details (e.g. source filenames). This
-  // can increase the size of the output by 2-3 times.
-  const bool TimeTraceVerbose;
 };
 
-bool llvm::isTimeTraceVerbose() {
-  return getTimeTraceProfilerInstance() &&
-         getTimeTraceProfilerInstance()->TimeTraceVerbose;
-}
-
 void llvm::timeTraceProfilerInitialize(unsigned TimeTraceGranularity,
-                                       StringRef ProcName,
-                                       bool TimeTraceVerbose) {
+                                       StringRef ProcName) {
   assert(TimeTraceProfilerInstance == nullptr &&
          "Profiler should not be initialized");
   TimeTraceProfilerInstance = new TimeTraceProfiler(
-      TimeTraceGranularity, llvm::sys::path::filename(ProcName),
-      TimeTraceVerbose);
+      TimeTraceGranularity, llvm::sys::path::filename(ProcName));
 }
 
 // Removes all TimeTraceProfilerInstances.
@@ -418,14 +381,6 @@ llvm::timeTraceProfilerBegin(StringRef Name,
   return nullptr;
 }
 
-TimeTraceProfilerEntry *
-llvm::timeTraceProfilerBegin(StringRef Name,
-                             llvm::function_ref<TimeTraceMetadata()> Metadata) {
-  if (TimeTraceProfilerInstance != nullptr)
-    return TimeTraceProfilerInstance->begin(std::string(Name), Metadata, false);
-  return nullptr;
-}
-
 TimeTraceProfilerEntry *llvm::timeTraceAsyncProfilerBegin(StringRef Name,
                                                           StringRef Detail) {
   if (TimeTraceProfilerInstance != nullptr)

From 9fb049c8c6a77026fa75a8d36b386a7f5a60613a Mon Sep 17 00:00:00 2001
From: OverMighty <its.overmighty@gmail.com>
Date: Thu, 18 Jul 2024 19:50:49 +0200
Subject: [PATCH 058/486] [libc][math][c23] Add {f,d}mul{l,f128} and
 f16mul{,f,l,f128} C23 math functions (#98972)

Part of #93566.

Fixes #94833.
---
 libc/config/linux/aarch64/entrypoints.txt     |   4 +
 libc/config/linux/x86_64/entrypoints.txt      |   8 +
 libc/docs/math/index.rst                      |   6 +-
 libc/spec/llvm_libc_ext.td                    |   8 +
 libc/spec/stdc.td                             |   9 +-
 .../__support/FPUtil/generic/CMakeLists.txt   |  17 ++
 libc/src/__support/FPUtil/generic/mul.h       | 105 +++++++++++
 libc/src/math/CMakeLists.txt                  |  10 +
 libc/src/math/dmulf128.h                      |  21 +++
 libc/src/math/dmull.h                         |  20 ++
 libc/src/math/f16mul.h                        |  21 +++
 libc/src/math/f16mulf.h                       |  21 +++
 libc/src/math/f16mulf128.h                    |  21 +++
 libc/src/math/f16mull.h                       |  21 +++
 libc/src/math/fmulf128.h                      |  21 +++
 libc/src/math/fmull.h                         |  20 ++
 libc/src/math/generic/CMakeLists.txt          | 108 ++++++++++-
 libc/src/math/generic/dmulf128.cpp            |  20 ++
 libc/src/math/generic/dmull.cpp               |  20 ++
 libc/src/math/generic/f16mul.cpp              |  20 ++
 libc/src/math/generic/f16mulf.cpp             |  20 ++
 libc/src/math/generic/f16mulf128.cpp          |  20 ++
 libc/src/math/generic/f16mull.cpp             |  20 ++
 libc/src/math/generic/fmul.cpp                | 115 +-----------
 libc/src/math/generic/fmulf128.cpp            |  20 ++
 libc/src/math/generic/fmull.cpp               |  20 ++
 libc/test/src/math/CMakeLists.txt             |  80 +++++++-
 libc/test/src/math/FMulTest.h                 | 121 -------------
 libc/test/src/math/MulTest.h                  |  95 ++++++++++
 libc/test/src/math/dmull_test.cpp             |  13 ++
 libc/test/src/math/f16mul_test.cpp            |  13 ++
 libc/test/src/math/f16mulf_test.cpp           |  13 ++
 libc/test/src/math/f16mull_test.cpp           |  13 ++
 libc/test/src/math/fmul_test.cpp              |   8 +-
 libc/test/src/math/fmull_test.cpp             |  13 ++
 libc/test/src/math/smoke/CMakeLists.txt       |  96 +++++++++-
 libc/test/src/math/smoke/FMulTest.h           | 104 -----------
 libc/test/src/math/smoke/MulTest.h            | 171 ++++++++++++++++++
 libc/test/src/math/smoke/dmulf128_test.cpp    |  13 ++
 libc/test/src/math/smoke/dmull_test.cpp       |  13 ++
 libc/test/src/math/smoke/f16mul_test.cpp      |  13 ++
 libc/test/src/math/smoke/f16mulf128_test.cpp  |  13 ++
 libc/test/src/math/smoke/f16mulf_test.cpp     |  13 ++
 libc/test/src/math/smoke/f16mull_test.cpp     |  13 ++
 libc/test/src/math/smoke/fmul_test.cpp        |   8 +-
 libc/test/src/math/smoke/fmulf128_test.cpp    |  13 ++
 libc/test/src/math/smoke/fmull_test.cpp       |  13 ++
 libc/utils/MPFRWrapper/MPFRUtils.cpp          |  21 ++-
 libc/utils/MPFRWrapper/MPFRUtils.h            |   2 +-
 49 files changed, 1227 insertions(+), 364 deletions(-)
 create mode 100644 libc/src/__support/FPUtil/generic/mul.h
 create mode 100644 libc/src/math/dmulf128.h
 create mode 100644 libc/src/math/dmull.h
 create mode 100644 libc/src/math/f16mul.h
 create mode 100644 libc/src/math/f16mulf.h
 create mode 100644 libc/src/math/f16mulf128.h
 create mode 100644 libc/src/math/f16mull.h
 create mode 100644 libc/src/math/fmulf128.h
 create mode 100644 libc/src/math/fmull.h
 create mode 100644 libc/src/math/generic/dmulf128.cpp
 create mode 100644 libc/src/math/generic/dmull.cpp
 create mode 100644 libc/src/math/generic/f16mul.cpp
 create mode 100644 libc/src/math/generic/f16mulf.cpp
 create mode 100644 libc/src/math/generic/f16mulf128.cpp
 create mode 100644 libc/src/math/generic/f16mull.cpp
 create mode 100644 libc/src/math/generic/fmulf128.cpp
 create mode 100644 libc/src/math/generic/fmull.cpp
 delete mode 100644 libc/test/src/math/FMulTest.h
 create mode 100644 libc/test/src/math/MulTest.h
 create mode 100644 libc/test/src/math/dmull_test.cpp
 create mode 100644 libc/test/src/math/f16mul_test.cpp
 create mode 100644 libc/test/src/math/f16mulf_test.cpp
 create mode 100644 libc/test/src/math/f16mull_test.cpp
 create mode 100644 libc/test/src/math/fmull_test.cpp
 delete mode 100644 libc/test/src/math/smoke/FMulTest.h
 create mode 100644 libc/test/src/math/smoke/MulTest.h
 create mode 100644 libc/test/src/math/smoke/dmulf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/dmull_test.cpp
 create mode 100644 libc/test/src/math/smoke/f16mul_test.cpp
 create mode 100644 libc/test/src/math/smoke/f16mulf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/f16mulf_test.cpp
 create mode 100644 libc/test/src/math/smoke/f16mull_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmulf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmull_test.cpp

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 9b718c3f81151..208889ba34a59 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -357,6 +357,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.cosf
     libc.src.math.coshf
     libc.src.math.cospif
+    libc.src.math.dmull
     libc.src.math.erff
     libc.src.math.exp
     libc.src.math.exp10
@@ -411,6 +412,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fmodf
     libc.src.math.fmodl
     libc.src.math.fmul
+    libc.src.math.fmull
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
@@ -531,6 +533,8 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.f16div
     libc.src.math.f16divf
     libc.src.math.f16fmaf
+    libc.src.math.f16mul
+    libc.src.math.f16mulf
     libc.src.math.f16sqrt
     libc.src.math.f16sqrtf
     libc.src.math.f16sub
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 4d19a28f4a2b3..cbdee084aa199 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -382,6 +382,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.cosf
     libc.src.math.coshf
     libc.src.math.cospif
+    libc.src.math.dmull
     libc.src.math.erff
     libc.src.math.exp
     libc.src.math.exp10
@@ -437,6 +438,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fmodf
     libc.src.math.fmodl
     libc.src.math.fmul
+    libc.src.math.fmull
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
@@ -561,6 +563,9 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.f16fma
     libc.src.math.f16fmaf
     libc.src.math.f16fmal
+    libc.src.math.f16mul
+    libc.src.math.f16mulf
+    libc.src.math.f16mull
     libc.src.math.f16sqrt
     libc.src.math.f16sqrtf
     libc.src.math.f16sqrtl
@@ -622,6 +627,7 @@ if(LIBC_TYPES_HAS_FLOAT16)
       libc.src.math.f16addf128
       libc.src.math.f16divf128
       libc.src.math.f16fmaf128
+      libc.src.math.f16mulf128
       libc.src.math.f16sqrtf128
       libc.src.math.f16subf128
     )
@@ -634,6 +640,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.canonicalizef128
     libc.src.math.ceilf128
     libc.src.math.copysignf128
+    libc.src.math.dmulf128
     libc.src.math.fabsf128
     libc.src.math.fdimf128
     libc.src.math.floorf128
@@ -648,6 +655,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.fminimum_numf128
     libc.src.math.fminimumf128
     libc.src.math.fmodf128
+    libc.src.math.fmulf128
     libc.src.math.frexpf128
     libc.src.math.fromfpf128
     libc.src.math.fromfpxf128
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index 205d14946535e..5fab9ce4df949 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -120,7 +120,7 @@ Basic Operations
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | dfma             | N/A              | N/A             |                        | N/A                  |                        | 7.12.14.5              | F.10.11                    |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| dmul             | N/A              | N/A             |                        | N/A                  |                        | 7.12.14.3              | F.10.11                    |
+| dmul             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | 7.12.14.3              | F.10.11                    |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | dsub             | N/A              | N/A             |                        | N/A                  |                        | 7.12.14.2              | F.10.11                    |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
@@ -130,6 +130,8 @@ Basic Operations
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | f16fma           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | 7.12.14.5              | F.10.11                    |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| f16mul           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | 7.12.14.5              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | f16sub           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | 7.12.14.2              | F.10.11                    |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | fabs             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.7.3               | F.10.4.3                   |
@@ -166,7 +168,7 @@ Basic Operations
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | fmod             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.10.1              | F.10.7.1                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmul             | N/A              | |check|         |                        | N/A                  |                        | 7.12.14.3              | F.10.11                    |
+| fmul             | N/A              | |check|         | |check|                | N/A                  | |check|\*              | 7.12.14.3              | F.10.11                    |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | frexp            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.7               | F.10.3.7                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/spec/llvm_libc_ext.td b/libc/spec/llvm_libc_ext.td
index 86215029831ca..55b354cae74e0 100644
--- a/libc/spec/llvm_libc_ext.td
+++ b/libc/spec/llvm_libc_ext.td
@@ -65,6 +65,14 @@ def LLVMLibcExt : StandardSpec<"llvm_libc_ext"> {
           GuardedFunctionSpec<"f16subf", RetValSpec<Float16Type>, [ArgSpec<FloatType>, ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"f16subl", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
 
+          GuardedFunctionSpec<"fmulf128", RetValSpec<FloatType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+
+          GuardedFunctionSpec<"dmulf128", RetValSpec<DoubleType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+
+          GuardedFunctionSpec<"f16mul", RetValSpec<Float16Type>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
+          GuardedFunctionSpec<"f16mulf", RetValSpec<Float16Type>, [ArgSpec<FloatType>, ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
+          GuardedFunctionSpec<"f16mull", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
+
           GuardedFunctionSpec<"f16div", RetValSpec<Float16Type>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"f16divf", RetValSpec<Float16Type>, [ArgSpec<FloatType>, ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"f16divl", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index a4c6b40b98388..18592e92d330a 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -475,8 +475,6 @@ def StdC : StandardSpec<"stdc"> {
           GuardedFunctionSpec<"fminimum_mag_numf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fminimum_mag_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
-          FunctionSpec<"fmul", RetValSpec<FloatType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-
           FunctionSpec<"fma", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fmaf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>, ArgSpec<FloatType>]>,
 
@@ -733,6 +731,13 @@ def StdC : StandardSpec<"stdc"> {
 
           GuardedFunctionSpec<"f16subf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
 
+          FunctionSpec<"fmul", RetValSpec<FloatType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fmull", RetValSpec<FloatType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+
+          FunctionSpec<"dmull", RetValSpec<DoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+
+          GuardedFunctionSpec<"f16mulf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
+
           GuardedFunctionSpec<"f16divf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
 
           GuardedFunctionSpec<"f16sqrtf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
diff --git a/libc/src/__support/FPUtil/generic/CMakeLists.txt b/libc/src/__support/FPUtil/generic/CMakeLists.txt
index c73f68723e232..43096aa529fc3 100644
--- a/libc/src/__support/FPUtil/generic/CMakeLists.txt
+++ b/libc/src/__support/FPUtil/generic/CMakeLists.txt
@@ -84,3 +84,20 @@ add_header_library(
     libc.src.__support.macros.attributes
     libc.src.__support.macros.optimization
 )
+
+add_header_library(
+  mul
+  HDRS
+    mul.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.CPP.bit
+    libc.src.__support.CPP.type_traits
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.dyadic_float
+    libc.src.__support.macros.attributes
+    libc.src.__support.macros.optimization
+)
diff --git a/libc/src/__support/FPUtil/generic/mul.h b/libc/src/__support/FPUtil/generic/mul.h
new file mode 100644
index 0000000000000..02fc69c6cb1ba
--- /dev/null
+++ b/libc/src/__support/FPUtil/generic/mul.h
@@ -0,0 +1,105 @@
+//===-- Multiplication of IEEE 754 floating-point numbers -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_GENERIC_MUL_H
+#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_GENERIC_MUL_H
+
+#include "hdr/errno_macros.h"
+#include "hdr/fenv_macros.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/CPP/type_traits.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/dyadic_float.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace fputil::generic {
+
+template <typename OutType, typename InType>
+LIBC_INLINE cpp::enable_if_t<cpp::is_floating_point_v<OutType> &&
+                                 cpp::is_floating_point_v<InType> &&
+                                 sizeof(OutType) <= sizeof(InType),
+                             OutType>
+mul(InType x, InType y) {
+  using OutFPBits = FPBits<OutType>;
+  using OutStorageType = typename OutFPBits::StorageType;
+  using InFPBits = FPBits<InType>;
+  using InStorageType = typename InFPBits::StorageType;
+  // The product of two p-digit numbers is a 2p-digit number.
+  using DyadicFloat =
+      DyadicFloat<cpp::bit_ceil(2 * static_cast<size_t>(InFPBits::SIG_LEN))>;
+
+  InFPBits x_bits(x);
+  InFPBits y_bits(y);
+
+  Sign result_sign = x_bits.sign() == y_bits.sign() ? Sign::POS : Sign::NEG;
+
+  if (LIBC_UNLIKELY(x_bits.is_inf_or_nan() || y_bits.is_inf_or_nan() ||
+                    x_bits.is_zero() || y_bits.is_zero())) {
+    if (x_bits.is_nan() || y_bits.is_nan()) {
+      if (x_bits.is_signaling_nan() || y_bits.is_signaling_nan())
+        raise_except_if_required(FE_INVALID);
+
+      if (x_bits.is_quiet_nan()) {
+        InStorageType x_payload = static_cast<InStorageType>(getpayload(x));
+        if ((x_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
+          return OutFPBits::quiet_nan(x_bits.sign(),
+                                      static_cast<OutStorageType>(x_payload))
+              .get_val();
+      }
+
+      if (y_bits.is_quiet_nan()) {
+        InStorageType y_payload = static_cast<InStorageType>(getpayload(y));
+        if ((y_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
+          return OutFPBits::quiet_nan(y_bits.sign(),
+                                      static_cast<OutStorageType>(y_payload))
+              .get_val();
+      }
+
+      return OutFPBits::quiet_nan().get_val();
+    }
+
+    if (x_bits.is_inf()) {
+      if (y_bits.is_zero()) {
+        set_errno_if_required(EDOM);
+        raise_except_if_required(FE_INVALID);
+        return OutFPBits::quiet_nan().get_val();
+      }
+
+      return OutFPBits::inf(result_sign).get_val();
+    }
+
+    if (y_bits.is_inf()) {
+      if (x_bits.is_zero()) {
+        set_errno_if_required(EDOM);
+        raise_except_if_required(FE_INVALID);
+        return OutFPBits::quiet_nan().get_val();
+      }
+
+      return OutFPBits::inf(result_sign).get_val();
+    }
+
+    // Now either x or y is zero, and the other one is finite.
+    return OutFPBits::zero(result_sign).get_val();
+  }
+
+  DyadicFloat xd(x);
+  DyadicFloat yd(y);
+
+  DyadicFloat result = quick_mul(xd, yd);
+  return result.template as<OutType, /*ShouldSignalExceptions=*/true>();
+}
+
+} // namespace fputil::generic
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_GENERIC_MUL_H
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index dc2339896f2bb..c4e33130e9090 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -86,6 +86,9 @@ add_math_entrypoint_object(cosh)
 add_math_entrypoint_object(coshf)
 add_math_entrypoint_object(cospif)
 
+add_math_entrypoint_object(dmull)
+add_math_entrypoint_object(dmulf128)
+
 add_math_entrypoint_object(erf)
 add_math_entrypoint_object(erff)
 
@@ -118,6 +121,11 @@ add_math_entrypoint_object(f16fmaf)
 add_math_entrypoint_object(f16fmal)
 add_math_entrypoint_object(f16fmaf128)
 
+add_math_entrypoint_object(f16mul)
+add_math_entrypoint_object(f16mulf)
+add_math_entrypoint_object(f16mull)
+add_math_entrypoint_object(f16mulf128)
+
 add_math_entrypoint_object(f16sqrt)
 add_math_entrypoint_object(f16sqrtf)
 add_math_entrypoint_object(f16sqrtl)
@@ -210,6 +218,8 @@ add_math_entrypoint_object(fminimum_mag_numf16)
 add_math_entrypoint_object(fminimum_mag_numf128)
 
 add_math_entrypoint_object(fmul)
+add_math_entrypoint_object(fmull)
+add_math_entrypoint_object(fmulf128)
 
 add_math_entrypoint_object(fmod)
 add_math_entrypoint_object(fmodf)
diff --git a/libc/src/math/dmulf128.h b/libc/src/math/dmulf128.h
new file mode 100644
index 0000000000000..623f22c910a41
--- /dev/null
+++ b/libc/src/math/dmulf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for dmulf128 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_DMULF128_H
+#define LLVM_LIBC_SRC_MATH_DMULF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+double dmulf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_DMULF128_H
diff --git a/libc/src/math/dmull.h b/libc/src/math/dmull.h
new file mode 100644
index 0000000000000..656776a603009
--- /dev/null
+++ b/libc/src/math/dmull.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for dmull -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_DMULL_H
+#define LLVM_LIBC_SRC_MATH_DMULL_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+double dmull(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_DMULL_H
diff --git a/libc/src/math/f16mul.h b/libc/src/math/f16mul.h
new file mode 100644
index 0000000000000..89403cf219271
--- /dev/null
+++ b/libc/src/math/f16mul.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for f16mul ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_F16MUL_H
+#define LLVM_LIBC_SRC_MATH_F16MUL_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float16 f16mul(double x, double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_F16MUL_H
diff --git a/libc/src/math/f16mulf.h b/libc/src/math/f16mulf.h
new file mode 100644
index 0000000000000..755886d6f14d0
--- /dev/null
+++ b/libc/src/math/f16mulf.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for f16mulf -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_F16MULF_H
+#define LLVM_LIBC_SRC_MATH_F16MULF_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float16 f16mulf(float x, float y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_F16MULF_H
diff --git a/libc/src/math/f16mulf128.h b/libc/src/math/f16mulf128.h
new file mode 100644
index 0000000000000..14371c57ca88c
--- /dev/null
+++ b/libc/src/math/f16mulf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for f16mulf128 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_F16MULF128_H
+#define LLVM_LIBC_SRC_MATH_F16MULF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float16 f16mulf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_F16MULF128_H
diff --git a/libc/src/math/f16mull.h b/libc/src/math/f16mull.h
new file mode 100644
index 0000000000000..a3177cadc1306
--- /dev/null
+++ b/libc/src/math/f16mull.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for f16mull -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_F16MULL_H
+#define LLVM_LIBC_SRC_MATH_F16MULL_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float16 f16mull(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_F16MULL_H
diff --git a/libc/src/math/fmulf128.h b/libc/src/math/fmulf128.h
new file mode 100644
index 0000000000000..94137ae87eb1e
--- /dev/null
+++ b/libc/src/math/fmulf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fmulf128 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMULF128_H
+#define LLVM_LIBC_SRC_MATH_FMULF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float fmulf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMULF128_H
diff --git a/libc/src/math/fmull.h b/libc/src/math/fmull.h
new file mode 100644
index 0000000000000..46e6c77cc66a2
--- /dev/null
+++ b/libc/src/math/fmull.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fmull -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMULL_H
+#define LLVM_LIBC_SRC_MATH_FMULL_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float fmull(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMULL_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 415ca3fbce796..9c86bac4a0cb7 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -2633,11 +2633,32 @@ add_entrypoint_object(
   HDRS
     ../fmul.h
   DEPENDS
-    libc.src.__support.FPUtil.basic_operations
-    libc.src.__support.uint128
-    libc.src.__support.CPP.bit
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.rounding_mode
+    libc.src.__support.FPUtil.generic.mul
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fmull
+  SRCS
+    fmull.cpp
+  HDRS
+    ../fmull.h
+  DEPENDS
+    libc.src.__support.FPUtil.generic.mul
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fmulf128
+  SRCS
+    fmulf128.cpp
+  HDRS
+    ../fmulf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.generic.mul
   COMPILE_OPTIONS
     -O3
 )
@@ -4200,3 +4221,80 @@ add_entrypoint_object(
     libc.src.__support.macros.optimization
     libc.src.__support.integer_literals
 )
+
+add_entrypoint_object(
+  dmull
+  SRCS
+    dmull.cpp
+  HDRS
+    ../dmull.h
+  DEPENDS
+    libc.src.__support.FPUtil.generic.mul
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  dmulf128
+  SRCS
+    dmulf128.cpp
+  HDRS
+    ../dmulf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.generic.mul
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  f16mul
+  SRCS
+    f16mul.cpp
+  HDRS
+    ../f16mul.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.generic.mul
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  f16mulf
+  SRCS
+    f16mulf.cpp
+  HDRS
+    ../f16mulf.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.generic.mul
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  f16mull
+  SRCS
+    f16mull.cpp
+  HDRS
+    ../f16mull.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.generic.mul
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  f16mulf128
+  SRCS
+    f16mulf128.cpp
+  HDRS
+    ../f16mulf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.generic.mul
+  COMPILE_OPTIONS
+    -O3
+)
diff --git a/libc/src/math/generic/dmulf128.cpp b/libc/src/math/generic/dmulf128.cpp
new file mode 100644
index 0000000000000..7e6ef95362c09
--- /dev/null
+++ b/libc/src/math/generic/dmulf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of dmulf128 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/dmulf128.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(double, dmulf128, (float128 x, float128 y)) {
+  return fputil::generic::mul<double>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/dmull.cpp b/libc/src/math/generic/dmull.cpp
new file mode 100644
index 0000000000000..428caa84a9977
--- /dev/null
+++ b/libc/src/math/generic/dmull.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of dmull function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/dmull.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(double, dmull, (long double x, long double y)) {
+  return fputil::generic::mul<double>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/f16mul.cpp b/libc/src/math/generic/f16mul.cpp
new file mode 100644
index 0000000000000..f7a5225b60f11
--- /dev/null
+++ b/libc/src/math/generic/f16mul.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of f16mul function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/f16mul.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float16, f16mul, (double x, double y)) {
+  return fputil::generic::mul<float16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/f16mulf.cpp b/libc/src/math/generic/f16mulf.cpp
new file mode 100644
index 0000000000000..2c04664f804ea
--- /dev/null
+++ b/libc/src/math/generic/f16mulf.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of f16mulf function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/f16mulf.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float16, f16mulf, (float x, float y)) {
+  return fputil::generic::mul<float16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/f16mulf128.cpp b/libc/src/math/generic/f16mulf128.cpp
new file mode 100644
index 0000000000000..7e2d6a0d194ae
--- /dev/null
+++ b/libc/src/math/generic/f16mulf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of f16mulf128 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/f16mulf128.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float16, f16mulf128, (float128 x, float128 y)) {
+  return fputil::generic::mul<float16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/f16mull.cpp b/libc/src/math/generic/f16mull.cpp
new file mode 100644
index 0000000000000..fc66fba4d9f23
--- /dev/null
+++ b/libc/src/math/generic/f16mull.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of f16mull function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/f16mull.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float16, f16mull, (long double x, long double y)) {
+  return fputil::generic::mul<float16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fmul.cpp b/libc/src/math/generic/fmul.cpp
index 16fa11e93df09..64c27d6e2f956 100644
--- a/libc/src/math/generic/fmul.cpp
+++ b/libc/src/math/generic/fmul.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of fmul function------------------------------------===//
+//===-- Implementation of fmul function -----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,123 +7,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/fmul.h"
-#include "src/__support/CPP/bit.h"
-#include "src/__support/FPUtil/BasicOperations.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/rounding_mode.h"
+#include "src/__support/FPUtil/generic/mul.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
-#include "src/__support/uint128.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(float, fmul, (double x, double y)) {
-  auto x_bits = fputil::FPBits<double>(x);
-
-  auto y_bits = fputil::FPBits<double>(y);
-
-  auto output_sign = (x_bits.sign() != y_bits.sign()) ? Sign::NEG : Sign::POS;
-
-  if (LIBC_UNLIKELY(x_bits.is_inf_or_nan() || y_bits.is_inf_or_nan() ||
-                    x_bits.is_zero() || y_bits.is_zero())) {
-    if (x_bits.is_nan())
-      return static_cast<float>(x);
-    if (y_bits.is_nan())
-      return static_cast<float>(y);
-    if (x_bits.is_inf())
-      return y_bits.is_zero()
-                 ? fputil::FPBits<float>::quiet_nan().get_val()
-                 : fputil::FPBits<float>::inf(output_sign).get_val();
-    if (y_bits.is_inf())
-      return x_bits.is_zero()
-                 ? fputil::FPBits<float>::quiet_nan().get_val()
-                 : fputil::FPBits<float>::inf(output_sign).get_val();
-    // Now either x or y is zero, and the other one is finite.
-    return fputil::FPBits<float>::zero(output_sign).get_val();
-  }
-
-  uint64_t mx, my;
-
-  // Get mantissa and append the hidden bit if needed.
-  mx = x_bits.get_explicit_mantissa();
-  my = y_bits.get_explicit_mantissa();
-
-  // Get the corresponding biased exponent.
-  int ex = x_bits.get_explicit_exponent();
-  int ey = y_bits.get_explicit_exponent();
-
-  // Count the number of leading zeros of the explicit mantissas.
-  int nx = cpp::countl_zero(mx);
-  int ny = cpp::countl_zero(my);
-  // Shift the leading 1 bit to the most significant bit.
-  mx <<= nx;
-  my <<= ny;
-
-  // Adjust exponent accordingly: If x or y are normal, we will only need to
-  // shift by (exponent length + sign bit = 11 bits. If x or y are denormal, we
-  // will need to shift more than 11 bits.
-  ex -= (nx - 11);
-  ey -= (ny - 11);
-
-  UInt128 product = static_cast<UInt128>(mx) * static_cast<UInt128>(my);
-  int32_t dm1;
-  uint64_t highs, lows;
-  uint64_t g, hight, lowt;
-  uint32_t m;
-  uint32_t b;
-  int c;
-
-  highs = static_cast<uint64_t>(product >> 64);
-  c = static_cast<int>(highs >= 0x8000000000000000);
-  lows = static_cast<uint64_t>(product);
-
-  lowt = (lows != 0);
-
-  dm1 = ex + ey + c + fputil::FPBits<float>::EXP_BIAS;
-
-  int round_mode = fputil::quick_get_round();
-  if (dm1 >= 255) {
-    if ((round_mode == FE_TOWARDZERO) ||
-        (round_mode == FE_UPWARD && output_sign.is_neg()) ||
-        (round_mode == FE_DOWNWARD && output_sign.is_pos())) {
-      return fputil::FPBits<float>::max_normal(output_sign).get_val();
-    }
-    return fputil::FPBits<float>::inf().get_val();
-  } else if (dm1 <= 0) {
-
-    int m_shift = 40 + c - dm1;
-    int g_shift = m_shift - 1;
-    int h_shift = 64 - g_shift;
-    m = (m_shift >= 64) ? 0 : static_cast<uint32_t>(highs >> m_shift);
-
-    g = g_shift >= 64 ? 0 : (highs >> g_shift) & 1;
-    hight = h_shift >= 64 ? highs : (highs << h_shift) != 0;
-
-    dm1 = 0;
-  } else {
-    m = static_cast<uint32_t>(highs >> (39 + c));
-    g = (highs >> (38 + c)) & 1;
-    hight = (highs << (26 - c)) != 0;
-  }
-
-  if (round_mode == FE_TONEAREST) {
-    b = g && ((hight && lowt) || ((m & 1) != 0));
-  } else if ((output_sign.is_neg() && round_mode == FE_DOWNWARD) ||
-             (output_sign.is_pos() && round_mode == FE_UPWARD)) {
-    b = (g == 0 && (hight && lowt) == 0) ? 0 : 1;
-  } else {
-    b = 0;
-  }
-
-  uint32_t exp16 = (dm1 << 23);
-
-  uint32_t m2 = m & fputil::FPBits<float>::FRACTION_MASK;
-
-  uint32_t result = (exp16 + m2) + b;
-
-  auto result_bits = fputil::FPBits<float>(result);
-  result_bits.set_sign(output_sign);
-  return result_bits.get_val();
+  return fputil::generic::mul<float>(x, y);
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fmulf128.cpp b/libc/src/math/generic/fmulf128.cpp
new file mode 100644
index 0000000000000..c0c55ace641b8
--- /dev/null
+++ b/libc/src/math/generic/fmulf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fmulf128 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmulf128.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float, fmulf128, (float128 x, float128 y)) {
+  return fputil::generic::mul<float>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fmull.cpp b/libc/src/math/generic/fmull.cpp
new file mode 100644
index 0000000000000..41ab165e7d09d
--- /dev/null
+++ b/libc/src/math/generic/fmull.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fmull function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmull.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float, fmull, (long double x, long double y)) {
+  return fputil::generic::mul<float>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 64b4d2c58fb6a..c28385f620cfd 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -1847,10 +1847,28 @@ add_fp_unittest(
   SRCS
     fmul_test.cpp
   HDRS
-    FMulTest.h
+    MulTest.h
   DEPENDS
     libc.src.math.fmul
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
 )
+
+add_fp_unittest(
+  fmull_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    fmull_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.src.math.fmull
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+)
+
 add_fp_unittest(
   asinhf_test
   NEED_MPFR
@@ -2237,6 +2255,66 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  dmull_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    dmull_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.src.math.dmull
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+)
+
+add_fp_unittest(
+  f16mul_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    f16mul_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.src.math.f16mul
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+)
+
+add_fp_unittest(
+  f16mulf_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    f16mulf_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.src.math.f16mulf
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+)
+
+add_fp_unittest(
+  f16mull_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    f16mull_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.src.math.f16mull
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+)
+
 add_subdirectory(generic)
 add_subdirectory(smoke)
 
diff --git a/libc/test/src/math/FMulTest.h b/libc/test/src/math/FMulTest.h
deleted file mode 100644
index 8ca33ea71b712..0000000000000
--- a/libc/test/src/math/FMulTest.h
+++ /dev/null
@@ -1,121 +0,0 @@
-//===-- Utility class to test fmul[f|l] -------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_TEST_SRC_MATH_FMULTEST_H
-#define LLVM_LIBC_TEST_SRC_MATH_FMULTEST_H
-
-#include "src/__support/FPUtil/FPBits.h"
-#include "test/UnitTest/FEnvSafeTest.h"
-#include "test/UnitTest/FPMatcher.h"
-#include "test/UnitTest/Test.h"
-#include "utils/MPFRWrapper/MPFRUtils.h"
-
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
-
-template <typename OutType, typename InType>
-class FmulMPFRTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
-
-  DECLARE_SPECIAL_CONSTANTS(InType)
-
-public:
-  typedef OutType (*FMulFunc)(InType, InType);
-
-  void testFMulMPFR(FMulFunc func) {
-    constexpr int N = 10;
-    mpfr::BinaryInput<InType> INPUTS[N] = {
-        {3.0, 5.0},
-        {0x1.0p1, 0x1.0p-131},
-        {0x1.0p2, 0x1.0p-129},
-        {1.0, 1.0},
-        {-0.0, -0.0},
-        {-0.0, 0.0},
-        {0.0, -0.0},
-        {0x1.0p100, 0x1.0p100},
-        {1.0, 1.0 + 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150},
-        {1.0, 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150}};
-
-    for (int i = 0; i < N; ++i) {
-      InType x = INPUTS[i].x;
-      InType y = INPUTS[i].y;
-      ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Fmul, INPUTS[i],
-                                     func(x, y), 0.5);
-    }
-  }
-
-  void testSpecialInputsMPFR(FMulFunc func) {
-    constexpr int N = 27;
-    mpfr::BinaryInput<InType> INPUTS[N] = {{inf, 0x1.0p-129},
-                                           {0x1.0p-129, inf},
-                                           {inf, 2.0},
-                                           {3.0, inf},
-                                           {0.0, 0.0},
-                                           {neg_inf, aNaN},
-                                           {aNaN, neg_inf},
-                                           {neg_inf, neg_inf},
-                                           {0.0, neg_inf},
-                                           {neg_inf, 0.0},
-                                           {neg_inf, 1.0},
-                                           {1.0, neg_inf},
-                                           {neg_inf, 0x1.0p-129},
-                                           {0x1.0p-129, neg_inf},
-                                           {0.0, 0x1.0p-129},
-                                           {inf, 0.0},
-                                           {0.0, inf},
-                                           {0.0, aNaN},
-                                           {2.0, aNaN},
-                                           {0x1.0p-129, aNaN},
-                                           {inf, aNaN},
-                                           {aNaN, aNaN},
-                                           {0.0, sNaN},
-                                           {2.0, sNaN},
-                                           {0x1.0p-129, sNaN},
-                                           {inf, sNaN},
-                                           {sNaN, sNaN}};
-
-    for (int i = 0; i < N; ++i) {
-      InType x = INPUTS[i].x;
-      InType y = INPUTS[i].y;
-      ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Fmul, INPUTS[i],
-                                     func(x, y), 0.5);
-    }
-  }
-
-  void testNormalRange(FMulFunc func) {
-    using FPBits = LIBC_NAMESPACE::fputil::FPBits<InType>;
-    using StorageType = typename FPBits::StorageType;
-    static constexpr StorageType MAX_NORMAL = FPBits::max_normal().uintval();
-    static constexpr StorageType MIN_NORMAL = FPBits::min_normal().uintval();
-
-    constexpr StorageType COUNT = 10'001;
-    constexpr StorageType STEP = (MAX_NORMAL - MIN_NORMAL) / COUNT;
-    for (int signs = 0; signs < 4; ++signs) {
-      for (StorageType v = MIN_NORMAL, w = MAX_NORMAL;
-           v <= MAX_NORMAL && w >= MIN_NORMAL; v += STEP, w -= STEP) {
-        InType x = FPBits(v).get_val(), y = FPBits(w).get_val();
-        if (signs % 2 == 1) {
-          x = -x;
-        }
-        if (signs >= 2) {
-          y = -y;
-        }
-
-        mpfr::BinaryInput<InType> input{x, y};
-        ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Fmul, input, func(x, y),
-                                       0.5);
-      }
-    }
-  }
-};
-
-#define LIST_FMUL_MPFR_TESTS(OutType, InType, func)                            \
-  using LlvmLibcFmulTest = FmulMPFRTest<OutType, InType>;                      \
-  TEST_F(LlvmLibcFmulTest, MulMpfr) { testFMulMPFR(&func); }                   \
-  TEST_F(LlvmLibcFmulTest, NanInfMpfr) { testSpecialInputsMPFR(&func); }       \
-  TEST_F(LlvmLibcFmulTest, NormalRange) { testNormalRange(&func); }
-
-#endif // LLVM_LIBC_TEST_SRC_MATH_FMULTEST_H
diff --git a/libc/test/src/math/MulTest.h b/libc/test/src/math/MulTest.h
new file mode 100644
index 0000000000000..cb81a795be36b
--- /dev/null
+++ b/libc/test/src/math/MulTest.h
@@ -0,0 +1,95 @@
+//===-- Utility class to test different flavors of float mul ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_MULTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_MULTEST_H
+
+#include "src/stdlib/rand.h"
+#include "src/stdlib/srand.h"
+#include "test/UnitTest/FEnvSafeTest.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+
+template <typename OutType, typename InType>
+class MulTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
+
+  struct InConstants {
+    DECLARE_SPECIAL_CONSTANTS(InType)
+  };
+
+  using InFPBits = typename InConstants::FPBits;
+  using InStorageType = typename InConstants::StorageType;
+
+  static constexpr InStorageType IN_MAX_NORMAL_U =
+      InFPBits::max_normal().uintval();
+  static constexpr InStorageType IN_MIN_NORMAL_U =
+      InFPBits::min_normal().uintval();
+  static constexpr InStorageType IN_MAX_SUBNORMAL_U =
+      InFPBits::max_subnormal().uintval();
+  static constexpr InStorageType IN_MIN_SUBNORMAL_U =
+      InFPBits::min_subnormal().uintval();
+
+  InStorageType get_random_bit_pattern() {
+    InStorageType bits{0};
+    for (InStorageType i = 0; i < sizeof(InStorageType) / 2; ++i)
+      bits = (bits << 2) + static_cast<uint16_t>(LIBC_NAMESPACE::rand());
+    return bits;
+  }
+
+public:
+  using MulFunc = OutType (*)(InType, InType);
+
+  void test_subnormal_range(MulFunc func) {
+    constexpr InStorageType COUNT = 10'001;
+    constexpr InStorageType STEP =
+        (IN_MAX_SUBNORMAL_U - IN_MIN_SUBNORMAL_U) / COUNT;
+    LIBC_NAMESPACE::srand(1);
+    for (int signs = 0; signs < 4; signs++) {
+      for (InStorageType i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
+        InType x = InFPBits(get_random_bit_pattern()).get_val();
+        InType y = InFPBits(v).get_val();
+        if ((signs & 1) != 0)
+          x = -x;
+        if ((signs & 2) != 0)
+          y = -y;
+        mpfr::BinaryInput<InType> input{x, y};
+        EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Mul, input, func(x, y),
+                                       0.5);
+      }
+    }
+  }
+
+  void test_normal_range(MulFunc func) {
+    constexpr InStorageType COUNT = 10'001;
+    constexpr InStorageType STEP = (IN_MAX_NORMAL_U - IN_MIN_NORMAL_U) / COUNT;
+    LIBC_NAMESPACE::srand(1);
+    for (int signs = 0; signs < 4; signs++) {
+      for (InStorageType i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
+        InType x = InFPBits(get_random_bit_pattern()).get_val();
+        InType y = InFPBits(v).get_val();
+        if ((signs & 1) != 0)
+          x = -x;
+        if ((signs & 2) != 0)
+          y = -y;
+        mpfr::BinaryInput<InType> input{x, y};
+        EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Mul, input, func(x, y),
+                                       0.5);
+      }
+    }
+  }
+};
+
+#define LIST_MUL_TESTS(OutType, InType, func)                                  \
+  using LlvmLibcMulTest = MulTest<OutType, InType>;                            \
+  TEST_F(LlvmLibcMulTest, SubnormalRange) { test_subnormal_range(&func); }     \
+  TEST_F(LlvmLibcMulTest, NormalRange) { test_normal_range(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_MULTEST_H
diff --git a/libc/test/src/math/dmull_test.cpp b/libc/test/src/math/dmull_test.cpp
new file mode 100644
index 0000000000000..1b9c9c2c24ed3
--- /dev/null
+++ b/libc/test/src/math/dmull_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for dmull -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/dmull.h"
+
+LIST_MUL_TESTS(double, long double, LIBC_NAMESPACE::dmull)
diff --git a/libc/test/src/math/f16mul_test.cpp b/libc/test/src/math/f16mul_test.cpp
new file mode 100644
index 0000000000000..49b443870c483
--- /dev/null
+++ b/libc/test/src/math/f16mul_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for f16mul ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/f16mul.h"
+
+LIST_MUL_TESTS(float16, double, LIBC_NAMESPACE::f16mul)
diff --git a/libc/test/src/math/f16mulf_test.cpp b/libc/test/src/math/f16mulf_test.cpp
new file mode 100644
index 0000000000000..bf2530863621d
--- /dev/null
+++ b/libc/test/src/math/f16mulf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for f16mulf ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/f16mulf.h"
+
+LIST_MUL_TESTS(float16, float, LIBC_NAMESPACE::f16mulf)
diff --git a/libc/test/src/math/f16mull_test.cpp b/libc/test/src/math/f16mull_test.cpp
new file mode 100644
index 0000000000000..5292ddb87b7f4
--- /dev/null
+++ b/libc/test/src/math/f16mull_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for f16mull ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/f16mull.h"
+
+LIST_MUL_TESTS(float16, long double, LIBC_NAMESPACE::f16mull)
diff --git a/libc/test/src/math/fmul_test.cpp b/libc/test/src/math/fmul_test.cpp
index 16eaa1a818daf..3f6df66456bac 100644
--- a/libc/test/src/math/fmul_test.cpp
+++ b/libc/test/src/math/fmul_test.cpp
@@ -1,13 +1,13 @@
-//===-- Unittests for fmul-------------------------------------------------===//
+//===-- Unittests for fmul ------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//===---------------------------------------------------------------------===//
+//===----------------------------------------------------------------------===//
 
-#include "FMulTest.h"
+#include "MulTest.h"
 
 #include "src/math/fmul.h"
 
-LIST_FMUL_MPFR_TESTS(float, double, LIBC_NAMESPACE::fmul)
+LIST_MUL_TESTS(float, double, LIBC_NAMESPACE::fmul)
diff --git a/libc/test/src/math/fmull_test.cpp b/libc/test/src/math/fmull_test.cpp
new file mode 100644
index 0000000000000..ef694063f20dc
--- /dev/null
+++ b/libc/test/src/math/fmull_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmull -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/fmull.h"
+
+LIST_MUL_TESTS(float, long double, LIBC_NAMESPACE::fmull)
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index 76d5919ad9156..c57aa9638ed30 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -2531,10 +2531,27 @@ add_fp_unittest(
   SRCS
     fmul_test.cpp
   HDRS
-    FMulTest.h
+    MulTest.h
   DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.basic_operations
     libc.src.math.fmul
-    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmull_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmull_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.math.fmull
 )
 
 add_fp_unittest(
@@ -3981,3 +3998,78 @@ add_fp_unittest(
   DEPENDS
     libc.src.math.cbrt
 )
+
+add_fp_unittest(
+  dmull_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    dmull_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.math.dmull
+)
+
+add_fp_unittest(
+  f16mul_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    f16mul_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.math.f16mul
+)
+
+add_fp_unittest(
+  f16mulf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    f16mulf_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.math.f16mulf
+)
+
+add_fp_unittest(
+  f16mull_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    f16mull_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.math.f16mull
+)
+
+add_fp_unittest(
+  f16mulf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    f16mulf128_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.math.f16mulf128
+)
diff --git a/libc/test/src/math/smoke/FMulTest.h b/libc/test/src/math/smoke/FMulTest.h
deleted file mode 100644
index 33fb82c8d2da1..0000000000000
--- a/libc/test/src/math/smoke/FMulTest.h
+++ /dev/null
@@ -1,104 +0,0 @@
-//===-- Utility class to test fmul[f|l] ---------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMULTEST_H
-#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMULTEST_H
-
-#include "test/UnitTest/FEnvSafeTest.h"
-#include "test/UnitTest/FPMatcher.h"
-#include "test/UnitTest/Test.h"
-
-template <typename T, typename R>
-class FmulTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
-
-  DECLARE_SPECIAL_CONSTANTS(T)
-
-public:
-  typedef T (*FMulFunc)(R, R);
-
-  void testMul(FMulFunc func) {
-
-    EXPECT_FP_EQ_ALL_ROUNDING(T(15.0), func(3.0, 5.0));
-    EXPECT_FP_EQ_ALL_ROUNDING(T(0x1.0p-130), func(0x1.0p1, 0x1.0p-131));
-    EXPECT_FP_EQ_ALL_ROUNDING(T(0x1.0p-127), func(0x1.0p2, 0x1.0p-129));
-    EXPECT_FP_EQ_ALL_ROUNDING(T(1.0), func(1.0, 1.0));
-
-    EXPECT_FP_EQ_ALL_ROUNDING(T(0.0), func(-0.0, -0.0));
-    EXPECT_FP_EQ_ALL_ROUNDING(T(-0.0), func(0.0, -0.0));
-    EXPECT_FP_EQ_ALL_ROUNDING(T(-0.0), func(-0.0, 0.0));
-
-    EXPECT_FP_EQ_ROUNDING_NEAREST(inf, func(0x1.0p100, 0x1.0p100));
-    EXPECT_FP_EQ_ROUNDING_UPWARD(inf, func(0x1.0p100, 0x1.0p100));
-    EXPECT_FP_EQ_ROUNDING_DOWNWARD(max_normal, func(0x1.0p100, 0x1.0p100));
-    EXPECT_FP_EQ_ROUNDING_TOWARD_ZERO(max_normal, func(0x1.0p100, 0x1.0p100));
-
-    EXPECT_FP_EQ_ROUNDING_NEAREST(
-        0x1p0, func(1.0, 1.0 + 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
-    EXPECT_FP_EQ_ROUNDING_DOWNWARD(
-        0x1p0, func(1.0, 1.0 + 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
-    EXPECT_FP_EQ_ROUNDING_TOWARD_ZERO(
-        0x1p0, func(1.0, 1.0 + 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
-    EXPECT_FP_EQ_ROUNDING_UPWARD(
-        0x1p0, func(1.0, 1.0 + 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
-
-    EXPECT_FP_EQ_ROUNDING_NEAREST(
-        0x1.0p-128f + 0x1.0p-148f,
-        func(1.0, 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
-    EXPECT_FP_EQ_ROUNDING_UPWARD(
-        0x1.0p-128f + 0x1.0p-148f,
-        func(1.0, 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
-    EXPECT_FP_EQ_ROUNDING_DOWNWARD(
-        0x1.0p-128f + 0x1.0p-149f,
-        func(1.0, 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
-    EXPECT_FP_EQ_ROUNDING_TOWARD_ZERO(
-        0x1.0p-128f + 0x1.0p-149f,
-        func(1.0, 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
-  }
-
-  void testSpecialInputs(FMulFunc func) {
-    EXPECT_FP_EQ_ALL_ROUNDING(inf, func(inf, 0x1.0p-129));
-    EXPECT_FP_EQ_ALL_ROUNDING(inf, func(0x1.0p-129, inf));
-    EXPECT_FP_EQ_ALL_ROUNDING(inf, func(inf, 2.0));
-    EXPECT_FP_EQ_ALL_ROUNDING(inf, func(3.0, inf));
-    EXPECT_FP_EQ_ALL_ROUNDING(0.0, func(0.0, 0.0));
-
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(neg_inf, aNaN));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(aNaN, neg_inf));
-    EXPECT_FP_EQ_ALL_ROUNDING(inf, func(neg_inf, neg_inf));
-
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0.0, neg_inf));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(neg_inf, 0.0));
-
-    EXPECT_FP_EQ_ALL_ROUNDING(neg_inf, func(neg_inf, 1.0));
-    EXPECT_FP_EQ_ALL_ROUNDING(neg_inf, func(1.0, neg_inf));
-
-    EXPECT_FP_EQ_ALL_ROUNDING(neg_inf, func(neg_inf, 0x1.0p-129));
-    EXPECT_FP_EQ_ALL_ROUNDING(neg_inf, func(0x1.0p-129, neg_inf));
-
-    EXPECT_FP_EQ_ALL_ROUNDING(0.0, func(0.0, 0x1.0p-129));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(inf, 0.0));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0.0, inf));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0.0, aNaN));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(2.0, aNaN));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0x1.0p-129, aNaN));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(inf, aNaN));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(aNaN, aNaN));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0.0, sNaN));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(2.0, sNaN));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0x1.0p-129, sNaN));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(inf, sNaN));
-    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(sNaN, sNaN));
-  }
-};
-
-#define LIST_FMUL_TESTS(T, R, func)                                            \
-  using LlvmLibcFmulTest = FmulTest<T, R>;                                     \
-  TEST_F(LlvmLibcFmulTest, Mul) { testMul(&func); }                            \
-  TEST_F(LlvmLibcFmulTest, NaNInf) { testSpecialInputs(&func); }
-
-#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMULTEST_H
diff --git a/libc/test/src/math/smoke/MulTest.h b/libc/test/src/math/smoke/MulTest.h
new file mode 100644
index 0000000000000..e2298eaeeb216
--- /dev/null
+++ b/libc/test/src/math/smoke/MulTest.h
@@ -0,0 +1,171 @@
+//===-- Utility class to test different flavors of float mul ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_MULTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_MULTEST_H
+
+#include "hdr/errno_macros.h"
+#include "hdr/fenv_macros.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "test/UnitTest/FEnvSafeTest.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename OutType, typename InType>
+class MulTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
+
+  DECLARE_SPECIAL_CONSTANTS(OutType)
+
+  struct InConstants {
+    DECLARE_SPECIAL_CONSTANTS(InType)
+  };
+
+  using InFPBits = typename InConstants::FPBits;
+  using InStorageType = typename InConstants::StorageType;
+
+  InConstants in;
+
+public:
+  using MulFunc = OutType (*)(InType, InType);
+
+  void test_special_numbers(MulFunc func) {
+    EXPECT_FP_IS_NAN(func(aNaN, aNaN));
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(sNaN, sNaN), FE_INVALID);
+
+    InType qnan_42 = InFPBits::quiet_nan(Sign::POS, 0x42).get_val();
+    EXPECT_FP_EQ(InType(0x42.0p+0),
+                 LIBC_NAMESPACE::fputil::getpayload(func(qnan_42, zero)));
+    EXPECT_FP_EQ(InType(0x42.0p+0),
+                 LIBC_NAMESPACE::fputil::getpayload(func(zero, qnan_42)));
+
+    if constexpr (sizeof(OutType) < sizeof(InType)) {
+      InStorageType max_payload = InFPBits::FRACTION_MASK >> 1;
+      InType qnan_max = InFPBits::quiet_nan(Sign::POS, max_payload).get_val();
+      EXPECT_FP_EQ(zero,
+                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_max, zero)));
+      EXPECT_FP_EQ(zero,
+                   LIBC_NAMESPACE::fputil::getpayload(func(zero, qnan_max)));
+      EXPECT_FP_EQ(InType(0x42.0p+0),
+                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_max, qnan_42)));
+      EXPECT_FP_EQ(InType(0x42.0p+0),
+                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_42, qnan_max)));
+    }
+
+    EXPECT_FP_EQ(inf, func(inf, InType(1.0)));
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, InType(1.0)));
+    EXPECT_FP_EQ(neg_inf, func(inf, InType(-1.0)));
+    EXPECT_FP_EQ(inf, func(neg_inf, InType(-1.0)));
+
+    EXPECT_FP_EQ_ALL_ROUNDING(zero, func(zero, zero));
+    EXPECT_FP_EQ_ALL_ROUNDING(zero, func(neg_zero, neg_zero));
+    EXPECT_FP_EQ_ALL_ROUNDING(neg_zero, func(zero, neg_zero));
+    EXPECT_FP_EQ_ALL_ROUNDING(neg_zero, func(neg_zero, zero));
+
+    EXPECT_FP_EQ_ALL_ROUNDING(OutType(1.0), func(1.0, 1.0));
+    EXPECT_FP_EQ_ALL_ROUNDING(OutType(15.0), func(3.0, 5.0));
+    EXPECT_FP_EQ_ALL_ROUNDING(OutType(0x1.0p-13), func(0x1.0p+1, 0x1.0p-14));
+    EXPECT_FP_EQ_ALL_ROUNDING(OutType(0x1.0p-10), func(0x1.0p+2, 0x1.0p-12));
+  }
+
+  void test_invalid_operations(MulFunc func) {
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(inf, zero), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(inf, neg_zero), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(neg_inf, zero), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(neg_inf, neg_zero), FE_INVALID);
+  }
+
+  void test_range_errors(MulFunc func) {
+    using namespace LIBC_NAMESPACE::fputil::testing;
+
+    if (ForceRoundingMode r(RoundingMode::Nearest); r.success) {
+      EXPECT_FP_EQ_WITH_EXCEPTION(inf, func(max_normal, max_normal),
+                                  FE_OVERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+      EXPECT_FP_EQ_WITH_EXCEPTION(neg_inf, func(neg_max_normal, max_normal),
+                                  FE_OVERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+
+      EXPECT_FP_EQ_WITH_EXCEPTION(zero, func(in.min_denormal, in.min_denormal),
+                                  FE_UNDERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+      EXPECT_FP_EQ_WITH_EXCEPTION(neg_zero,
+                                  func(in.neg_min_denormal, in.min_denormal),
+                                  FE_UNDERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+    }
+
+    if (ForceRoundingMode r(RoundingMode::TowardZero); r.success) {
+      EXPECT_FP_EQ_WITH_EXCEPTION(max_normal, func(max_normal, max_normal),
+                                  FE_OVERFLOW | FE_INEXACT);
+      EXPECT_FP_EQ_WITH_EXCEPTION(neg_max_normal,
+                                  func(neg_max_normal, max_normal),
+                                  FE_OVERFLOW | FE_INEXACT);
+
+      EXPECT_FP_EQ_WITH_EXCEPTION(zero, func(in.min_denormal, in.min_denormal),
+                                  FE_UNDERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+      EXPECT_FP_EQ_WITH_EXCEPTION(neg_zero,
+                                  func(in.neg_min_denormal, in.min_denormal),
+                                  FE_UNDERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+    }
+
+    if (ForceRoundingMode r(RoundingMode::Downward); r.success) {
+      EXPECT_FP_EQ_WITH_EXCEPTION(max_normal, func(max_normal, max_normal),
+                                  FE_OVERFLOW | FE_INEXACT);
+      EXPECT_FP_EQ_WITH_EXCEPTION(neg_inf, func(neg_max_normal, max_normal),
+                                  FE_OVERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+
+      EXPECT_FP_EQ_WITH_EXCEPTION(zero, func(in.min_denormal, in.min_denormal),
+                                  FE_UNDERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+      EXPECT_FP_EQ_WITH_EXCEPTION(neg_min_denormal,
+                                  func(in.neg_min_denormal, in.min_denormal),
+                                  FE_UNDERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+    }
+
+    if (ForceRoundingMode r(RoundingMode::Upward); r.success) {
+      EXPECT_FP_EQ_WITH_EXCEPTION(inf, func(max_normal, max_normal),
+                                  FE_OVERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+      EXPECT_FP_EQ_WITH_EXCEPTION(neg_max_normal,
+                                  func(neg_max_normal, max_normal),
+                                  FE_OVERFLOW | FE_INEXACT);
+
+      EXPECT_FP_EQ_WITH_EXCEPTION(min_denormal,
+                                  func(in.min_denormal, in.min_denormal),
+                                  FE_UNDERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+      EXPECT_FP_EQ_WITH_EXCEPTION(neg_zero,
+                                  func(in.neg_min_denormal, in.min_denormal),
+                                  FE_UNDERFLOW | FE_INEXACT);
+      EXPECT_MATH_ERRNO(ERANGE);
+    }
+  }
+
+  void test_inexact_results(MulFunc func) {
+    InFPBits x_bits = InFPBits::one();
+    x_bits.set_mantissa(InFPBits::SIG_MASK);
+    InType x = x_bits.get_val();
+    func(x, x);
+    EXPECT_FP_EXCEPTION(FE_INEXACT);
+  }
+};
+
+#define LIST_MUL_TESTS(OutType, InType, func)                                  \
+  using LlvmLibcMulTest = MulTest<OutType, InType>;                            \
+  TEST_F(LlvmLibcMulTest, SpecialNumbers) { test_special_numbers(&func); }     \
+  TEST_F(LlvmLibcMulTest, InvalidOperations) {                                 \
+    test_invalid_operations(&func);                                            \
+  }                                                                            \
+  TEST_F(LlvmLibcMulTest, RangeErrors) { test_range_errors(&func); }           \
+  TEST_F(LlvmLibcMulTest, InexactResults) { test_inexact_results(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_MULTEST_H
diff --git a/libc/test/src/math/smoke/dmulf128_test.cpp b/libc/test/src/math/smoke/dmulf128_test.cpp
new file mode 100644
index 0000000000000..2ee2d10b9a19b
--- /dev/null
+++ b/libc/test/src/math/smoke/dmulf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for dmulf128 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/dmulf128.h"
+
+LIST_MUL_TESTS(double, float128, LIBC_NAMESPACE::dmulf128)
diff --git a/libc/test/src/math/smoke/dmull_test.cpp b/libc/test/src/math/smoke/dmull_test.cpp
new file mode 100644
index 0000000000000..1b9c9c2c24ed3
--- /dev/null
+++ b/libc/test/src/math/smoke/dmull_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for dmull -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/dmull.h"
+
+LIST_MUL_TESTS(double, long double, LIBC_NAMESPACE::dmull)
diff --git a/libc/test/src/math/smoke/f16mul_test.cpp b/libc/test/src/math/smoke/f16mul_test.cpp
new file mode 100644
index 0000000000000..49b443870c483
--- /dev/null
+++ b/libc/test/src/math/smoke/f16mul_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for f16mul ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/f16mul.h"
+
+LIST_MUL_TESTS(float16, double, LIBC_NAMESPACE::f16mul)
diff --git a/libc/test/src/math/smoke/f16mulf128_test.cpp b/libc/test/src/math/smoke/f16mulf128_test.cpp
new file mode 100644
index 0000000000000..46e52cf068a7b
--- /dev/null
+++ b/libc/test/src/math/smoke/f16mulf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for f16mulf128 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/f16mulf128.h"
+
+LIST_MUL_TESTS(float16, float128, LIBC_NAMESPACE::f16mulf128)
diff --git a/libc/test/src/math/smoke/f16mulf_test.cpp b/libc/test/src/math/smoke/f16mulf_test.cpp
new file mode 100644
index 0000000000000..bf2530863621d
--- /dev/null
+++ b/libc/test/src/math/smoke/f16mulf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for f16mulf ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/f16mulf.h"
+
+LIST_MUL_TESTS(float16, float, LIBC_NAMESPACE::f16mulf)
diff --git a/libc/test/src/math/smoke/f16mull_test.cpp b/libc/test/src/math/smoke/f16mull_test.cpp
new file mode 100644
index 0000000000000..5292ddb87b7f4
--- /dev/null
+++ b/libc/test/src/math/smoke/f16mull_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for f16mull ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/f16mull.h"
+
+LIST_MUL_TESTS(float16, long double, LIBC_NAMESPACE::f16mull)
diff --git a/libc/test/src/math/smoke/fmul_test.cpp b/libc/test/src/math/smoke/fmul_test.cpp
index 0eb664f7411ee..3f6df66456bac 100644
--- a/libc/test/src/math/smoke/fmul_test.cpp
+++ b/libc/test/src/math/smoke/fmul_test.cpp
@@ -1,13 +1,13 @@
-//===-- Unittests for fmul-------------------------------------------------===//
+//===-- Unittests for fmul ------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//===---------------------------------------------------------------------===//
+//===----------------------------------------------------------------------===//
 
-#include "FMulTest.h"
+#include "MulTest.h"
 
 #include "src/math/fmul.h"
 
-LIST_FMUL_TESTS(float, double, LIBC_NAMESPACE::fmul)
+LIST_MUL_TESTS(float, double, LIBC_NAMESPACE::fmul)
diff --git a/libc/test/src/math/smoke/fmulf128_test.cpp b/libc/test/src/math/smoke/fmulf128_test.cpp
new file mode 100644
index 0000000000000..37c8d1cf9908d
--- /dev/null
+++ b/libc/test/src/math/smoke/fmulf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmulf128 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/fmulf128.h"
+
+LIST_MUL_TESTS(float, float128, LIBC_NAMESPACE::fmulf128)
diff --git a/libc/test/src/math/smoke/fmull_test.cpp b/libc/test/src/math/smoke/fmull_test.cpp
new file mode 100644
index 0000000000000..ef694063f20dc
--- /dev/null
+++ b/libc/test/src/math/smoke/fmull_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmull -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/fmull.h"
+
+LIST_MUL_TESTS(float, long double, LIBC_NAMESPACE::fmull)
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp
index b67a9da40bd7b..bb63e0b9f4de0 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.cpp
+++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp
@@ -539,7 +539,7 @@ class MPFRNumber {
     return result;
   }
 
-  MPFRNumber fmul(const MPFRNumber &b) {
+  MPFRNumber mul(const MPFRNumber &b) {
     MPFRNumber result(*this);
     mpfr_mul(result.value, value, b.value, mpfr_rounding);
     return result;
@@ -800,12 +800,12 @@ binary_operation_one_output(Operation op, InputType x, InputType y,
     return inputX.fmod(inputY);
   case Operation::Hypot:
     return inputX.hypot(inputY);
+  case Operation::Mul:
+    return inputX.mul(inputY);
   case Operation::Pow:
     return inputX.pow(inputY);
   case Operation::Sub:
     return inputX.sub(inputY);
-  case Operation::Fmul:
-    return inputX.fmul(inputY);
   default:
     __builtin_unreachable();
   }
@@ -1013,15 +1013,18 @@ void explain_binary_operation_one_output_error(
 template void
 explain_binary_operation_one_output_error(Operation, const BinaryInput<float> &,
                                           float, double, RoundingMode);
+template void explain_binary_operation_one_output_error(
+    Operation, const BinaryInput<double> &, float, double, RoundingMode);
 template void explain_binary_operation_one_output_error(
     Operation, const BinaryInput<double> &, double, double, RoundingMode);
+template void explain_binary_operation_one_output_error(
+    Operation, const BinaryInput<long double> &, float, double, RoundingMode);
+template void explain_binary_operation_one_output_error(
+    Operation, const BinaryInput<long double> &, double, double, RoundingMode);
 template void
 explain_binary_operation_one_output_error(Operation,
                                           const BinaryInput<long double> &,
                                           long double, double, RoundingMode);
-
-template void explain_binary_operation_one_output_error(
-    Operation, const BinaryInput<double> &, float, double, RoundingMode);
 #ifdef LIBC_TYPES_HAS_FLOAT16
 template void explain_binary_operation_one_output_error(
     Operation, const BinaryInput<float16> &, float16, double, RoundingMode);
@@ -1195,6 +1198,12 @@ template bool compare_binary_operation_one_output(Operation,
                                                   const BinaryInput<double> &,
                                                   double, double, RoundingMode);
 template bool
+compare_binary_operation_one_output(Operation, const BinaryInput<long double> &,
+                                    float, double, RoundingMode);
+template bool
+compare_binary_operation_one_output(Operation, const BinaryInput<long double> &,
+                                    double, double, RoundingMode);
+template bool
 compare_binary_operation_one_output(Operation, const BinaryInput<long double> &,
                                     long double, double, RoundingMode);
 
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.h b/libc/utils/MPFRWrapper/MPFRUtils.h
index 28390af9ee6d8..8d51fa4e47726 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.h
+++ b/libc/utils/MPFRWrapper/MPFRUtils.h
@@ -79,9 +79,9 @@ enum class Operation : int {
   Div,
   Fmod,
   Hypot,
+  Mul,
   Pow,
   Sub,
-  Fmul,
   EndBinaryOperationsSingleOutput,
 
   // Operations which take two floating point numbers of the same type as

From b37bdadbe784339e455915368a4893d3bd4a1193 Mon Sep 17 00:00:00 2001
From: aaryanshukla <53713108+aaryanshukla@users.noreply.github.com>
Date: Thu, 18 Jul 2024 10:56:26 -0700
Subject: [PATCH 059/486] [libc] newheadergen: adding h_def_file arg to test
 (#99397)

- spacing with _NOEXCEPT
---
 .../class_implementation/classes/function.py  |  2 +-
 libc/newhdrgen/header.py                      |  8 +--
 libc/newhdrgen/tests/output/test_small.h      | 52 -------------------
 libc/newhdrgen/tests/test_integration.py      |  1 +
 4 files changed, 6 insertions(+), 57 deletions(-)
 delete mode 100644 libc/newhdrgen/tests/output/test_small.h

diff --git a/libc/newhdrgen/class_implementation/classes/function.py b/libc/newhdrgen/class_implementation/classes/function.py
index ccfd93547c1d8..845ef1aebf54b 100644
--- a/libc/newhdrgen/class_implementation/classes/function.py
+++ b/libc/newhdrgen/class_implementation/classes/function.py
@@ -26,7 +26,7 @@ def __str__(self):
         attributes_str = " ".join(self.attributes)
         arguments_str = ", ".join(self.arguments)
         if attributes_str == "":
-            result = f"{self.return_type} {self.name}({arguments_str});"
+            result = f"{self.return_type} {self.name}({arguments_str})"
         else:
             result = f"{attributes_str} {self.return_type} {self.name}({arguments_str})"
         return result
diff --git a/libc/newhdrgen/header.py b/libc/newhdrgen/header.py
index 69de81eebb719..141e3c9b2736b 100644
--- a/libc/newhdrgen/header.py
+++ b/libc/newhdrgen/header.py
@@ -60,16 +60,16 @@ def __str__(self):
         current_guard = None
         for function in self.functions:
             if function.guard == None:
-                content.append(str(function) + "__NOEXCEPT")
+                content.append(str(function) + " __NOEXCEPT;")
                 content.append("")
             else:
                 if current_guard == None:
                     current_guard = function.guard
                     content.append(f"#ifdef {current_guard}")
-                    content.append(str(function) + "__NOEXCEPT")
+                    content.append(str(function) + " __NOEXCEPT;")
                     content.append("")
                 elif current_guard == function.guard:
-                    content.append(str(function) + "__NOEXCEPT")
+                    content.append(str(function) + " __NOEXCEPT;")
                     content.append("")
                 else:
                     content.pop()
@@ -77,7 +77,7 @@ def __str__(self):
                     content.append("")
                     current_guard = function.guard
                     content.append(f"#ifdef {current_guard}")
-                    content.append(str(function) + "__NOEXCEPT")
+                    content.append(str(function) + " __NOEXCEPT;")
                     content.append("")
         if current_guard != None:
             content.pop()
diff --git a/libc/newhdrgen/tests/output/test_small.h b/libc/newhdrgen/tests/output/test_small.h
deleted file mode 100644
index a777976134b04..0000000000000
--- a/libc/newhdrgen/tests/output/test_small.h
+++ /dev/null
@@ -1,52 +0,0 @@
-//===-- C standard library header test_small-------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_TEST_SMALL_H
-#define LLVM_LIBC_TEST_SMALL_H
-
-#include "__llvm-libc-common.h"
-#include "llvm-libc-macros/float16-macros.h"
-#include "llvm-libc-macros/test_small-macros.h"
-#include "llvm-libc-types/float128.h"
-
-#define MACRO_A 1
-
-#define MACRO_B 2
-
-#include <llvm-libc-types/type_a.h>
-#include <llvm-libc-types/type_b.h>
-
-enum {
-  enum_a = value_1,
-  enum_b = value_2,
-};
-
-__BEGIN_C_DECLS
-
-CONST_FUNC_A void func_a() __NOEXCEPT;
-
-#ifdef LIBC_TYPES_HAS_FLOAT128
-float128 func_b() __NOEXCEPT;
-#endif // LIBC_TYPES_HAS_FLOAT128
-
-#ifdef LIBC_TYPES_HAS_FLOAT16
-_Float16 func_c(int, float) __NOEXCEPT;
-
-_Float16 func_d(int, float) __NOEXCEPT;
-#endif // LIBC_TYPES_HAS_FLOAT16
-
-#ifdef LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128
-_Float16 func_e(float128) __NOEXCEPT;
-#endif // LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128
-
-extern obj object_1;
-extern obj object_2;
-
-__END_C_DECLS
-
-#endif // LLVM_LIBC_TEST_SMALL_H
diff --git a/libc/newhdrgen/tests/test_integration.py b/libc/newhdrgen/tests/test_integration.py
index f12d18bc04a49..628a37b11c309 100644
--- a/libc/newhdrgen/tests/test_integration.py
+++ b/libc/newhdrgen/tests/test_integration.py
@@ -25,6 +25,7 @@ def run_script(self, yaml_file, h_def_file, output_dir):
                 "python3",
                 str(self.source_dir / "libc/newhdrgen/yaml_to_classes.py"),
                 str(yaml_file),
+                "--h_def_file",
                 str(h_def_file),
                 "--output_dir",
                 str(output_dir),

From 78e3bfc120c8a23e246f544a5e9fb122828a21a7 Mon Sep 17 00:00:00 2001
From: Vladislav Dzhidzhoev <vdzhidzhoev@accesssoftek.com>
Date: Thu, 18 Jul 2024 20:04:21 +0200
Subject: [PATCH 060/486] [LLDB][test] Drop OS/HOST_OS detection code from
 Makefile.rules (#99535)

Remove commands for OS/HOST_OS detection from Makefile.rules to simplify
it, since logic for these variables has been implemented in
`lldb/packages/Python/lldbsuite/test/lldbplatformutil.py`
(7021e44b2f0e11717c0d82456bad0fed4a0b48f9).
---
 .../Python/lldbsuite/test/make/Makefile.rules | 25 -------------------
 1 file changed, 25 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
index 3d562285ce9cc..597aa94566c24 100644
--- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
+++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
@@ -46,31 +46,6 @@ LLDB_BASE_DIR := $(THIS_FILE_DIR)/../../../../../
 # according to variable values).
 .DEFAULT_GOAL := all
 
-#----------------------------------------------------------------------
-# If OS is not defined, use 'uname -s' to determine the OS name.
-#
-# GNUWin32 uname gives "windows32" or "server version windows32" while
-# some versions of MSYS uname return "MSYS_NT*", but most environments
-# standardize on "Windows_NT", so we'll make it consistent here. 
-# When running tests from Visual Studio, the environment variable isn't
-# inherited all the way down to the process spawned for make.
-#----------------------------------------------------------------------
-ifeq "$(HOST_OS)" ""
-  HOST_OS := $(shell uname -s)
-endif
-
-ifneq (,$(findstring windows32,$(HOST_OS)))
-	HOST_OS := Windows_NT
-endif
-
-ifneq (,$(findstring MSYS_NT,$(HOST_OS)))
-	HOST_OS := Windows_NT
-endif
-
-ifeq "$(OS)" ""
-	OS := $(HOST_OS)
-endif
-
 #----------------------------------------------------------------------
 # If OS is Windows, force SHELL to be cmd
 #

From c5432d31cb339262451215f6cf9c356a514a1770 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Thu, 18 Jul 2024 11:06:33 -0700
Subject: [PATCH 061/486] [SandboxIR][Tracker] Track eraseFromParent() (#99431)

This patch adds tracking support for Instruction::eraseFromParent(). The
Instruction is not actually being erased, but instead it is detached
from the instruction list and drops its Use edges. The original
instruction position and Use edges are saved in the `EraseFromParent`
change object, and are being used during `revert()` to restore the
original state.
---
 llvm/include/llvm/SandboxIR/SandboxIR.h  |  4 +-
 llvm/include/llvm/SandboxIR/Tracker.h    | 40 +++++++++++++++-
 llvm/lib/SandboxIR/SandboxIR.cpp         | 25 ++++++++--
 llvm/lib/SandboxIR/Tracker.cpp           | 59 ++++++++++++++++++++++++
 llvm/unittests/SandboxIR/TrackerTest.cpp | 52 +++++++++++++++++++++
 5 files changed, 173 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index c5d59ba47ca31..a9f0177eb9338 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -493,6 +493,7 @@ class Instruction : public sandboxir::User {
   /// \Returns the LLVM IR Instructions that this SandboxIR maps to in program
   /// order.
   virtual SmallVector<llvm::Instruction *, 1> getLLVMInstrs() const = 0;
+  friend class EraseFromParent; // For getLLVMInstrs().
 
 public:
   static const char *getOpcodeName(Opcode Opc);
@@ -658,6 +659,7 @@ class Context {
   friend void Instruction::eraseFromParent(); // For detach().
   /// Take ownership of VPtr and store it in `LLVMValueToValueMap`.
   Value *registerValue(std::unique_ptr<Value> &&VPtr);
+  friend class EraseFromParent; // For registerValue().
   /// This is the actual function that creates sandboxir values for \p V,
   /// and among others handles all instruction types.
   Value *getOrCreateValueInternal(llvm::Value *V, llvm::User *U = nullptr);
@@ -682,7 +684,7 @@ class Context {
   friend class BasicBlock; // For getOrCreateValue().
 
 public:
-  Context(LLVMContext &LLVMCtx) : LLVMCtx(LLVMCtx) {}
+  Context(LLVMContext &LLVMCtx) : LLVMCtx(LLVMCtx), IRTracker(*this) {}
 
   Tracker &getTracker() { return IRTracker; }
   /// Convenience function for `getTracker().save()`
diff --git a/llvm/include/llvm/SandboxIR/Tracker.h b/llvm/include/llvm/SandboxIR/Tracker.h
index 2d0904f5665b1..a91b9f178b8aa 100644
--- a/llvm/include/llvm/SandboxIR/Tracker.h
+++ b/llvm/include/llvm/SandboxIR/Tracker.h
@@ -40,6 +40,7 @@
 #ifndef LLVM_SANDBOXIR_TRACKER_H
 #define LLVM_SANDBOXIR_TRACKER_H
 
+#include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
@@ -99,6 +100,41 @@ class UseSet : public IRChangeBase {
 #endif
 };
 
+class EraseFromParent : public IRChangeBase {
+  /// Contains all the data we need to restore an "erased" (i.e., detached)
+  /// instruction: the instruction itself and its operands in order.
+  struct InstrAndOperands {
+    /// The operands that got dropped.
+    SmallVector<llvm::Value *> Operands;
+    /// The instruction that got "erased".
+    llvm::Instruction *LLVMI;
+  };
+  /// The instruction data is in reverse program order, which helps create the
+  /// original program order during revert().
+  SmallVector<InstrAndOperands> InstrData;
+  /// This is either the next Instruction in the stream, or the parent
+  /// BasicBlock if at the end of the BB.
+  PointerUnion<llvm::Instruction *, llvm::BasicBlock *> NextLLVMIOrBB;
+  /// We take ownership of the "erased" instruction.
+  std::unique_ptr<sandboxir::Value> ErasedIPtr;
+
+public:
+  EraseFromParent(std::unique_ptr<sandboxir::Value> &&IPtr, Tracker &Tracker);
+  void revert() final;
+  void accept() final;
+#ifndef NDEBUG
+  void dump(raw_ostream &OS) const final {
+    dumpCommon(OS);
+    OS << "EraseFromParent";
+  }
+  LLVM_DUMP_METHOD void dump() const final;
+  friend raw_ostream &operator<<(raw_ostream &OS, const EraseFromParent &C) {
+    C.dump(OS);
+    return OS;
+  }
+#endif
+};
+
 /// The tracker collects all the change objects and implements the main API for
 /// saving / reverting / accepting.
 class Tracker {
@@ -116,6 +152,7 @@ class Tracker {
 #endif
   /// The current state of the tracker.
   TrackerState State = TrackerState::Disabled;
+  Context &Ctx;
 
 public:
 #ifndef NDEBUG
@@ -124,8 +161,9 @@ class Tracker {
   bool InMiddleOfCreatingChange = false;
 #endif // NDEBUG
 
-  Tracker() = default;
+  explicit Tracker(Context &Ctx) : Ctx(Ctx) {}
   ~Tracker();
+  Context &getContext() const { return Ctx; }
   /// Record \p Change and take ownership. This is the main function used to
   /// track Sandbox IR changes.
   void track(std::unique_ptr<IRChangeBase> &&Change);
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 944869a37989c..c6daf1a586546 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -341,11 +341,26 @@ void Instruction::removeFromParent() {
 void Instruction::eraseFromParent() {
   assert(users().empty() && "Still connected to users, can't erase!");
   std::unique_ptr<Value> Detached = Ctx.detach(this);
-  // We don't have Tracking yet, so just erase the LLVM IR instructions.
-  // Erase in reverse to avoid erasing nstructions with attached uses.
-  auto Instrs = getLLVMInstrs();
-  for (llvm::Instruction *I : reverse(Instrs))
-    I->eraseFromParent();
+  auto LLVMInstrs = getLLVMInstrs();
+
+  auto &Tracker = Ctx.getTracker();
+  if (Tracker.isTracking()) {
+    Tracker.track(
+        std::make_unique<EraseFromParent>(std::move(Detached), Tracker));
+    // We don't actually delete the IR instruction, because then it would be
+    // impossible to bring it back from the dead at the same memory location.
+    // Instead we remove it from its BB and track its current location.
+    for (llvm::Instruction *I : LLVMInstrs)
+      I->removeFromParent();
+    // TODO: Multi-instructions need special treatment because some of the
+    // references are internal to the instruction.
+    for (llvm::Instruction *I : LLVMInstrs)
+      I->dropAllReferences();
+  } else {
+    // Erase in reverse to avoid erasing nstructions with attached uses.
+    for (llvm::Instruction *I : reverse(LLVMInstrs))
+      I->eraseFromParent();
+  }
 }
 
 void Instruction::moveBefore(BasicBlock &BB, const BBIterator &WhereIt) {
diff --git a/llvm/lib/SandboxIR/Tracker.cpp b/llvm/lib/SandboxIR/Tracker.cpp
index 1182f5c55d10b..2336a0067abbd 100644
--- a/llvm/lib/SandboxIR/Tracker.cpp
+++ b/llvm/lib/SandboxIR/Tracker.cpp
@@ -41,6 +41,65 @@ Tracker::~Tracker() {
   assert(Changes.empty() && "You must accept or revert changes!");
 }
 
+EraseFromParent::EraseFromParent(std::unique_ptr<sandboxir::Value> &&ErasedIPtr,
+                                 Tracker &Tracker)
+    : IRChangeBase(Tracker), ErasedIPtr(std::move(ErasedIPtr)) {
+  auto *I = cast<Instruction>(this->ErasedIPtr.get());
+  auto LLVMInstrs = I->getLLVMInstrs();
+  // Iterate in reverse program order.
+  for (auto *LLVMI : reverse(LLVMInstrs)) {
+    SmallVector<llvm::Value *> Operands;
+    Operands.reserve(LLVMI->getNumOperands());
+    for (auto [OpNum, Use] : enumerate(LLVMI->operands()))
+      Operands.push_back(Use.get());
+    InstrData.push_back({Operands, LLVMI});
+  }
+  assert(is_sorted(InstrData,
+                   [](const auto &D0, const auto &D1) {
+                     return D0.LLVMI->comesBefore(D1.LLVMI);
+                   }) &&
+         "Expected reverse program order!");
+  auto *BotLLVMI = cast<llvm::Instruction>(I->Val);
+  if (BotLLVMI->getNextNode() != nullptr)
+    NextLLVMIOrBB = BotLLVMI->getNextNode();
+  else
+    NextLLVMIOrBB = BotLLVMI->getParent();
+}
+
+void EraseFromParent::accept() {
+  for (const auto &IData : InstrData)
+    IData.LLVMI->deleteValue();
+}
+
+void EraseFromParent::revert() {
+  // Place the bottom-most instruction first.
+  auto [Operands, BotLLVMI] = InstrData[0];
+  if (auto *NextLLVMI = NextLLVMIOrBB.dyn_cast<llvm::Instruction *>()) {
+    BotLLVMI->insertBefore(NextLLVMI);
+  } else {
+    auto *LLVMBB = NextLLVMIOrBB.get<llvm::BasicBlock *>();
+    BotLLVMI->insertInto(LLVMBB, LLVMBB->end());
+  }
+  for (auto [OpNum, Op] : enumerate(Operands))
+    BotLLVMI->setOperand(OpNum, Op);
+
+  // Go over the rest of the instructions and stack them on top.
+  for (auto [Operands, LLVMI] : drop_begin(InstrData)) {
+    LLVMI->insertBefore(BotLLVMI);
+    for (auto [OpNum, Op] : enumerate(Operands))
+      LLVMI->setOperand(OpNum, Op);
+    BotLLVMI = LLVMI;
+  }
+  Parent.getContext().registerValue(std::move(ErasedIPtr));
+}
+
+#ifndef NDEBUG
+void EraseFromParent::dump() const {
+  dump(dbgs());
+  dbgs() << "\n";
+}
+#endif
+
 void Tracker::track(std::unique_ptr<IRChangeBase> &&Change) {
   assert(State == TrackerState::Record && "The tracker should be tracking!");
   Changes.push_back(std::move(Change));
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index f090dc521c32b..4e5dccf3a231f 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -146,3 +146,55 @@ define void @foo(ptr %ptr) {
   Ctx.accept();
   EXPECT_EQ(St0->getOperand(0), Ld1);
 }
+
+// TODO: Test multi-instruction patterns.
+TEST_F(TrackerTest, EraseFromParent) {
+  parseIR(C, R"IR(
+define void @foo(i32 %v1) {
+  %add0 = add i32 %v1, %v1
+  %add1 = add i32 %add0, %v1
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto *F = Ctx.createFunction(&LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  sandboxir::Instruction *Add0 = &*It++;
+  sandboxir::Instruction *Add1 = &*It++;
+  sandboxir::Instruction *Ret = &*It++;
+
+  Ctx.save();
+  // Check erase.
+  Add1->eraseFromParent();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+  EXPECT_EQ(Add0->getNumUses(), 0u);
+
+  // Check revert().
+  Ctx.revert();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+  EXPECT_EQ(Add1->getOperand(0), Add0);
+
+  // Same for the last instruction in the block.
+  Ctx.save();
+  Ret->eraseFromParent();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(It, BB->end());
+  Ctx.revert();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+}

From 507c18b445ef88d985d95181db8107f669aed998 Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Thu, 18 Jul 2024 18:19:11 +0000
Subject: [PATCH 062/486] [clang-tidy] Few tiny fixes after #99084

Update documentation, and correct configuration
---
 .../bugprone/UnusedReturnValueCheck.cpp       | 12 +++---
 .../checks/bugprone/unused-return-value.rst   | 40 ++++++++++---------
 2 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp
index 955a9b94dfaf6..1da5b222c2e8f 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp
@@ -138,13 +138,13 @@ UnusedReturnValueCheck::UnusedReturnValueCheck(llvm::StringRef Name,
                                           "^::sigismember$;"
                                           "^::strcasecmp$;"
                                           "^::strsignal$;"
-                                          "^::ttyname"))),
+                                          "^::ttyname$"))),
       CheckedReturnTypes(utils::options::parseStringList(
-          Options.get("CheckedReturnTypes", "::std::error_code$;"
-                                            "::std::error_condition$;"
-                                            "::std::errc$;"
-                                            "::std::expected$;"
-                                            "::boost::system::error_code"))),
+          Options.get("CheckedReturnTypes", "^::std::error_code$;"
+                                            "^::std::error_condition$;"
+                                            "^::std::errc$;"
+                                            "^::std::expected$;"
+                                            "^::boost::system::error_code$"))),
       AllowCastToVoid(Options.get("AllowCastToVoid", false)) {}
 
 UnusedReturnValueCheck::UnusedReturnValueCheck(
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst
index 9205ba98729c4..10ae0fe3243a0 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst
@@ -16,23 +16,26 @@ Options
    This parameter supports regexp. The function is checked if the name
    and scope matches, with any arguments.
    By default the following functions are checked:
-   ``::std::async$, ::std::launder$, ::std::remove$, ::std::remove_if$, ::std::unique$,
-   ::std::unique_ptr::release$, ::std::basic_string::empty$, ::std::vector::empty$,
-   ::std::back_inserter$, ::std::distance$, ::std::find$, ::std::find_if$, ::std::inserter$,
-   ::std::lower_bound$, ::std::make_pair$, ::std::map::count$, ::std::map::find$,
-   ::std::map::lower_bound$, ::std::multimap::equal_range$, ::std::multimap::upper_bound$,
-   ::std::set::count$, ::std::set::find$, ::std::setfill$, ::std::setprecision$,
-   ::std::setw$, ::std::upper_bound$, ::std::vector::at$, ::bsearch$, ::ferror$,
-   ::feof$, ::isalnum$, ::isalpha$, ::isblank$, ::iscntrl$, ::isdigit$, ::isgraph$,
-   ::islower$, ::isprint$, ::ispunct$, ::isspace$, ::isupper$, ::iswalnum$, ::iswprint$,
-   ::iswspace$, ::isxdigit$, ::memchr$, ::memcmp$, ::strcmp$, ::strcoll$, ::strncmp$,
-   ::strpbrk$, ::strrchr$, ::strspn$, ::strstr$, ::wcscmp$, ::access$, ::bind$,
-   ::connect$, ::difftime$, ::dlsym$, ::fnmatch$, ::getaddrinfo$, ::getopt$,
-   ::htonl$, ::htons$, ::iconv_open$, ::inet_addr$, isascii$, isatty$, ::mmap$,
-   ::newlocale$, ::openat$, ::pathconf$, ::pthread_equal$, ::pthread_getspecific$,
-   ::pthread_mutex_trylock$, ::readdir$, ::readlink$, ::recvmsg$, ::regexec$, ::scandir$,
-   ::semget$, ::setjmp$, ::shm_open$, ::shmget$, ::sigismember$, ::strcasecmp$, ::strsignal$,
-   ::ttyname$``
+   ``^::std::async$, ^::std::launder$, ^::std::remove$, ^::std::remove_if$,
+   ^::std::unique$, ^::std::unique_ptr::release$, ^::std::basic_string::empty$,
+   ^::std::vector::empty$, ^::std::back_inserter$, ^::std::distance$,
+   ^::std::find$, ^::std::find_if$, ^::std::inserter$, ^::std::lower_bound$,
+   ^::std::make_pair$, ^::std::map::count$, ^::std::map::find$,
+   ^::std::map::lower_bound$, ^::std::multimap::equal_range$,
+   ^::std::multimap::upper_bound$, ^::std::set::count$, ^::std::set::find$,
+   ^::std::setfill$, ^::std::setprecision$, ^::std::setw$, ^::std::upper_bound$,
+   ^::std::vector::at$, ^::bsearch$, ^::ferror$, ^::feof$, ^::isalnum$,
+   ^::isalpha$, ^::isblank$, ^::iscntrl$, ^::isdigit$, ^::isgraph$, ^::islower$,
+   ^::isprint$, ^::ispunct$, ^::isspace$, ^::isupper$, ^::iswalnum$,
+   ^::iswprint$, ^::iswspace$, ^::isxdigit$, ^::memchr$, ^::memcmp$, ^::strcmp$,
+   ^::strcoll$, ^::strncmp$, ^::strpbrk$, ^::strrchr$, ^::strspn$, ^::strstr$,
+   ^::wcscmp$, ^::access$, ^::bind$, ^::connect$, ^::difftime$, ^::dlsym$,
+   ^::fnmatch$, ^::getaddrinfo$, ^::getopt$, ^::htonl$, ^::htons$,
+   ^::iconv_open$, ^::inet_addr$, isascii$, isatty$, ^::mmap$, ^::newlocale$,
+   ^::openat$, ^::pathconf$, ^::pthread_equal$, ^::pthread_getspecific$,
+   ^::pthread_mutex_trylock$, ^::readdir$, ^::readlink$, ^::recvmsg$,
+   ^::regexec$, ^::scandir$, ^::semget$, ^::setjmp$, ^::shm_open$, ^::shmget$,
+   ^::sigismember$, ^::strcasecmp$, ^::strsignal$, ^::ttyname$``
 
    - ``std::async()``. Not using the return value makes the call synchronous.
    - ``std::launder()``. Not using the return value usually means that the
@@ -54,7 +57,8 @@ Options
 
    Semicolon-separated list of function return types to check.
    By default the following function return types are checked:
-   `::std::error_code`, `::std::error_condition`, `::std::errc`, `::std::expected`, `::boost::system::error_code`
+   `^::std::error_code$`, `^::std::error_condition$`, `^::std::errc$`,
+   `^::std::expected$`, `^::boost::system::error_code$`
 
 .. option:: AllowCastToVoid
 

From 5431a31f87387763cca8d014e7c07394bab7a1ad Mon Sep 17 00:00:00 2001
From: Rafael Stahl <dummdoof-doof@web.de>
Date: Thu, 18 Jul 2024 20:21:19 +0200
Subject: [PATCH 063/486] [clang-tidy][NFC] Fix gsl::not_null template
 parameter (#99472)

`T` is expected to be a pointer type.

https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-nullptr
---
 .../cppcoreguidelines/avoid-const-or-ref-data-members.rst     | 2 +-
 .../cppcoreguidelines/avoid-const-or-ref-data-members.cpp     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/avoid-const-or-ref-data-members.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/avoid-const-or-ref-data-members.rst
index 5783280478dc1..57c4829431e76 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/avoid-const-or-ref-data-members.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/avoid-const-or-ref-data-members.rst
@@ -35,7 +35,7 @@ Examples:
     int* x;
     std::unique_ptr<int> x;
     std::shared_ptr<int> x;
-    gsl::not_null<int> x;
+    gsl::not_null<int*> x;
   };
 
   // Bad, rvalue reference member
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/avoid-const-or-ref-data-members.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/avoid-const-or-ref-data-members.cpp
index 5a5d05bb4e94e..e3864be134da3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/avoid-const-or-ref-data-members.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/avoid-const-or-ref-data-members.cpp
@@ -18,7 +18,7 @@ struct Ok {
   const int *pc;
   std::unique_ptr<int> up;
   std::shared_ptr<int> sp;
-  gsl::not_null<int> n;
+  gsl::not_null<int*> n;
 };
 
 struct ConstMember {
@@ -60,7 +60,7 @@ struct Ok2 {
   const Foo *pc;
   std::unique_ptr<Foo> up;
   std::shared_ptr<Foo> sp;
-  gsl::not_null<Foo> n;
+  gsl::not_null<Foo*> n;
 };
 
 struct ConstMember2 {

From 280d90d0fdb2734af6c071064c6f87a8fe8d06d0 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Thu, 18 Jul 2024 11:23:35 -0700
Subject: [PATCH 064/486] AMDGPU: Add back half and bfloat support for
 global_load_tr16 pats (#99540)

half and bfloat are common types for 16-bit elements. The support of
them was original there and dropped due to some reasons. This work adds
the support of the float types back.
---
 clang/include/clang/Basic/BuiltinsAMDGPU.def  |  4 ++
 clang/lib/CodeGen/CGBuiltin.cpp               | 10 +++-
 ...uiltins-amdgcn-global-load-tr-gfx11-err.cl | 15 ++++--
 ...ins-amdgcn-global-load-tr-gfx12-w32-err.cl |  6 ++-
 ...ins-amdgcn-global-load-tr-gfx12-w64-err.cl |  6 ++-
 .../builtins-amdgcn-global-load-tr-w32.cl     | 22 +++++++++
 .../builtins-amdgcn-global-load-tr-w64.cl     | 22 +++++++++
 llvm/lib/Target/AMDGPU/FLATInstructions.td    |  4 ++
 .../UniformityAnalysis/AMDGPU/intrinsics.ll   | 36 ++++++++++++++
 .../AMDGPU/llvm.amdgcn.global.load.tr-w32.ll  | 48 +++++++++++++++++--
 .../AMDGPU/llvm.amdgcn.global.load.tr-w64.ll  | 48 +++++++++++++++++--
 11 files changed, 207 insertions(+), 14 deletions(-)

diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index 56bba448e12a4..e62315eea277a 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -445,8 +445,12 @@ TARGET_BUILTIN(__builtin_amdgcn_s_get_barrier_state, "Uii", "n", "gfx12-insts")
 
 TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b64_v2i32, "V2iV2i*1", "nc", "gfx12-insts,wavefrontsize32")
 TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8i16, "V8sV8s*1", "nc", "gfx12-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8f16, "V8hV8h*1", "nc", "gfx12-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8bf16, "V8yV8y*1", "nc", "gfx12-insts,wavefrontsize32")
 TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b64_i32, "ii*1", "nc", "gfx12-insts,wavefrontsize64")
 TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4i16, "V4sV4s*1", "nc", "gfx12-insts,wavefrontsize64")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4f16, "V4hV4h*1", "nc", "gfx12-insts,wavefrontsize64")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4bf16, "V4yV4y*1", "nc", "gfx12-insts,wavefrontsize64")
 
 //===----------------------------------------------------------------------===//
 // WMMA builtins.
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 67027f8aa93f3..2ad62d6ee0bb2 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -18725,7 +18725,11 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
-  case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: {
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: {
 
     Intrinsic::ID IID;
     switch (BuiltinID) {
@@ -18734,7 +18738,11 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
       IID = Intrinsic::amdgcn_global_load_tr_b64;
       break;
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
       IID = Intrinsic::amdgcn_global_load_tr_b128;
       break;
     }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl
index 1fcb1d721ad72..8242ae6a98c40 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl
@@ -5,13 +5,22 @@
 
 typedef int    v2i   __attribute__((ext_vector_type(2)));
 typedef short  v8s   __attribute__((ext_vector_type(8)));
+typedef half   v8h   __attribute__((ext_vector_type(8)));
+typedef __bf16 v8y   __attribute__((ext_vector_type(8)));
 typedef short  v4s   __attribute__((ext_vector_type(4)));
+typedef  half  v4h   __attribute__((ext_vector_type(4)));
+typedef __bf16 v4y   __attribute__((ext_vector_type(4)));
 
-void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global int* int_inptr, global v4s* v4s_inptr)
+void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, global v8y* v8y_inptr,
+                           global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr, global v4y* v4y_inptr)
 {
   v2i out_1 = __builtin_amdgcn_global_load_tr_b64_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_v2i32' needs target feature gfx12-insts,wavefrontsize32}}
   v8s out_2 = __builtin_amdgcn_global_load_tr_b128_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8i16' needs target feature gfx12-insts,wavefrontsize32}}
+  v8h out_3 = __builtin_amdgcn_global_load_tr_b128_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8f16' needs target feature gfx12-insts,wavefrontsize32}}
+  v8y out_4 = __builtin_amdgcn_global_load_tr_b128_v8bf16(v8y_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8bf16' needs target feature gfx12-insts,wavefrontsize32}}
 
-  int out_3 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}}
-  v4s out_4 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}}
+  int out_5 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}}
+  v4s out_6 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}}
+  v4h out_7 = __builtin_amdgcn_global_load_tr_b128_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4f16' needs target feature gfx12-insts,wavefrontsize64}}
+  v4y out_8 = __builtin_amdgcn_global_load_tr_b128_v4bf16(v4y_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4bf16' needs target feature gfx12-insts,wavefrontsize64}}
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl
index 7a36881c051b1..6f7a93ef897ac 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl
@@ -4,9 +4,13 @@
 // REQUIRES: amdgpu-registered-target
 
 typedef short  v4s   __attribute__((ext_vector_type(4)));
+typedef  half  v4h   __attribute__((ext_vector_type(4)));
+typedef __bf16 v4y   __attribute__((ext_vector_type(4)));
 
-void amdgcn_global_load_tr(global int* int_inptr, global v4s* v4s_inptr)
+void amdgcn_global_load_tr(global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr, global v4y* v4y_inptr)
 {
   int out_1 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}}
   v4s out_2 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}}
+  v4h out_3 = __builtin_amdgcn_global_load_tr_b128_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4f16' needs target feature gfx12-insts,wavefrontsize64}}
+  v4y out_4 = __builtin_amdgcn_global_load_tr_b128_v4bf16(v4y_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4bf16' needs target feature gfx12-insts,wavefrontsize64}}
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl
index 9155ee6e61822..b7323f1b41c2a 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl
@@ -5,9 +5,13 @@
 
 typedef int    v2i   __attribute__((ext_vector_type(2)));
 typedef short  v8s   __attribute__((ext_vector_type(8)));
+typedef half   v8h   __attribute__((ext_vector_type(8)));
+typedef __bf16 v8y   __attribute__((ext_vector_type(8)));
 
-void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr)
+void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, global v8y* v8y_inptr)
 {
   v2i out_1 = __builtin_amdgcn_global_load_tr_b64_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_v2i32' needs target feature gfx12-insts,wavefrontsize32}}
   v8s out_2 = __builtin_amdgcn_global_load_tr_b128_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8i16' needs target feature gfx12-insts,wavefrontsize32}}
+  v8h out_3 = __builtin_amdgcn_global_load_tr_b128_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8f16' needs target feature gfx12-insts,wavefrontsize32}}
+  v8y out_4 = __builtin_amdgcn_global_load_tr_b128_v8bf16(v8y_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8bf16' needs target feature gfx12-insts,wavefrontsize32}}
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl
index ce8b2c2c7c5ba..186fc4eacfaaf 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl
@@ -4,6 +4,8 @@
 
 typedef int    v2i   __attribute__((ext_vector_type(2)));
 typedef short  v8s   __attribute__((ext_vector_type(8)));
+typedef half   v8h   __attribute__((ext_vector_type(8)));
+typedef __bf16 v8y   __attribute__((ext_vector_type(8)));
 
 // CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b64_v2i32(
 // CHECK-GFX1200-NEXT:  entry:
@@ -24,3 +26,23 @@ v8s test_amdgcn_global_load_tr_b128_v8i16(global v8s* inptr)
 {
   return __builtin_amdgcn_global_load_tr_b128_v8i16(inptr);
 }
+
+// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8f16(
+// CHECK-GFX1200-NEXT:  entry:
+// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1200-NEXT:    ret <8 x half> [[TMP0]]
+//
+v8h test_amdgcn_global_load_tr_b128_v8f16(global v8h* inptr)
+{
+  return __builtin_amdgcn_global_load_tr_b128_v8f16(inptr);
+}
+
+// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8bf16(
+// CHECK-GFX1200-NEXT:  entry:
+// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1200-NEXT:    ret <8 x bfloat> [[TMP0]]
+//
+v8y test_amdgcn_global_load_tr_b128_v8bf16(global v8y* inptr)
+{
+  return __builtin_amdgcn_global_load_tr_b128_v8bf16(inptr);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl
index b0eed07627f41..b6627f1c8114d 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl
@@ -3,6 +3,8 @@
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1200 -target-feature +wavefrontsize64 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-GFX1200
 
 typedef short  v4s   __attribute__((ext_vector_type(4)));
+typedef  half  v4h   __attribute__((ext_vector_type(4)));
+typedef __bf16 v4y   __attribute__((ext_vector_type(4)));
 
 // CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b64_i32(
 // CHECK-GFX1200-NEXT:  entry:
@@ -23,3 +25,23 @@ v4s test_amdgcn_global_load_tr_b128_v4i16(global v4s* inptr)
 {
   return __builtin_amdgcn_global_load_tr_b128_v4i16(inptr);
 }
+
+// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4f16(
+// CHECK-GFX1200-NEXT:  entry:
+// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.global.load.tr.b128.v4f16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1200-NEXT:    ret <4 x half> [[TMP0]]
+//
+v4h test_amdgcn_global_load_tr_b128_v4f16(global v4h* inptr)
+{
+  return __builtin_amdgcn_global_load_tr_b128_v4f16(inptr);
+}
+
+// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4bf16(
+// CHECK-GFX1200-NEXT:  entry:
+// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <4 x bfloat> @llvm.amdgcn.global.load.tr.b128.v4bf16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1200-NEXT:    ret <4 x bfloat> [[TMP0]]
+//
+v4y test_amdgcn_global_load_tr_b128_v4bf16(global v4y* inptr)
+{
+  return __builtin_amdgcn_global_load_tr_b128_v4bf16(inptr);
+}
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 16dc019ede810..56cc6ffa19096 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -1590,10 +1590,14 @@ let OtherPredicates = [isGFX12Plus] in {
   let WaveSizePredicate = isWave32 in {
     defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w32, int_amdgcn_global_load_tr_b64, v2i32>;
     defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, v8i16>;
+    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, v8f16>;
+    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, v8bf16>;
   }
   let WaveSizePredicate = isWave64 in {
     defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w64, int_amdgcn_global_load_tr_b64, i32>;
     defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, v4i16>;
+    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, v4f16>;
+    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, v4bf16>;
   }
 }
 
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
index 680c998a4b39f..b215fc2c2ae74 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
@@ -213,6 +213,22 @@ bb:
   ret void
 }
 
+; CHECK: DIVERGENT: %tmp0 = call <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16(ptr addrspace(1) %addr)
+define amdgpu_kernel void @global_load_tr_b128_v8f16(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16(ptr addrspace(1) %addr)
+  store <8 x half> %tmp0, ptr addrspace(1) %out, align 16
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16(ptr addrspace(1) %addr)
+define amdgpu_kernel void @global_load_tr_b128_v8bf16(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16(ptr addrspace(1) %addr)
+  store <8 x bfloat> %tmp0, ptr addrspace(1) %out, align 16
+  ret void
+}
+
 ; CHECK: DIVERGENT: %tmp0 = call i32 @llvm.amdgcn.global.load.tr.b64.i32(ptr addrspace(1) %addr)
 define amdgpu_kernel void @global_load_tr_b64_i32(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
 bb:
@@ -229,6 +245,22 @@ bb:
   ret void
 }
 
+; CHECK: DIVERGENT: %tmp0 = call <4 x half> @llvm.amdgcn.global.load.tr.b128.v4f16(ptr addrspace(1) %addr)
+define amdgpu_kernel void @global_load_tr_b128_v4f16(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <4 x half> @llvm.amdgcn.global.load.tr.b128.v4f16(ptr addrspace(1) %addr)
+  store <4 x half> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <4 x bfloat> @llvm.amdgcn.global.load.tr.b128.v4bf16(ptr addrspace(1) %addr)
+define amdgpu_kernel void @global_load_tr_b128_v4bf16(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <4 x bfloat> @llvm.amdgcn.global.load.tr.b128.v4bf16(ptr addrspace(1) %addr)
+  store <4 x bfloat> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
 declare i32 @llvm.amdgcn.ds.swizzle(i32, i32) #1
 declare i32 @llvm.amdgcn.permlane16.i32(i32, i32, i32, i32, i1, i1) #1
 declare i32 @llvm.amdgcn.permlanex16.i32(i32, i32, i32, i32, i1, i1) #1
@@ -258,8 +290,12 @@ declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.bf8(<2 x i32>, <4 x i32
 
 declare <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32(ptr addrspace(1))
 declare <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16(ptr addrspace(1))
+declare <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16(ptr addrspace(1))
+declare <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16(ptr addrspace(1))
 declare i32 @llvm.amdgcn.global.load.tr.b64.i32(ptr addrspace(1))
 declare <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16(ptr addrspace(1))
+declare <4 x half> @llvm.amdgcn.global.load.tr.b128.v4f16(ptr addrspace(1))
+declare <4 x bfloat> @llvm.amdgcn.global.load.tr.b128.v4bf16(ptr addrspace(1))
 
 attributes #0 = { nounwind convergent }
 attributes #1 = { nounwind readnone convergent }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll
index 291c249e4b738..0e659b758cd0f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll
@@ -4,9 +4,11 @@
 
 declare <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32.p1(ptr addrspace(1))
 declare <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16.p1(ptr addrspace(1))
+declare <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16.p1(ptr addrspace(1))
+declare <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16.p1(ptr addrspace(1))
 
-define amdgpu_kernel void @global_load_tr_b64(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-LABEL: global_load_tr_b64:
+define amdgpu_kernel void @global_load_tr_b64_v2i32(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX12-LABEL: global_load_tr_b64_v2i32:
 ; GFX12:       ; %bb.0: ; %entry
 ; GFX12-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
@@ -24,8 +26,8 @@ entry:
   ret void
 }
 
-define amdgpu_kernel void @global_load_tr_b128(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-LABEL: global_load_tr_b128:
+define amdgpu_kernel void @global_load_tr_b128_v8i16(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX12-LABEL: global_load_tr_b128_v8i16:
 ; GFX12:       ; %bb.0: ; %entry
 ; GFX12-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
 ; GFX12-NEXT:    v_mov_b32_e32 v4, 0
@@ -42,3 +44,41 @@ entry:
   store <8 x i16> %val, ptr addrspace(1) %use
   ret void
 }
+
+define amdgpu_kernel void @global_load_tr_b128_v8f16(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX12-LABEL: global_load_tr_b128_v8f16:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_load_tr_b128 v[0:3], v4, s[0:1] offset:32
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16.p1(ptr addrspace(1) %gep)
+  store <8 x half> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_kernel void @global_load_tr_b128_v8bf16(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX12-LABEL: global_load_tr_b128_v8bf16:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_load_tr_b128 v[0:3], v4, s[0:1] offset:32
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16.p1(ptr addrspace(1) %gep)
+  store <8 x bfloat> %val, ptr addrspace(1) %use
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll
index 12742f4f7127b..d941830e8dafc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll
@@ -4,9 +4,11 @@
 
 declare i32 @llvm.amdgcn.global.load.tr.b64.i32.p1(ptr addrspace(1))
 declare <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16.p1(ptr addrspace(1))
+declare <4 x half> @llvm.amdgcn.global.load.tr.b128.v4f16.p1(ptr addrspace(1))
+declare <4 x bfloat> @llvm.amdgcn.global.load.tr.b128.v4bf16.p1(ptr addrspace(1))
 
-define amdgpu_kernel void @global_load_tr_b64(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-LABEL: global_load_tr_b64:
+define amdgpu_kernel void @global_load_tr_b64_i32(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX12-LABEL: global_load_tr_b64_i32:
 ; GFX12:       ; %bb.0: ; %entry
 ; GFX12-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
@@ -24,8 +26,8 @@ entry:
   ret void
 }
 
-define amdgpu_kernel void @global_load_tr_b128(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-LABEL: global_load_tr_b128:
+define amdgpu_kernel void @global_load_tr_b128_v4i16(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX12-LABEL: global_load_tr_b128_v4i16:
 ; GFX12:       ; %bb.0: ; %entry
 ; GFX12-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
@@ -42,3 +44,41 @@ entry:
   store <4 x i16> %val, ptr addrspace(1) %use
   ret void
 }
+
+define amdgpu_kernel void @global_load_tr_b128_v4f16(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX12-LABEL: global_load_tr_b128_v4f16:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_load_tr_b128 v[0:1], v2, s[0:1] offset:32
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <4 x half> @llvm.amdgcn.global.load.tr.b128.v4f16.p1(ptr addrspace(1) %gep)
+  store <4 x half> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_kernel void @global_load_tr_b128_v4bf16(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX12-LABEL: global_load_tr_b128_v4bf16:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_load_tr_b128 v[0:1], v2, s[0:1] offset:32
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <4 x bfloat> @llvm.amdgcn.global.load.tr.b128.v4bf16.p1(ptr addrspace(1) %gep)
+  store <4 x bfloat> %val, ptr addrspace(1) %use
+  ret void
+}

From 9527d77aefcf214944a4c8bd284dde3ffe9dff60 Mon Sep 17 00:00:00 2001
From: Angel Zhang <angel.zhang@amd.com>
Date: Thu, 18 Jul 2024 14:31:15 -0400
Subject: [PATCH 065/486] [mlir][spirv] Restructure code in
 `SPIRVConversion.cpp`. NFC. (#99393)

---
 .../SPIRV/Transforms/SPIRVConversion.cpp      | 500 +++++++++---------
 1 file changed, 251 insertions(+), 249 deletions(-)

diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index e3a09ef1ff684..bf5044437fd09 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -40,6 +40,8 @@
 
 using namespace mlir;
 
+namespace {
+
 //===----------------------------------------------------------------------===//
 // Utility functions
 //===----------------------------------------------------------------------===//
@@ -171,18 +173,6 @@ static spirv::ScalarType getIndexType(MLIRContext *ctx,
       IntegerType::get(ctx, options.use64bitIndex ? 64 : 32));
 }
 
-Type SPIRVTypeConverter::getIndexType() const {
-  return ::getIndexType(getContext(), options);
-}
-
-MLIRContext *SPIRVTypeConverter::getContext() const {
-  return targetEnv.getAttr().getContext();
-}
-
-bool SPIRVTypeConverter::allows(spirv::Capability capability) const {
-  return targetEnv.allows(capability);
-}
-
 // TODO: This is a utility function that should probably be exposed by the
 // SPIR-V dialect. Keeping it local till the use case arises.
 static std::optional<int64_t>
@@ -673,9 +663,9 @@ static Type convertMemrefType(const spirv::TargetEnv &targetEnv,
 /// This function is meant to handle the **compute** side; so it does not
 /// involve storage classes in its logic. The storage side is expected to be
 /// handled by MemRef conversion logic.
-std::optional<Value> castToSourceType(const spirv::TargetEnv &targetEnv,
-                                      OpBuilder &builder, Type type,
-                                      ValueRange inputs, Location loc) {
+static std::optional<Value> castToSourceType(const spirv::TargetEnv &targetEnv,
+                                             OpBuilder &builder, Type type,
+                                             ValueRange inputs, Location loc) {
   // We can only cast one value in SPIR-V.
   if (inputs.size() != 1) {
     auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
@@ -731,140 +721,185 @@ std::optional<Value> castToSourceType(const spirv::TargetEnv &targetEnv,
 }
 
 //===----------------------------------------------------------------------===//
-// SPIRVTypeConverter
+// Builtin Variables
 //===----------------------------------------------------------------------===//
 
-SPIRVTypeConverter::SPIRVTypeConverter(spirv::TargetEnvAttr targetAttr,
-                                       const SPIRVConversionOptions &options)
-    : targetEnv(targetAttr), options(options) {
-  // Add conversions. The order matters here: later ones will be tried earlier.
+static spirv::GlobalVariableOp getBuiltinVariable(Block &body,
+                                                  spirv::BuiltIn builtin) {
+  // Look through all global variables in the given `body` block and check if
+  // there is a spirv.GlobalVariable that has the same `builtin` attribute.
+  for (auto varOp : body.getOps<spirv::GlobalVariableOp>()) {
+    if (auto builtinAttr = varOp->getAttrOfType<StringAttr>(
+            spirv::SPIRVDialect::getAttributeName(
+                spirv::Decoration::BuiltIn))) {
+      auto varBuiltIn = spirv::symbolizeBuiltIn(builtinAttr.getValue());
+      if (varBuiltIn && *varBuiltIn == builtin) {
+        return varOp;
+      }
+    }
+  }
+  return nullptr;
+}
 
-  // Allow all SPIR-V dialect specific types. This assumes all builtin types
-  // adopted in the SPIR-V dialect (i.e., IntegerType, FloatType, VectorType)
-  // were tried before.
-  //
-  // TODO: This assumes that the SPIR-V types are valid to use in the given
-  // target environment, which should be the case if the whole pipeline is
-  // driven by the same target environment. Still, we probably still want to
-  // validate and convert to be safe.
-  addConversion([](spirv::SPIRVType type) { return type; });
+/// Gets name of global variable for a builtin.
+std::string getBuiltinVarName(spirv::BuiltIn builtin, StringRef prefix,
+                              StringRef suffix) {
+  return Twine(prefix).concat(stringifyBuiltIn(builtin)).concat(suffix).str();
+}
 
-  addConversion([this](IndexType /*indexType*/) { return getIndexType(); });
+/// Gets or inserts a global variable for a builtin within `body` block.
+static spirv::GlobalVariableOp
+getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
+                           Type integerType, OpBuilder &builder,
+                           StringRef prefix, StringRef suffix) {
+  if (auto varOp = getBuiltinVariable(body, builtin))
+    return varOp;
 
-  addConversion([this](IntegerType intType) -> std::optional<Type> {
-    if (auto scalarType = dyn_cast<spirv::ScalarType>(intType))
-      return convertScalarType(this->targetEnv, this->options, scalarType);
-    if (intType.getWidth() < 8)
-      return convertSubByteIntegerType(this->options, intType);
-    return Type();
-  });
+  OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPointToStart(&body);
 
-  addConversion([this](FloatType floatType) -> std::optional<Type> {
-    if (auto scalarType = dyn_cast<spirv::ScalarType>(floatType))
-      return convertScalarType(this->targetEnv, this->options, scalarType);
-    return Type();
-  });
+  spirv::GlobalVariableOp newVarOp;
+  switch (builtin) {
+  case spirv::BuiltIn::NumWorkgroups:
+  case spirv::BuiltIn::WorkgroupSize:
+  case spirv::BuiltIn::WorkgroupId:
+  case spirv::BuiltIn::LocalInvocationId:
+  case spirv::BuiltIn::GlobalInvocationId: {
+    auto ptrType = spirv::PointerType::get(VectorType::get({3}, integerType),
+                                           spirv::StorageClass::Input);
+    std::string name = getBuiltinVarName(builtin, prefix, suffix);
+    newVarOp =
+        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
+    break;
+  }
+  case spirv::BuiltIn::SubgroupId:
+  case spirv::BuiltIn::NumSubgroups:
+  case spirv::BuiltIn::SubgroupSize: {
+    auto ptrType =
+        spirv::PointerType::get(integerType, spirv::StorageClass::Input);
+    std::string name = getBuiltinVarName(builtin, prefix, suffix);
+    newVarOp =
+        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
+    break;
+  }
+  default:
+    emitError(loc, "unimplemented builtin variable generation for ")
+        << stringifyBuiltIn(builtin);
+  }
+  return newVarOp;
+}
 
-  addConversion([this](ComplexType complexType) {
-    return convertComplexType(this->targetEnv, this->options, complexType);
-  });
+//===----------------------------------------------------------------------===//
+// Push constant storage
+//===----------------------------------------------------------------------===//
 
-  addConversion([this](VectorType vectorType) {
-    return convertVectorType(this->targetEnv, this->options, vectorType);
-  });
+/// Returns the pointer type for the push constant storage containing
+/// `elementCount` 32-bit integer values.
+static spirv::PointerType getPushConstantStorageType(unsigned elementCount,
+                                                     Builder &builder,
+                                                     Type indexType) {
+  auto arrayType = spirv::ArrayType::get(indexType, elementCount,
+                                         /*stride=*/4);
+  auto structType = spirv::StructType::get({arrayType}, /*offsetInfo=*/0);
+  return spirv::PointerType::get(structType, spirv::StorageClass::PushConstant);
+}
 
-  addConversion([this](TensorType tensorType) {
-    return convertTensorType(this->targetEnv, this->options, tensorType);
-  });
+/// Returns the push constant varible containing `elementCount` 32-bit integer
+/// values in `body`. Returns null op if such an op does not exit.
+static spirv::GlobalVariableOp getPushConstantVariable(Block &body,
+                                                       unsigned elementCount) {
+  for (auto varOp : body.getOps<spirv::GlobalVariableOp>()) {
+    auto ptrType = dyn_cast<spirv::PointerType>(varOp.getType());
+    if (!ptrType)
+      continue;
 
-  addConversion([this](MemRefType memRefType) {
-    return convertMemrefType(this->targetEnv, this->options, memRefType);
-  });
+    // Note that Vulkan requires "There must be no more than one push constant
+    // block statically used per shader entry point." So we should always reuse
+    // the existing one.
+    if (ptrType.getStorageClass() == spirv::StorageClass::PushConstant) {
+      auto numElements = cast<spirv::ArrayType>(
+                             cast<spirv::StructType>(ptrType.getPointeeType())
+                                 .getElementType(0))
+                             .getNumElements();
+      if (numElements == elementCount)
+        return varOp;
+    }
+  }
+  return nullptr;
+}
 
-  // Register some last line of defense casting logic.
-  addSourceMaterialization(
-      [this](OpBuilder &builder, Type type, ValueRange inputs, Location loc) {
-        return castToSourceType(this->targetEnv, builder, type, inputs, loc);
-      });
-  addTargetMaterialization([](OpBuilder &builder, Type type, ValueRange inputs,
-                              Location loc) {
-    auto cast = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
-    return std::optional<Value>(cast.getResult(0));
-  });
+/// Gets or inserts a global variable for push constant storage containing
+/// `elementCount` 32-bit integer values in `block`.
+static spirv::GlobalVariableOp
+getOrInsertPushConstantVariable(Location loc, Block &block,
+                                unsigned elementCount, OpBuilder &b,
+                                Type indexType) {
+  if (auto varOp = getPushConstantVariable(block, elementCount))
+    return varOp;
+
+  auto builder = OpBuilder::atBlockBegin(&block, b.getListener());
+  auto type = getPushConstantStorageType(elementCount, builder, indexType);
+  const char *name = "__push_constant_var__";
+  return builder.create<spirv::GlobalVariableOp>(loc, type, name,
+                                                 /*initializer=*/nullptr);
 }
 
 //===----------------------------------------------------------------------===//
 // func::FuncOp Conversion Patterns
 //===----------------------------------------------------------------------===//
 
-namespace {
 /// A pattern for rewriting function signature to convert arguments of functions
 /// to be of valid SPIR-V types.
-class FuncOpConversion final : public OpConversionPattern<func::FuncOp> {
-public:
+struct FuncOpConversion final : OpConversionPattern<func::FuncOp> {
   using OpConversionPattern<func::FuncOp>::OpConversionPattern;
 
   LogicalResult
   matchAndRewrite(func::FuncOp funcOp, OpAdaptor adaptor,
-                  ConversionPatternRewriter &rewriter) const override;
-};
-} // namespace
-
-LogicalResult
-FuncOpConversion::matchAndRewrite(func::FuncOp funcOp, OpAdaptor adaptor,
-                                  ConversionPatternRewriter &rewriter) const {
-  auto fnType = funcOp.getFunctionType();
-  if (fnType.getNumResults() > 1)
-    return failure();
-
-  TypeConverter::SignatureConversion signatureConverter(fnType.getNumInputs());
-  for (const auto &argType : enumerate(fnType.getInputs())) {
-    auto convertedType = getTypeConverter()->convertType(argType.value());
-    if (!convertedType)
-      return failure();
-    signatureConverter.addInputs(argType.index(), convertedType);
-  }
-
-  Type resultType;
-  if (fnType.getNumResults() == 1) {
-    resultType = getTypeConverter()->convertType(fnType.getResult(0));
-    if (!resultType)
+                  ConversionPatternRewriter &rewriter) const override {
+    FunctionType fnType = funcOp.getFunctionType();
+    if (fnType.getNumResults() > 1)
       return failure();
-  }
-
-  // Create the converted spirv.func op.
-  auto newFuncOp = rewriter.create<spirv::FuncOp>(
-      funcOp.getLoc(), funcOp.getName(),
-      rewriter.getFunctionType(signatureConverter.getConvertedTypes(),
-                               resultType ? TypeRange(resultType)
-                                          : TypeRange()));
 
-  // Copy over all attributes other than the function name and type.
-  for (const auto &namedAttr : funcOp->getAttrs()) {
-    if (namedAttr.getName() != funcOp.getFunctionTypeAttrName() &&
-        namedAttr.getName() != SymbolTable::getSymbolAttrName())
-      newFuncOp->setAttr(namedAttr.getName(), namedAttr.getValue());
-  }
+    TypeConverter::SignatureConversion signatureConverter(
+        fnType.getNumInputs());
+    for (const auto &argType : enumerate(fnType.getInputs())) {
+      auto convertedType = getTypeConverter()->convertType(argType.value());
+      if (!convertedType)
+        return failure();
+      signatureConverter.addInputs(argType.index(), convertedType);
+    }
 
-  rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
-                              newFuncOp.end());
-  if (failed(rewriter.convertRegionTypes(
-          &newFuncOp.getBody(), *getTypeConverter(), &signatureConverter)))
-    return failure();
-  rewriter.eraseOp(funcOp);
-  return success();
-}
+    Type resultType;
+    if (fnType.getNumResults() == 1) {
+      resultType = getTypeConverter()->convertType(fnType.getResult(0));
+      if (!resultType)
+        return failure();
+    }
 
-void mlir::populateBuiltinFuncToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
-                                              RewritePatternSet &patterns) {
-  patterns.add<FuncOpConversion>(typeConverter, patterns.getContext());
-}
+    // Create the converted spirv.func op.
+    auto newFuncOp = rewriter.create<spirv::FuncOp>(
+        funcOp.getLoc(), funcOp.getName(),
+        rewriter.getFunctionType(signatureConverter.getConvertedTypes(),
+                                 resultType ? TypeRange(resultType)
+                                            : TypeRange()));
+
+    // Copy over all attributes other than the function name and type.
+    for (const auto &namedAttr : funcOp->getAttrs()) {
+      if (namedAttr.getName() != funcOp.getFunctionTypeAttrName() &&
+          namedAttr.getName() != SymbolTable::getSymbolAttrName())
+        newFuncOp->setAttr(namedAttr.getName(), namedAttr.getValue());
+    }
 
-//===----------------------------------------------------------------------===//
-// func::FuncOp Conversion Patterns
-//===----------------------------------------------------------------------===//
+    rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
+                                newFuncOp.end());
+    if (failed(rewriter.convertRegionTypes(
+            &newFuncOp.getBody(), *getTypeConverter(), &signatureConverter)))
+      return failure();
+    rewriter.eraseOp(funcOp);
+    return success();
+  }
+};
 
-namespace {
 /// A pattern for rewriting function signature to convert vector arguments of
 /// functions to be of valid types
 struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
@@ -1015,17 +1050,11 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
     return success();
   }
 };
-} // namespace
-
-void mlir::populateFuncOpVectorRewritePatterns(RewritePatternSet &patterns) {
-  patterns.add<FuncOpVectorUnroll>(patterns.getContext());
-}
 
 //===----------------------------------------------------------------------===//
 // func::ReturnOp Conversion Patterns
 //===----------------------------------------------------------------------===//
 
-namespace {
 /// A pattern for rewriting function signature and the return op to convert
 /// vectors to be of valid types.
 struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
@@ -1097,81 +1126,13 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
     return success();
   }
 };
-} // namespace
 
-void mlir::populateReturnOpVectorRewritePatterns(RewritePatternSet &patterns) {
-  patterns.add<ReturnOpVectorUnroll>(patterns.getContext());
-}
+} // namespace
 
 //===----------------------------------------------------------------------===//
-// Builtin Variables
+// Public function for builtin variables
 //===----------------------------------------------------------------------===//
 
-static spirv::GlobalVariableOp getBuiltinVariable(Block &body,
-                                                  spirv::BuiltIn builtin) {
-  // Look through all global variables in the given `body` block and check if
-  // there is a spirv.GlobalVariable that has the same `builtin` attribute.
-  for (auto varOp : body.getOps<spirv::GlobalVariableOp>()) {
-    if (auto builtinAttr = varOp->getAttrOfType<StringAttr>(
-            spirv::SPIRVDialect::getAttributeName(
-                spirv::Decoration::BuiltIn))) {
-      auto varBuiltIn = spirv::symbolizeBuiltIn(builtinAttr.getValue());
-      if (varBuiltIn && *varBuiltIn == builtin) {
-        return varOp;
-      }
-    }
-  }
-  return nullptr;
-}
-
-/// Gets name of global variable for a builtin.
-static std::string getBuiltinVarName(spirv::BuiltIn builtin, StringRef prefix,
-                                     StringRef suffix) {
-  return Twine(prefix).concat(stringifyBuiltIn(builtin)).concat(suffix).str();
-}
-
-/// Gets or inserts a global variable for a builtin within `body` block.
-static spirv::GlobalVariableOp
-getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
-                           Type integerType, OpBuilder &builder,
-                           StringRef prefix, StringRef suffix) {
-  if (auto varOp = getBuiltinVariable(body, builtin))
-    return varOp;
-
-  OpBuilder::InsertionGuard guard(builder);
-  builder.setInsertionPointToStart(&body);
-
-  spirv::GlobalVariableOp newVarOp;
-  switch (builtin) {
-  case spirv::BuiltIn::NumWorkgroups:
-  case spirv::BuiltIn::WorkgroupSize:
-  case spirv::BuiltIn::WorkgroupId:
-  case spirv::BuiltIn::LocalInvocationId:
-  case spirv::BuiltIn::GlobalInvocationId: {
-    auto ptrType = spirv::PointerType::get(VectorType::get({3}, integerType),
-                                           spirv::StorageClass::Input);
-    std::string name = getBuiltinVarName(builtin, prefix, suffix);
-    newVarOp =
-        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
-    break;
-  }
-  case spirv::BuiltIn::SubgroupId:
-  case spirv::BuiltIn::NumSubgroups:
-  case spirv::BuiltIn::SubgroupSize: {
-    auto ptrType =
-        spirv::PointerType::get(integerType, spirv::StorageClass::Input);
-    std::string name = getBuiltinVarName(builtin, prefix, suffix);
-    newVarOp =
-        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
-    break;
-  }
-  default:
-    emitError(loc, "unimplemented builtin variable generation for ")
-        << stringifyBuiltIn(builtin);
-  }
-  return newVarOp;
-}
-
 Value mlir::spirv::getBuiltinVariableValue(Operation *op,
                                            spirv::BuiltIn builtin,
                                            Type integerType, OpBuilder &builder,
@@ -1190,60 +1151,9 @@ Value mlir::spirv::getBuiltinVariableValue(Operation *op,
 }
 
 //===----------------------------------------------------------------------===//
-// Push constant storage
+// Public function for pushing constant storage
 //===----------------------------------------------------------------------===//
 
-/// Returns the pointer type for the push constant storage containing
-/// `elementCount` 32-bit integer values.
-static spirv::PointerType getPushConstantStorageType(unsigned elementCount,
-                                                     Builder &builder,
-                                                     Type indexType) {
-  auto arrayType = spirv::ArrayType::get(indexType, elementCount,
-                                         /*stride=*/4);
-  auto structType = spirv::StructType::get({arrayType}, /*offsetInfo=*/0);
-  return spirv::PointerType::get(structType, spirv::StorageClass::PushConstant);
-}
-
-/// Returns the push constant varible containing `elementCount` 32-bit integer
-/// values in `body`. Returns null op if such an op does not exit.
-static spirv::GlobalVariableOp getPushConstantVariable(Block &body,
-                                                       unsigned elementCount) {
-  for (auto varOp : body.getOps<spirv::GlobalVariableOp>()) {
-    auto ptrType = dyn_cast<spirv::PointerType>(varOp.getType());
-    if (!ptrType)
-      continue;
-
-    // Note that Vulkan requires "There must be no more than one push constant
-    // block statically used per shader entry point." So we should always reuse
-    // the existing one.
-    if (ptrType.getStorageClass() == spirv::StorageClass::PushConstant) {
-      auto numElements = cast<spirv::ArrayType>(
-                             cast<spirv::StructType>(ptrType.getPointeeType())
-                                 .getElementType(0))
-                             .getNumElements();
-      if (numElements == elementCount)
-        return varOp;
-    }
-  }
-  return nullptr;
-}
-
-/// Gets or inserts a global variable for push constant storage containing
-/// `elementCount` 32-bit integer values in `block`.
-static spirv::GlobalVariableOp
-getOrInsertPushConstantVariable(Location loc, Block &block,
-                                unsigned elementCount, OpBuilder &b,
-                                Type indexType) {
-  if (auto varOp = getPushConstantVariable(block, elementCount))
-    return varOp;
-
-  auto builder = OpBuilder::atBlockBegin(&block, b.getListener());
-  auto type = getPushConstantStorageType(elementCount, builder, indexType);
-  const char *name = "__push_constant_var__";
-  return builder.create<spirv::GlobalVariableOp>(loc, type, name,
-                                                 /*initializer=*/nullptr);
-}
-
 Value spirv::getPushConstantValue(Operation *op, unsigned elementCount,
                                   unsigned offset, Type integerType,
                                   OpBuilder &builder) {
@@ -1267,7 +1177,7 @@ Value spirv::getPushConstantValue(Operation *op, unsigned elementCount,
 }
 
 //===----------------------------------------------------------------------===//
-// Index calculation
+// Public functions for index calculation
 //===----------------------------------------------------------------------===//
 
 Value mlir::spirv::linearizeIndex(ValueRange indices, ArrayRef<int64_t> strides,
@@ -1375,6 +1285,81 @@ Value mlir::spirv::getElementPtr(const SPIRVTypeConverter &typeConverter,
                              builder);
 }
 
+//===----------------------------------------------------------------------===//
+// SPIR-V TypeConverter
+//===----------------------------------------------------------------------===//
+
+SPIRVTypeConverter::SPIRVTypeConverter(spirv::TargetEnvAttr targetAttr,
+                                       const SPIRVConversionOptions &options)
+    : targetEnv(targetAttr), options(options) {
+  // Add conversions. The order matters here: later ones will be tried earlier.
+
+  // Allow all SPIR-V dialect specific types. This assumes all builtin types
+  // adopted in the SPIR-V dialect (i.e., IntegerType, FloatType, VectorType)
+  // were tried before.
+  //
+  // TODO: This assumes that the SPIR-V types are valid to use in the given
+  // target environment, which should be the case if the whole pipeline is
+  // driven by the same target environment. Still, we probably still want to
+  // validate and convert to be safe.
+  addConversion([](spirv::SPIRVType type) { return type; });
+
+  addConversion([this](IndexType /*indexType*/) { return getIndexType(); });
+
+  addConversion([this](IntegerType intType) -> std::optional<Type> {
+    if (auto scalarType = dyn_cast<spirv::ScalarType>(intType))
+      return convertScalarType(this->targetEnv, this->options, scalarType);
+    if (intType.getWidth() < 8)
+      return convertSubByteIntegerType(this->options, intType);
+    return Type();
+  });
+
+  addConversion([this](FloatType floatType) -> std::optional<Type> {
+    if (auto scalarType = dyn_cast<spirv::ScalarType>(floatType))
+      return convertScalarType(this->targetEnv, this->options, scalarType);
+    return Type();
+  });
+
+  addConversion([this](ComplexType complexType) {
+    return convertComplexType(this->targetEnv, this->options, complexType);
+  });
+
+  addConversion([this](VectorType vectorType) {
+    return convertVectorType(this->targetEnv, this->options, vectorType);
+  });
+
+  addConversion([this](TensorType tensorType) {
+    return convertTensorType(this->targetEnv, this->options, tensorType);
+  });
+
+  addConversion([this](MemRefType memRefType) {
+    return convertMemrefType(this->targetEnv, this->options, memRefType);
+  });
+
+  // Register some last line of defense casting logic.
+  addSourceMaterialization(
+      [this](OpBuilder &builder, Type type, ValueRange inputs, Location loc) {
+        return castToSourceType(this->targetEnv, builder, type, inputs, loc);
+      });
+  addTargetMaterialization([](OpBuilder &builder, Type type, ValueRange inputs,
+                              Location loc) {
+    auto cast = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    return std::optional<Value>(cast.getResult(0));
+  });
+}
+
+Type SPIRVTypeConverter::getIndexType() const {
+  return ::getIndexType(getContext(), options);
+}
+
+MLIRContext *SPIRVTypeConverter::getContext() const {
+  return targetEnv.getAttr().getContext();
+}
+
+bool SPIRVTypeConverter::allows(spirv::Capability capability) const {
+  return targetEnv.allows(capability);
+}
+
 //===----------------------------------------------------------------------===//
 // SPIR-V ConversionTarget
 //===----------------------------------------------------------------------===//
@@ -1468,3 +1453,20 @@ bool SPIRVConversionTarget::isLegalOp(Operation *op) {
 
   return true;
 }
+
+//===----------------------------------------------------------------------===//
+// Public functions for populating patterns
+//===----------------------------------------------------------------------===//
+
+void mlir::populateBuiltinFuncToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                              RewritePatternSet &patterns) {
+  patterns.add<FuncOpConversion>(typeConverter, patterns.getContext());
+}
+
+void mlir::populateFuncOpVectorRewritePatterns(RewritePatternSet &patterns) {
+  patterns.add<FuncOpVectorUnroll>(patterns.getContext());
+}
+
+void mlir::populateReturnOpVectorRewritePatterns(RewritePatternSet &patterns) {
+  patterns.add<ReturnOpVectorUnroll>(patterns.getContext());
+}

From f6f88f4b99638821af803d1911ab6a7dac04880b Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 18 Jul 2024 13:36:55 -0500
Subject: [PATCH 066/486] [LLVM] Silence compiler-rt warning in runtimes build
 (#99525)

Summary:
The `compiler-rt` project wants `LLVM_CMAKE_DIR` but the
`llvm_ExternalProject_add` interface sets the `LLVM_CONFIG_PATH`. This
patch just makes the utility pass that as well.
---
 llvm/cmake/modules/LLVMExternalProjectUtils.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
index eef0c16f6847e..cd071d50bdce9 100644
--- a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
+++ b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
@@ -350,6 +350,7 @@ function(llvm_ExternalProject_Add name source_dir)
                ${sysroot_arg}
                -DLLVM_BINARY_DIR=${PROJECT_BINARY_DIR}
                -DLLVM_CONFIG_PATH=${llvm_config_path}
+               -DLLVM_CMAKE_DIR=${LLVM_CMAKE_DIR}
                -DLLVM_ENABLE_WERROR=${LLVM_ENABLE_WERROR}
                -DLLVM_HOST_TRIPLE=${LLVM_HOST_TRIPLE}
                -DLLVM_HAVE_LINK_VERSION_SCRIPT=${LLVM_HAVE_LINK_VERSION_SCRIPT}

From 5e8cd29d62a72ed18e7bc782554d7f14eccec0ee Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 18 Jul 2024 11:32:56 -0700
Subject: [PATCH 067/486] [RISCV] Add coverage for vector combine reduce(cast
 x) transformation

This covers both the existing trunc transform - basically checking
that it performs sanely with the RISCV cost model - and a planned
change to handle sext/zext as well.
---
 .../VectorCombine/RISCV/vecreduce-of-cast.ll  | 126 ++++++++++++++++++
 1 file changed, 126 insertions(+)
 create mode 100644 llvm/test/Transforms/VectorCombine/RISCV/vecreduce-of-cast.ll

diff --git a/llvm/test/Transforms/VectorCombine/RISCV/vecreduce-of-cast.ll b/llvm/test/Transforms/VectorCombine/RISCV/vecreduce-of-cast.ll
new file mode 100644
index 0000000000000..9b1aa19f85c21
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/RISCV/vecreduce-of-cast.ll
@@ -0,0 +1,126 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=vector-combine -S -mtriple=riscv32 -mattr=+v | FileCheck  %s
+; RUN: opt < %s -passes=vector-combine -S -mtriple=riscv64 -mattr=+v | FileCheck  %s
+
+;
+; Fold reduce(cast(X)) -> trunc(cast(X)) if more cost efficient
+;
+
+define i32 @reduce_add_trunc_v8i64_to_v8i32(<8 x i64> %a0)  {
+; CHECK-LABEL: @reduce_add_trunc_v8i64_to_v8i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[A0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %tr = trunc <8 x i64> %a0 to <8 x i32>
+  %red = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %tr)
+  ret i32 %red
+}
+
+define i16 @reduce_add_trunc_v8i64_to_v8i16(<8 x i64> %a0)  {
+; CHECK-LABEL: @reduce_add_trunc_v8i64_to_v8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[A0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = trunc i64 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[RED]]
+;
+  %tr = trunc <8 x i64> %a0 to <8 x i16>
+  %red = tail call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %tr)
+  ret i16 %red
+}
+
+define i8 @reduce_add_trunc_v8i64_to_v8i8(<8 x i64> %a0)  {
+; CHECK-LABEL: @reduce_add_trunc_v8i64_to_v8i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[A0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = trunc i64 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[RED]]
+;
+  %tr = trunc <8 x i64> %a0 to <8 x i8>
+  %red = tail call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %tr)
+  ret i8 %red
+}
+
+define i8 @reduce_or_trunc_v8i32_i8(<8 x i32> %a0)  {
+; CHECK-LABEL: @reduce_or_trunc_v8i32_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[A0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = trunc i32 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[RED]]
+;
+  %tr = trunc <8 x i32> %a0 to <8 x i8>
+  %red = tail call i8 @llvm.vector.reduce.or.v8i32(<8 x i8> %tr)
+  ret i8 %red
+}
+
+define i8 @reduce_xor_trunc_v16i64_i8(<16 x i64> %a0)  {
+; CHECK-LABEL: @reduce_xor_trunc_v16i64_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> [[A0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = trunc i64 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[RED]]
+;
+  %tr = trunc <16 x i64> %a0 to <16 x i8>
+  %red = tail call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> %tr)
+  ret i8 %red
+}
+
+define i16 @reduce_mul_trunc_v8i64_i16(<8 x i64> %a0)  {
+; CHECK-LABEL: @reduce_mul_trunc_v8i64_i16(
+; CHECK-NEXT:    [[TR:%.*]] = trunc <8 x i64> [[A0:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[RED:%.*]] = tail call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> [[TR]])
+; CHECK-NEXT:    ret i16 [[RED]]
+;
+  %tr = trunc <8 x i64> %a0 to <8 x i16>
+  %red = tail call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %tr)
+  ret i16 %red
+}
+
+define i32 @reduce_or_sext_v8i8_to_v8i32(<8 x i8> %a0)  {
+; CHECK-LABEL: @reduce_or_sext_v8i8_to_v8i32(
+; CHECK-NEXT:    [[TR:%.*]] = sext <8 x i8> [[A0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[RED:%.*]] = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TR]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %tr = sext <8 x i8> %a0 to <8 x i32>
+  %red = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %tr)
+  ret i32 %red
+}
+
+define i32 @reduce_or_sext_v8i16_to_v8i32(<8 x i16> %a0)  {
+; CHECK-LABEL: @reduce_or_sext_v8i16_to_v8i32(
+; CHECK-NEXT:    [[TR:%.*]] = sext <8 x i16> [[A0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[RED:%.*]] = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TR]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %tr = sext <8 x i16> %a0 to <8 x i32>
+  %red = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %tr)
+  ret i32 %red
+}
+
+define i32 @reduce_or_zext_v8i8_to_v8i32(<8 x i8> %a0)  {
+; CHECK-LABEL: @reduce_or_zext_v8i8_to_v8i32(
+; CHECK-NEXT:    [[TR:%.*]] = zext <8 x i8> [[A0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[RED:%.*]] = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TR]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %tr = zext <8 x i8> %a0 to <8 x i32>
+  %red = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %tr)
+  ret i32 %red
+}
+
+define i32 @reduce_or_zext_v8i16_to_v8i32(<8 x i16> %a0)  {
+; CHECK-LABEL: @reduce_or_zext_v8i16_to_v8i32(
+; CHECK-NEXT:    [[TR:%.*]] = zext <8 x i16> [[A0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[RED:%.*]] = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TR]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %tr = zext <8 x i16> %a0 to <8 x i32>
+  %red = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %tr)
+  ret i32 %red
+}
+
+declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
+declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)
+declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)
+declare i8 @llvm.vector.reduce.or.v8i8(<8 x i8>)
+declare i8 @llvm.vector.reduce.xor.v16i8(<16 x i8>)
+declare i16 @llvm.vector.reduce.and.v16i16(<16 x i16>)
+declare i16 @llvm.vector.reduce.mul.v8i16(<8 x i16>)
+

From eed72d4381261bfe1acb693fb8751c05765c4831 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Thu, 18 Jul 2024 20:13:45 +0100
Subject: [PATCH 068/486] [mlir][ArmSME] Support filling liveness 'holes' in
 the tile allocator (#98350)

Holes in a live range are points where the corresponding value does not
need to be in a tile/register. If the tile allocator keeps track of
these holes it can reuse tiles for more values (avoiding spills).

Take this simple example:

```mlir
func.func @example(%cond: i1) {
  %tileA = arm_sme.get_tile : vector<[4]x[4]xf32>
  cf.cond_br %cond, ^bb2, ^bb1
^bb1:
  // If we end up here we never use %tileA again!
  "test.some_use"(%tileB) : (vector<[4]x[4]xf32>) -> ()
  cf.br ^bb3
^bb2:
  "test.some_use"(%tileA) : (vector<[4]x[4]xf32>) -> ()
  cf.br ^bb3
^bb3:
  return
}
```

If you were to calculate the liveness of %tileA and %tileB. You'd see
there is a hole in the liveness of %tileA in bb1:

```
      %tileA  %tileB
^bb0:  Live
^bb1:          Live
^bb2:  Live
```

The tile allocator can make use of that hole and reuse the tile ID it
assigned to %tileA for %tileB.
---
 .../ArmSME/Transforms/TileAllocation.cpp      | 134 ++++++++++---
 .../ArmSME/tile-allocation-liveness.mlir      | 178 ++++++++++++++++++
 2 files changed, 283 insertions(+), 29 deletions(-)

diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index 733e758b43907..5cac770b03ed1 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -153,10 +153,18 @@ class TileAllocator {
     return failure();
   }
 
+  /// Acquires a specific tile ID. Asserts the tile is initially free.
+  void acquireTileId(ArmSMETileType tileType, unsigned tileId) {
+    TileMask tileMask = getMasks(tileType)[tileId];
+    assert((tilesInUse & tileMask) == TileMask::kNone &&
+           "cannot acquire allocated tile!");
+    tilesInUse |= tileMask;
+  }
+
   /// Releases a previously allocated tile ID.
   void releaseTileId(ArmSMETileType tileType, unsigned tileId) {
     TileMask tileMask = getMasks(tileType)[tileId];
-    assert((tilesInUse & tileMask) != TileMask::kNone &&
+    assert((tilesInUse & tileMask) == tileMask &&
            "cannot release unallocated tile!");
     tilesInUse ^= tileMask;
   }
@@ -289,6 +297,11 @@ struct LiveRange {
         .valid();
   }
 
+  /// Returns true if this range is active at `point` in the program.
+  bool overlaps(uint64_t point) const {
+    return ranges->lookup(point) == kValidLiveRange;
+  }
+
   /// Unions this live range with `otherRange`, aborts if the ranges overlap.
   void unionWith(LiveRange const &otherRange) {
     for (auto it = otherRange.ranges->begin(); it != otherRange.ranges->end();
@@ -488,76 +501,139 @@ coalesceTileLiveRanges(DenseMap<Value, LiveRange> &initialLiveRanges) {
   return std::move(coalescedLiveRanges);
 }
 
-/// Choose a live range to spill (via some heuristics). This picks either an
-/// active live range from `activeRanges` or the new live range `newRange`.
-LiveRange *chooseSpillUsingHeuristics(ArrayRef<LiveRange *> activeRanges,
-                                      LiveRange *newRange) {
+/// Choose a live range to spill (via some heuristics). This picks either a live
+/// range from `overlappingRanges`, or the new live range `newRange`.
+template <typename OverlappingRangesIterator>
+LiveRange *
+chooseSpillUsingHeuristics(OverlappingRangesIterator overlappingRanges,
+                           LiveRange *newRange) {
   // Heuristic: Spill trivially copyable operations (usually free).
-  auto isTrivialSpill = [&](LiveRange *allocatedRange) {
-    return isTileTypeGreaterOrEqual(allocatedRange->getTileType(),
+  auto isTrivialSpill = [&](LiveRange &allocatedRange) {
+    return isTileTypeGreaterOrEqual(allocatedRange.getTileType(),
                                     newRange->getTileType()) &&
-           allocatedRange->values.size() == 1 &&
+           allocatedRange.values.size() == 1 &&
            isTriviallyCloneableTileOp(
-               allocatedRange->values[0]
-                   .getDefiningOp<ArmSMETileOpInterface>());
+               allocatedRange.values[0].getDefiningOp<ArmSMETileOpInterface>());
   };
-  if (isTrivialSpill(newRange))
+  if (isTrivialSpill(*newRange))
     return newRange;
-  auto trivialSpill = llvm::find_if(activeRanges, isTrivialSpill);
-  if (trivialSpill != activeRanges.end())
-    return *trivialSpill;
+  auto trivialSpill = llvm::find_if(overlappingRanges, isTrivialSpill);
+  if (trivialSpill != overlappingRanges.end())
+    return &*trivialSpill;
 
   // Heuristic: Spill the range that ends last (with a compatible tile type).
-  auto isSmallerTileTypeOrEndsEarlier = [](LiveRange *a, LiveRange *b) {
-    return !isTileTypeGreaterOrEqual(a->getTileType(), b->getTileType()) ||
-           a->end() < b->end();
+  auto isSmallerTileTypeOrEndsEarlier = [](LiveRange &a, LiveRange &b) {
+    return !isTileTypeGreaterOrEqual(a.getTileType(), b.getTileType()) ||
+           a.end() < b.end();
   };
-  LiveRange *lastActiveLiveRange = *std::max_element(
-      activeRanges.begin(), activeRanges.end(), isSmallerTileTypeOrEndsEarlier);
-  if (!isSmallerTileTypeOrEndsEarlier(lastActiveLiveRange, newRange))
-    return lastActiveLiveRange;
+  LiveRange &latestEndingLiveRange =
+      *std::max_element(overlappingRanges.begin(), overlappingRanges.end(),
+                        isSmallerTileTypeOrEndsEarlier);
+  if (!isSmallerTileTypeOrEndsEarlier(latestEndingLiveRange, *newRange))
+    return &latestEndingLiveRange;
   return newRange;
 }
 
 /// Greedily allocate tile IDs to live ranges. Spill using simple heuristics.
-/// Note: This does not attempt to fill holes in active live ranges.
 void allocateTilesToLiveRanges(
     ArrayRef<LiveRange *> liveRangesSortedByStartPoint) {
   TileAllocator tileAllocator;
+  // `activeRanges` = Live ranges that need to be in a tile at the
+  // `currentPoint` in the program.
   SetVector<LiveRange *> activeRanges;
+  // `inactiveRanges` = Live ranges that _do not_ need to be in a tile
+  // at the `currentPoint` in the program but could become active again later.
+  // An inactive section of a live range can be seen as a 'hole' in the live
+  // range, where it is possible to reuse the live range's tile ID _before_ it
+  // has ended. By identifying 'holes', the allocator can reuse tiles more
+  // often, which helps avoid costly tile spills.
+  SetVector<LiveRange *> inactiveRanges;
   for (LiveRange *nextRange : liveRangesSortedByStartPoint) {
-    // Release tile IDs from live ranges that have ended.
+    auto currentPoint = nextRange->start();
+    // 1. Update the `activeRanges` at `currentPoint`.
     activeRanges.remove_if([&](LiveRange *activeRange) {
-      if (activeRange->end() <= nextRange->start()) {
+      // Check for live ranges that have expired.
+      if (activeRange->end() <= currentPoint) {
         tileAllocator.releaseTileId(activeRange->getTileType(),
                                     *activeRange->tileId);
         return true;
       }
+      // Check for live ranges that have become inactive.
+      if (!activeRange->overlaps(currentPoint)) {
+        tileAllocator.releaseTileId(activeRange->getTileType(),
+                                    *activeRange->tileId);
+        inactiveRanges.insert(activeRange);
+        return true;
+      }
       return false;
     });
+    // 2. Update the `inactiveRanges` at `currentPoint`.
+    inactiveRanges.remove_if([&](LiveRange *inactiveRange) {
+      // Check for live ranges that have expired.
+      if (inactiveRange->end() <= currentPoint) {
+        return true;
+      }
+      // Check for live ranges that have become active.
+      if (inactiveRange->overlaps(currentPoint)) {
+        tileAllocator.acquireTileId(inactiveRange->getTileType(),
+                                    *inactiveRange->tileId);
+        activeRanges.insert(inactiveRange);
+        return true;
+      }
+      return false;
+    });
+
+    // 3. Collect inactive live ranges that overlap with the new live range.
+    // Note: The overlap checks in steps 1 and 2 only look at the `currentPoint`
+    // whereas this checks if there is an overlap at any future point too.
+    SmallVector<LiveRange *> overlappingInactiveRanges;
+    for (LiveRange *inactiveRange : inactiveRanges) {
+      if (inactiveRange->overlaps(*nextRange)) {
+        // We need to reserve the tile IDs of overlapping inactive ranges to
+        // prevent two (overlapping) live ranges from getting the same tile ID.
+        tileAllocator.acquireTileId(inactiveRange->getTileType(),
+                                    *inactiveRange->tileId);
+        overlappingInactiveRanges.push_back(inactiveRange);
+      }
+    }
 
-    // Allocate a tile ID to `nextRange`.
+    // 4. Allocate a tile ID to `nextRange`.
     auto rangeTileType = nextRange->getTileType();
     auto tileId = tileAllocator.allocateTileId(rangeTileType);
     if (succeeded(tileId)) {
       nextRange->tileId = *tileId;
     } else {
+      // Create an iterator over all overlapping live ranges.
+      auto allOverlappingRanges = llvm::concat<LiveRange>(
+          llvm::make_pointee_range(activeRanges.getArrayRef()),
+          llvm::make_pointee_range(overlappingInactiveRanges));
+      // Choose an overlapping live range to spill.
       LiveRange *rangeToSpill =
-          chooseSpillUsingHeuristics(activeRanges.getArrayRef(), nextRange);
+          chooseSpillUsingHeuristics(allOverlappingRanges, nextRange);
       if (rangeToSpill != nextRange) {
-        // Spill an active live range (so release its tile ID first).
+        // Spill an (in)active live range (so release its tile ID first).
         tileAllocator.releaseTileId(rangeToSpill->getTileType(),
                                     *rangeToSpill->tileId);
-        activeRanges.remove(rangeToSpill);
         // This will always succeed after a spill (of an active live range).
         nextRange->tileId = *tileAllocator.allocateTileId(rangeTileType);
+        // Remove the live range from the active/inactive sets.
+        if (!activeRanges.remove(rangeToSpill)) {
+          bool removed = inactiveRanges.remove(rangeToSpill);
+          assert(removed && "expected a range to be removed!");
+        }
       }
       rangeToSpill->tileId = tileAllocator.allocateInMemoryTileId();
     }
 
-    // Insert the live range into the active ranges.
+    // 5. Insert the live range into the active ranges.
     if (nextRange->tileId < kInMemoryTileIdBase)
       activeRanges.insert(nextRange);
+
+    // 6. Release tiles reserved for inactive live ranges (in step 3).
+    for (LiveRange *range : overlappingInactiveRanges) {
+      if (*range->tileId < kInMemoryTileIdBase)
+        tileAllocator.releaseTileId(range->getTileType(), *range->tileId);
+    }
   }
 }
 
diff --git a/mlir/test/Dialect/ArmSME/tile-allocation-liveness.mlir b/mlir/test/Dialect/ArmSME/tile-allocation-liveness.mlir
index 9c22b29ac22e7..2e1f3d1ee10a9 100644
--- a/mlir/test/Dialect/ArmSME/tile-allocation-liveness.mlir
+++ b/mlir/test/Dialect/ArmSME/tile-allocation-liveness.mlir
@@ -430,3 +430,181 @@ func.func @cond_branch_with_backedge(%slice: vector<[4]xf32>) {
   // Live here: %finalTileA, %finalTileB, %finalTileC, %finalTileD
   return
 }
+
+// -----
+
+//  CHECK-LIVE-RANGE-LABEL: @fill_holes_in_tile_liveness
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb0:
+//   CHECK-LIVE-RANGE-NEXT: S  arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: E  cf.cond_br
+//   CHECK-LIVE-RANGE-NEXT: ^bb1:
+//   CHECK-LIVE-RANGE-NEXT:  S arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT:  | test.dummy
+//   CHECK-LIVE-RANGE-NEXT:  E test.some_use
+//   CHECK-LIVE-RANGE-NEXT:    cf.br
+//   CHECK-LIVE-RANGE-NEXT: ^bb2:
+//   CHECK-LIVE-RANGE-NEXT: |  test.dummy
+//   CHECK-LIVE-RANGE-NEXT: |  test.dummy
+//   CHECK-LIVE-RANGE-NEXT: |  test.dummy
+//   CHECK-LIVE-RANGE-NEXT: E  test.some_use
+//   CHECK-LIVE-RANGE-NEXT:    cf.br
+
+// Here there's a 'hole' in the liveness of %tileA (in bb1) where another value
+// can reuse the tile ID assigned to %tileA. The liveness for %tileB is
+// entirely within the 'hole' in %tileA's live range, so %tileB should get the
+// same tile ID as %tileA.
+
+// CHECK-LABEL: @fill_holes_in_tile_liveness
+func.func @fill_holes_in_tile_liveness(%cond: i1) {
+  // CHECK: arm_sme.get_tile {tile_id = [[TILE_ID_A:.*]] : i32}
+  %tileA = arm_sme.get_tile : vector<[4]x[4]xf32>
+  cf.cond_br %cond, ^bb2, ^bb1
+^bb1:
+  // CHECK: arm_sme.get_tile {tile_id = [[TILE_ID_A]] : i32}
+  %tileB = arm_sme.get_tile : vector<[4]x[4]xf32>
+  "test.dummy"(): () -> ()
+  "test.some_use"(%tileB) : (vector<[4]x[4]xf32>) -> ()
+  cf.br ^bb3
+^bb2:
+  "test.dummy"(): () -> ()
+  "test.dummy"(): () -> ()
+  "test.dummy"(): () -> ()
+  "test.some_use"(%tileA) : (vector<[4]x[4]xf32>) -> ()
+  cf.br ^bb3
+^bb3:
+  return
+}
+
+// -----
+
+//  CHECK-LIVE-RANGE-LABEL: @holes_in_tile_liveness_inactive_overlaps
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb0:
+//   CHECK-LIVE-RANGE-NEXT: S  arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: E  cf.cond_br
+//   CHECK-LIVE-RANGE-NEXT: ^bb1:
+//   CHECK-LIVE-RANGE-NEXT:  S arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT:  | test.dummy
+//   CHECK-LIVE-RANGE-NEXT:  | test.some_use
+//   CHECK-LIVE-RANGE-NEXT:  | arm_sme.copy_tile
+//   CHECK-LIVE-RANGE-NEXT:  E cf.br
+//   CHECK-LIVE-RANGE-NEXT: ^bb2:
+//   CHECK-LIVE-RANGE-NEXT: |  test.dummy
+//   CHECK-LIVE-RANGE-NEXT: |  test.dummy
+//   CHECK-LIVE-RANGE-NEXT: |  test.dummy
+//   CHECK-LIVE-RANGE-NEXT: |S arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: E| test.some_use
+//   CHECK-LIVE-RANGE-NEXT:  | arm_sme.copy_tile
+//   CHECK-LIVE-RANGE-NEXT:  E cf.br
+//   CHECK-LIVE-RANGE-NEXT: ^bb3:
+//   CHECK-LIVE-RANGE-NEXT:  E test.some_use
+//   CHECK-LIVE-RANGE-NEXT:    func.return
+
+// This tests an edge case in inactive live ranges. The first live range is
+// inactive at the start of ^bb1. If the tile allocator did not check if the
+// second live range overlapped the first it would wrongly re-use tile ID 0
+// (as the first live range is inactive so tile ID 0 is free). This would mean
+// in ^bb2 two overlapping live ranges would have the same tile ID (bad!).
+
+// CHECK-LABEL: @holes_in_tile_liveness_inactive_overlaps
+func.func @holes_in_tile_liveness_inactive_overlaps(%cond: i1) {
+  // CHECK: arm_sme.get_tile {tile_id = 0 : i32}
+  %tileA = arm_sme.get_tile : vector<[4]x[4]xf32>
+  cf.cond_br %cond, ^bb2, ^bb1
+^bb1:
+  // CHECK: arm_sme.get_tile {tile_id = 1 : i32}
+  %tileB = arm_sme.get_tile : vector<[4]x[4]xf32>
+  "test.dummy"(): () -> ()
+  "test.some_use"(%tileB) : (vector<[4]x[4]xf32>) -> ()
+  cf.br ^bb3(%tileB: vector<[4]x[4]xf32>)
+^bb2:
+  "test.dummy"(): () -> ()
+  "test.dummy"(): () -> ()
+  "test.dummy"(): () -> ()
+  // CHECK: arm_sme.get_tile {tile_id = 1 : i32}
+  %tileC = arm_sme.get_tile : vector<[4]x[4]xf32>
+  "test.some_use"(%tileA) : (vector<[4]x[4]xf32>) -> ()
+  cf.br ^bb3(%tileC: vector<[4]x[4]xf32>)
+^bb3(%tile: vector<[4]x[4]xf32>):
+  "test.some_use"(%tile) : (vector<[4]x[4]xf32>) -> ()
+  return
+}
+
+// -----
+
+// This is the same as the previous example, but changes the tile types to
+// vector<[16]x[16]xi8>. This means in bb1 the allocator will need to spill the
+// first live range (which is inactive).
+
+// Note: The live ranges are the same as the previous example (so are not checked).
+
+// CHECK-LABEL: @spill_inactive_live_range
+func.func @spill_inactive_live_range(%cond: i1) {
+  // CHECK: arm_sme.get_tile {tile_id = 16 : i32}
+  %tileA = arm_sme.get_tile : vector<[16]x[16]xi8>
+  cf.cond_br %cond, ^bb2, ^bb1
+^bb1:
+  // CHECK: arm_sme.get_tile {tile_id = 0 : i32}
+  %tileB = arm_sme.get_tile : vector<[16]x[16]xi8>
+  "test.dummy"(): () -> ()
+  "test.some_use"(%tileB) : (vector<[16]x[16]xi8>) -> ()
+  cf.br ^bb3(%tileB: vector<[16]x[16]xi8>)
+^bb2:
+  "test.dummy"(): () -> ()
+  "test.dummy"(): () -> ()
+  "test.dummy"(): () -> ()
+  // CHECK: arm_sme.get_tile {tile_id = 0 : i32}
+  %tileC = arm_sme.get_tile : vector<[16]x[16]xi8>
+  "test.some_use"(%tileA) : (vector<[16]x[16]xi8>) -> ()
+  cf.br ^bb3(%tileC: vector<[16]x[16]xi8>)
+^bb3(%tile: vector<[16]x[16]xi8>):
+  "test.some_use"(%tile) : (vector<[16]x[16]xi8>) -> ()
+  return
+}
+
+// -----
+
+//  CHECK-LIVE-RANGE-LABEL: @reactivate_inactive_live_range
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb0:
+//   CHECK-LIVE-RANGE-NEXT: S   arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: E   cf.cond_br
+//   CHECK-LIVE-RANGE-NEXT: ^bb1:
+//   CHECK-LIVE-RANGE-NEXT:  S  arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT:  |  test.dummy
+//   CHECK-LIVE-RANGE-NEXT:  E  test.some_use
+//   CHECK-LIVE-RANGE-NEXT:     cf.br
+//   CHECK-LIVE-RANGE-NEXT: ^bb2:
+//   CHECK-LIVE-RANGE-NEXT: | S arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: | | test.dummy
+//   CHECK-LIVE-RANGE-NEXT: | | test.dummy
+//   CHECK-LIVE-RANGE-NEXT: | E test.some_use
+//   CHECK-LIVE-RANGE-NEXT: E   test.some_use
+//   CHECK-LIVE-RANGE-NEXT:     cf.br
+
+// Here the live range for %tileA becomes inactive in bb1 (so %tileB gets tile
+// ID 0 too). Then in bb2 the live range for tileA is reactivated as it overlaps
+// with the start of %tileC's live range (which means %tileC gets tile ID 1).
+
+func.func @reactivate_inactive_live_range(%cond: i1) {
+  // CHECK: arm_sme.get_tile {tile_id = 0 : i32}
+  %tileA = arm_sme.get_tile : vector<[4]x[4]xf32>
+  cf.cond_br %cond, ^bb2, ^bb1
+^bb1:
+  // CHECK: arm_sme.get_tile {tile_id = 0 : i32}
+  %tileB = arm_sme.get_tile : vector<[16]x[16]xi8>
+  "test.dummy"(): () -> ()
+  "test.some_use"(%tileB) : (vector<[16]x[16]xi8>) -> ()
+  cf.br ^bb3
+^bb2:
+  // CHECK: arm_sme.get_tile {tile_id = 1 : i32}
+  %tileC = arm_sme.get_tile : vector<[4]x[4]xf32>
+  "test.dummy"(): () -> ()
+  "test.dummy"(): () -> ()
+  "test.some_use"(%tileC) : (vector<[4]x[4]xf32>) -> ()
+  "test.some_use"(%tileA) : (vector<[4]x[4]xf32>) -> ()
+  cf.br ^bb3
+^bb3:
+  return
+}

From e2c3cd7f3d0cd40bd8506ab305573d61a1ae25d9 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Thu, 18 Jul 2024 12:16:55 -0700
Subject: [PATCH 069/486] AMDGPU: Loop over the types for global_load_tr16 pats
 (NFC) (#99551)

---
 llvm/lib/Target/AMDGPU/FLATInstructions.td | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 56cc6ffa19096..88c6039473338 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -1589,15 +1589,13 @@ let OtherPredicates = [isGFX12Plus] in {
 
   let WaveSizePredicate = isWave32 in {
     defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w32, int_amdgcn_global_load_tr_b64, v2i32>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, v8i16>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, v8f16>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, v8bf16>;
+    foreach vt = [v8i16, v8f16, v8bf16] in
+      defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, vt>;
   }
   let WaveSizePredicate = isWave64 in {
     defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w64, int_amdgcn_global_load_tr_b64, i32>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, v4i16>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, v4f16>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, v4bf16>;
+    foreach vt = [v4i16, v4f16, v4bf16] in
+      defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, vt>;
   }
 }
 

From 82cca0c77e935b4972c31745d94edef616970b6c Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma@quicinc.com>
Date: Thu, 18 Jul 2024 12:32:13 -0700
Subject: [PATCH 070/486] [IR] Unify max alignment for arguments with generic
 max align. (#99257)

The 2^14 limit was completely arbitrary; the generic limit is still
arbitrary, but at least it's the same arbitrary limit as everything
else.

While I'm here, also add a verifier check for the ByValOrByRefSize.
---
 llvm/include/llvm/CodeGen/TargetCallingConv.h |  8 ++--
 llvm/lib/IR/Verifier.cpp                      | 38 +++++++++++--------
 llvm/test/Verifier/byval-size-limit.ll        |  4 ++
 llvm/test/Verifier/param-align.ll             | 14 +++----
 llvm/test/Verifier/param-attr-align.ll        |  4 +-
 llvm/test/Verifier/param-ret-align.ll         | 12 +++---
 6 files changed, 46 insertions(+), 34 deletions(-)
 create mode 100644 llvm/test/Verifier/byval-size-limit.ll

diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h
index 0ff4e1fe657f4..cb0055633f4f3 100644
--- a/llvm/include/llvm/CodeGen/TargetCallingConv.h
+++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h
@@ -45,9 +45,9 @@ namespace ISD {
     unsigned IsHva : 1;        ///< HVA field for
     unsigned IsHvaStart : 1;   ///< HVA structure start
     unsigned IsSecArgPass : 1; ///< Second argument
-    unsigned MemAlign : 4;     ///< Log 2 of alignment when arg is passed in memory
-                               ///< (including byval/byref). The max alignment is
-                               ///< verified in IR verification.
+    unsigned MemAlign : 6; ///< Log 2 of alignment when arg is passed in memory
+                           ///< (including byval/byref). The max alignment is
+                           ///< verified in IR verification.
     unsigned OrigAlign : 5;    ///< Log 2 of original alignment
     unsigned IsInConsecutiveRegsLast : 1;
     unsigned IsInConsecutiveRegs : 1;
@@ -67,7 +67,7 @@ namespace ISD {
           IsSecArgPass(0), MemAlign(0), OrigAlign(0),
           IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
           IsCopyElisionCandidate(0), IsPointer(0) {
-      static_assert(sizeof(*this) == 3 * sizeof(unsigned), "flags are too big");
+      static_assert(sizeof(*this) == 4 * sizeof(unsigned), "flags are too big");
     }
 
     bool isZExt() const { return IsZExt; }
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 75a53c1c99734..c5c407637cbf3 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -324,13 +324,6 @@ namespace {
 
 class Verifier : public InstVisitor<Verifier>, VerifierSupport {
   friend class InstVisitor<Verifier>;
-
-  // ISD::ArgFlagsTy::MemAlign only have 4 bits for alignment, so
-  // the alignment size should not exceed 2^15. Since encode(Align)
-  // would plus the shift value by 1, the alignment size should
-  // not exceed 2^14, otherwise it can NOT be properly lowered
-  // in backend.
-  static constexpr unsigned ParamMaxAlignment = 1 << 14;
   DominatorTree DT;
 
   /// When verifying a basic block, keep track of all of the
@@ -2021,31 +2014,43 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
   }
 
   if (isa<PointerType>(Ty)) {
+    if (Attrs.hasAttribute(Attribute::Alignment)) {
+      Align AttrAlign = Attrs.getAlignment().valueOrOne();
+      Check(AttrAlign.value() <= Value::MaximumAlignment,
+            "huge alignment values are unsupported", V);
+    }
     if (Attrs.hasAttribute(Attribute::ByVal)) {
-      if (Attrs.hasAttribute(Attribute::Alignment)) {
-        Align AttrAlign = Attrs.getAlignment().valueOrOne();
-        Align MaxAlign(ParamMaxAlignment);
-        Check(AttrAlign <= MaxAlign,
-              "Attribute 'align' exceed the max size 2^14", V);
-      }
       SmallPtrSet<Type *, 4> Visited;
       Check(Attrs.getByValType()->isSized(&Visited),
             "Attribute 'byval' does not support unsized types!", V);
+      Check(DL.getTypeAllocSize(Attrs.getByValType()).getKnownMinValue() <
+                (1ULL << 32),
+            "huge 'byval' arguments are unsupported", V);
     }
     if (Attrs.hasAttribute(Attribute::ByRef)) {
       SmallPtrSet<Type *, 4> Visited;
       Check(Attrs.getByRefType()->isSized(&Visited),
             "Attribute 'byref' does not support unsized types!", V);
+      Check(DL.getTypeAllocSize(Attrs.getByRefType()).getKnownMinValue() <
+                (1ULL << 32),
+            "huge 'byref' arguments are unsupported", V);
     }
     if (Attrs.hasAttribute(Attribute::InAlloca)) {
       SmallPtrSet<Type *, 4> Visited;
       Check(Attrs.getInAllocaType()->isSized(&Visited),
             "Attribute 'inalloca' does not support unsized types!", V);
+      Check(DL.getTypeAllocSize(Attrs.getInAllocaType()).getKnownMinValue() <
+                (1ULL << 32),
+            "huge 'inalloca' arguments are unsupported", V);
     }
     if (Attrs.hasAttribute(Attribute::Preallocated)) {
       SmallPtrSet<Type *, 4> Visited;
       Check(Attrs.getPreallocatedType()->isSized(&Visited),
             "Attribute 'preallocated' does not support unsized types!", V);
+      Check(
+          DL.getTypeAllocSize(Attrs.getPreallocatedType()).getKnownMinValue() <
+              (1ULL << 32),
+          "huge 'preallocated' arguments are unsupported", V);
     }
   }
 
@@ -3511,12 +3516,15 @@ void Verifier::visitCallBase(CallBase &Call) {
         "not allowed. Please use the @llvm.amdgpu.cs.chain intrinsic instead.",
         Call);
 
+  // Disallow passing/returning values with alignment higher than we can
+  // represent.
+  // FIXME: Consider making DataLayout cap the alignment, so this isn't
+  // necessary.
   auto VerifyTypeAlign = [&](Type *Ty, const Twine &Message) {
     if (!Ty->isSized())
       return;
     Align ABIAlign = DL.getABITypeAlign(Ty);
-    Align MaxAlign(ParamMaxAlignment);
-    Check(ABIAlign <= MaxAlign,
+    Check(ABIAlign.value() <= Value::MaximumAlignment,
           "Incorrect alignment of " + Message + " to called function!", Call);
   };
 
diff --git a/llvm/test/Verifier/byval-size-limit.ll b/llvm/test/Verifier/byval-size-limit.ll
new file mode 100644
index 0000000000000..3eb462b063636
--- /dev/null
+++ b/llvm/test/Verifier/byval-size-limit.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: huge 'byval' arguments are unsupported
+define void @f(ptr byval([2147483648 x i16])) { ret void }
diff --git a/llvm/test/Verifier/param-align.ll b/llvm/test/Verifier/param-align.ll
index bfd01cbc9faa5..caa8f9ac41ea5 100644
--- a/llvm/test/Verifier/param-align.ll
+++ b/llvm/test/Verifier/param-align.ll
@@ -2,19 +2,19 @@
 
 ; Large vector for intrinsics is valid
 ; CHECK-NOT: llvm.fshr
-define dso_local <8192 x i32> @test_intrin(<8192 x i32> %l, <8192 x i32> %r, <8192 x i32> %amt) {
+define dso_local <2147483648 x i32> @test_intrin(<2147483648 x i32> %l, <2147483648 x i32> %r, <2147483648 x i32> %amt) {
 entry:
-  %b = call <8192 x i32> @llvm.fshr.v8192i32(<8192 x i32> %l, <8192 x i32> %r, <8192 x i32> %amt)
-  ret <8192 x i32> %b
+  %b = call <2147483648 x i32> @llvm.fshr.v8192i32(<2147483648 x i32> %l, <2147483648 x i32> %r, <2147483648 x i32> %amt)
+  ret <2147483648 x i32> %b
 }
-declare <8192 x i32> @llvm.fshr.v8192i32 (<8192 x i32> %l, <8192 x i32> %r, <8192 x i32> %amt)
+declare <2147483648 x i32> @llvm.fshr.v8192i32 (<2147483648 x i32> %l, <2147483648 x i32> %r, <2147483648 x i32> %amt)
 
 ; CHECK: Incorrect alignment of argument passed to called function!
 ; CHECK: bar
-define dso_local void @foo(<8192 x float> noundef %vec) {
+define dso_local void @foo(<2147483648 x float> noundef %vec) {
 entry:
-  call void @bar(<8192 x float> %vec)
+  call void @bar(<2147483648 x float> %vec)
   ret void
 }
 
-declare dso_local void @bar(<8192 x float>)
+declare dso_local void @bar(<2147483648 x float>)
diff --git a/llvm/test/Verifier/param-attr-align.ll b/llvm/test/Verifier/param-attr-align.ll
index 038bfa3494f89..700efe5376841 100644
--- a/llvm/test/Verifier/param-attr-align.ll
+++ b/llvm/test/Verifier/param-attr-align.ll
@@ -1,9 +1,9 @@
 ; RUN: not llvm-as < %s 2>&1 | FileCheck %s
 
-; CHECK: Attribute 'align' exceed the max size 2^14
+; CHECK: huge alignments are not supported yet
 define dso_local void @foo(ptr %p) {
 entry:
-  call void @bar(ptr noundef byval(<8 x float>) align 32768 %p)
+  call void @bar(ptr noundef byval(<8 x float>) align 8589934592 %p)
   ret void
 }
 
diff --git a/llvm/test/Verifier/param-ret-align.ll b/llvm/test/Verifier/param-ret-align.ll
index dd302c38b53d2..98cbb4ee88a89 100644
--- a/llvm/test/Verifier/param-ret-align.ll
+++ b/llvm/test/Verifier/param-ret-align.ll
@@ -2,19 +2,19 @@
 
 ; Large vector for intrinsics is valid
 ; CHECK-NOT: llvm.fshr
-define dso_local <8192 x i32> @test_intrin(<8192 x i32> %l, <8192 x i32> %r, <8192 x i32> %amt) {
+define dso_local <2147483648 x i32> @test_intrin(<2147483648 x i32> %l, <2147483648 x i32> %r, <2147483648 x i32> %amt) {
 entry:
-  %b = call <8192 x i32> @llvm.fshr.v8192i32(<8192 x i32> %l, <8192 x i32> %r, <8192 x i32> %amt)
-  ret <8192 x i32> %b
+  %b = call <2147483648 x i32> @llvm.fshr.v8192i32(<2147483648 x i32> %l, <2147483648 x i32> %r, <2147483648 x i32> %amt)
+  ret <2147483648 x i32> %b
 }
-declare <8192 x i32> @llvm.fshr.v8192i32 (<8192 x i32> %l, <8192 x i32> %r, <8192 x i32> %amt)
+declare <2147483648 x i32> @llvm.fshr.v2147483648i32 (<2147483648 x i32> %l, <2147483648 x i32> %r, <2147483648 x i32> %amt)
 
 ; CHECK: Incorrect alignment of return type to called function!
 ; CHECK: bar
 define dso_local void @foo() {
 entry:
-  call <8192 x float> @bar()
+  call <2147483648 x float> @bar()
   ret void
 }
 
-declare dso_local <8192 x float> @bar()
+declare dso_local <2147483648 x float> @bar()

From 892c58cf7490c219ff8fc4dc0d2497e062a9c665 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Thu, 18 Jul 2024 15:33:03 -0400
Subject: [PATCH 071/486] [Clang][AMDGPU] Add builtins for instrinsic
 `llvm.amdgcn.raw.ptr.buffer.load` (#99258)

---
 clang/include/clang/Basic/BuiltinsAMDGPU.def  |   6 +
 clang/lib/CodeGen/CGBuiltin.cpp               |  33 ++++
 .../builtins-amdgcn-raw-buffer-load.cl        | 172 ++++++++++++++++++
 .../builtins-amdgcn-raw-buffer-load-error.cl  |  33 ++++
 4 files changed, 244 insertions(+)
 create mode 100644 clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load.cl
 create mode 100644 clang/test/SemaOpenCL/builtins-amdgcn-raw-buffer-load-error.cl

diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index e62315eea277a..774cbaa74f8af 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -155,6 +155,12 @@ BUILTIN(__builtin_amdgcn_raw_buffer_store_b32, "viQbiiIi", "n")
 BUILTIN(__builtin_amdgcn_raw_buffer_store_b64, "vV2iQbiiIi", "n")
 BUILTIN(__builtin_amdgcn_raw_buffer_store_b96, "vV3iQbiiIi", "n")
 BUILTIN(__builtin_amdgcn_raw_buffer_store_b128, "vV4iQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_load_b8, "UcQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_load_b16, "UsQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_load_b32, "UiQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_load_b64, "V2UiQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_load_b96, "V3UiQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_load_b128, "V4UiQbiiIi", "n")
 
 //===----------------------------------------------------------------------===//
 // Ballot builtins.
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 2ad62d6ee0bb2..f426570b17e15 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -19185,6 +19185,39 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128:
     return emitBuiltinWithOneOverloadedType<5>(
         *this, E, Intrinsic::amdgcn_raw_ptr_buffer_store);
+  case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
+  case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
+  case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
+  case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
+  case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
+  case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128: {
+    llvm::Type *RetTy = nullptr;
+    switch (BuiltinID) {
+    case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
+      RetTy = Int8Ty;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
+      RetTy = Int16Ty;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
+      RetTy = Int32Ty;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
+      RetTy = llvm::FixedVectorType::get(Int32Ty, /*NumElements=*/2);
+      break;
+    case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
+      RetTy = llvm::FixedVectorType::get(Int32Ty, /*NumElements=*/3);
+      break;
+    case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128:
+      RetTy = llvm::FixedVectorType::get(Int32Ty, /*NumElements=*/4);
+      break;
+    }
+    Function *F =
+        CGM.getIntrinsic(Intrinsic::amdgcn_raw_ptr_buffer_load, RetTy);
+    return Builder.CreateCall(
+        F, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)),
+            EmitScalarExpr(E->getArg(2)), EmitScalarExpr(E->getArg(3))});
+  }
   default:
     return nullptr;
   }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load.cl
new file mode 100644
index 0000000000000..3403b69e07e4b
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load.cl
@@ -0,0 +1,172 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu verde -emit-llvm -o - %s | FileCheck %s
+
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned int v2u32 __attribute__((ext_vector_type(2)));
+typedef unsigned int v3u32 __attribute__((ext_vector_type(3)));
+typedef unsigned int v4u32 __attribute__((ext_vector_type(4)));
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+u8 test_amdgcn_raw_ptr_buffer_load_b8(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b8(rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+u16 test_amdgcn_raw_ptr_buffer_load_b16(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b16(rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+u32 test_amdgcn_raw_ptr_buffer_load_b32(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b32(rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2u32 test_amdgcn_raw_ptr_buffer_load_b64(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b64(rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b96(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT:    ret <3 x i32> [[TMP0]]
+//
+v3u32 test_amdgcn_raw_ptr_buffer_load_b96(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b96(rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b128(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT:    ret <4 x i32> [[TMP0]]
+//
+v4u32 test_amdgcn_raw_ptr_buffer_load_b128(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b128(rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b8_non_const_offset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+u8 test_amdgcn_raw_ptr_buffer_load_b8_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b8(rsrc, offset, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b16_non_const_offset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+u16 test_amdgcn_raw_ptr_buffer_load_b16_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b16(rsrc, offset, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b32_non_const_offset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+u32 test_amdgcn_raw_ptr_buffer_load_b32_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b32(rsrc, offset, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b64_non_const_offset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2u32 test_amdgcn_raw_ptr_buffer_load_b64_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b64(rsrc, offset, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b96_non_const_offset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT:    ret <3 x i32> [[TMP0]]
+//
+v3u32 test_amdgcn_raw_ptr_buffer_load_b96_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b96(rsrc, offset, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b128_non_const_offset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT:    ret <4 x i32> [[TMP0]]
+//
+v4u32 test_amdgcn_raw_ptr_buffer_load_b128_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b128(rsrc, offset, /*soffset=*/0, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b8_non_const_soffset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+u8 test_amdgcn_raw_ptr_buffer_load_b8_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b8(rsrc, /*offset=*/0, soffset, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b16_non_const_soffset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+u16 test_amdgcn_raw_ptr_buffer_load_b16_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b16(rsrc, /*offset=*/0, soffset, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b32_non_const_soffset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+u32 test_amdgcn_raw_ptr_buffer_load_b32_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b32(rsrc, /*offset=*/0, soffset, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b64_non_const_soffset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2u32 test_amdgcn_raw_ptr_buffer_load_b64_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b64(rsrc, /*offset=*/0, soffset, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b96_non_const_soffset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT:    ret <3 x i32> [[TMP0]]
+//
+v3u32 test_amdgcn_raw_ptr_buffer_load_b96_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b96(rsrc, /*offset=*/0, soffset, /*aux=*/0);
+}
+
+// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b128_non_const_soffset(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT:    ret <4 x i32> [[TMP0]]
+//
+v4u32 test_amdgcn_raw_ptr_buffer_load_b128_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+  return __builtin_amdgcn_raw_buffer_load_b128(rsrc, /*offset=*/0, soffset, /*aux=*/0);
+}
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-raw-buffer-load-error.cl b/clang/test/SemaOpenCL/builtins-amdgcn-raw-buffer-load-error.cl
new file mode 100644
index 0000000000000..5d123c8e81d87
--- /dev/null
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-raw-buffer-load-error.cl
@@ -0,0 +1,33 @@
+// RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu verde -S -verify -o - %s
+// REQUIRES: amdgpu-registered-target
+
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned int v2u32 __attribute__((ext_vector_type(2)));
+typedef unsigned int v3u32 __attribute__((ext_vector_type(3)));
+typedef unsigned int v4u32 __attribute__((ext_vector_type(4)));
+
+u8 test_amdgcn_raw_ptr_buffer_load_b8(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+  return __builtin_amdgcn_raw_buffer_load_b8(rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_load_b8' must be a constant integer}}
+}
+
+u16 test_amdgcn_raw_ptr_buffer_load_b16(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+  return __builtin_amdgcn_raw_buffer_load_b16(rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_load_b16' must be a constant integer}}
+}
+
+u32 test_amdgcn_raw_ptr_buffer_load_b32(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+  return __builtin_amdgcn_raw_buffer_load_b32(rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_load_b32' must be a constant integer}}
+}
+
+v2u32 test_amdgcn_raw_ptr_buffer_load_b64(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+  return __builtin_amdgcn_raw_buffer_load_b64(rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_load_b64' must be a constant integer}}
+}
+
+v3u32 test_amdgcn_raw_ptr_buffer_load_b96(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+  return __builtin_amdgcn_raw_buffer_load_b96(rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_load_b96' must be a constant integer}}
+}
+
+v4u32 test_amdgcn_raw_ptr_buffer_load_b128(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+  return __builtin_amdgcn_raw_buffer_load_b128(rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_load_b128' must be a constant integer}}
+}

From 52d947b5c14173b0aee96e419a04a49f83e5a283 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 18 Jul 2024 12:53:36 -0700
Subject: [PATCH 072/486] [LV] Remove unnecessary variable from
 InnerLoopVectorizer::createBitOrPointerCast. NFC

DstVTy is already a VectorType, we don't need to cast it again. This
used to be a cast to FixedVectorType that was changed to support
scalable vectors.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 748db418fee8c..fbca4cdcbcfcd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2767,18 +2767,17 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) {
 Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy,
                                                    const DataLayout &DL) {
   // Verify that V is a vector type with same number of elements as DstVTy.
-  auto *DstFVTy = cast<VectorType>(DstVTy);
-  auto VF = DstFVTy->getElementCount();
+  auto VF = DstVTy->getElementCount();
   auto *SrcVecTy = cast<VectorType>(V->getType());
   assert(VF == SrcVecTy->getElementCount() && "Vector dimensions do not match");
   Type *SrcElemTy = SrcVecTy->getElementType();
-  Type *DstElemTy = DstFVTy->getElementType();
+  Type *DstElemTy = DstVTy->getElementType();
   assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&
          "Vector elements must have same size");
 
   // Do a direct cast if element types are castable.
   if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) {
-    return Builder.CreateBitOrPointerCast(V, DstFVTy);
+    return Builder.CreateBitOrPointerCast(V, DstVTy);
   }
   // V cannot be directly casted to desired vector type.
   // May happen when V is a floating point vector but DstVTy is a vector of
@@ -2792,7 +2791,7 @@ Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy,
       IntegerType::getIntNTy(V->getContext(), DL.getTypeSizeInBits(SrcElemTy));
   auto *VecIntTy = VectorType::get(IntTy, VF);
   Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);
-  return Builder.CreateBitOrPointerCast(CastVal, DstFVTy);
+  return Builder.CreateBitOrPointerCast(CastVal, DstVTy);
 }
 
 void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {

From eb7d54a84bd6b8f85ceb94d96d09b50b494a3f9c Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Thu, 18 Jul 2024 12:58:10 -0700
Subject: [PATCH 073/486] [msan] Precommit MSan Arm NEON vst tests with
 origin-tracking (#99555)

This adds an abridged copy of neon_vst.ll (from
https://github.com/llvm/llvm-project/commit/ff0821583eab1651ff126bbf4f881e6163b67435),
but with origin tracking enabled.

The test will be updated when MSan's Arm NEON support is improved (e.g.,
https://github.com/llvm/llvm-project/pull/99360).
---
 .../AArch64/neon_vst_origins.ll               | 158 ++++++++++++++++++
 1 file changed, 158 insertions(+)
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst_origins.ll

diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst_origins.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst_origins.ll
new file mode 100644
index 0000000000000..ff4c4c24bf37a
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst_origins.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; Test memory sanitizer instrumentation for Arm NEON VST instructions, with
+; origin tracking. These tests are deliberately shorter than neon_vst.ll, due
+; to the verbosity of the output.
+;
+; RUN: opt < %s -passes=msan -msan-track-origins=2 -S | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/AArch64/arm64-st1.ll
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android9001"
+
+; -----------------------------------------------------------------------------------------------------------------------------------------------
+
+define void @st2_16b(<16 x i8> %A, <16 x i8> %B, ptr %P) nounwind sanitize_memory {
+;
+; CHECK-LABEL: define void @st2_16b
+; CHECK-SAME: (<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 16) to ptr), align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 32) to ptr), align 4
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP2]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF0]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP4]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF0]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP6]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> [[A]], <16 x i8> [[B]], ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> %A, <16 x i8> %B, ptr %P)
+  ret void
+}
+
+define void @st3_16b(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, ptr %P) nounwind sanitize_memory {
+;
+; CHECK-LABEL: define void @st3_16b
+; CHECK-SAME: (<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <16 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 16) to ptr), align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 32) to ptr), align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 48) to ptr), align 4
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP2]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF0]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP4]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x i8> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF0]]
+; CHECK:       16:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP6]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       17:
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK:       18:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP8]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       19:
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8> [[A]], <16 x i8> [[B]], <16 x i8> [[C]], ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, ptr %P)
+  ret void
+}
+
+define void @st4_16b(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, ptr %P) nounwind sanitize_memory {
+;
+; CHECK-LABEL: define void @st4_16b
+; CHECK-SAME: (<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <16 x i8> [[C:%.*]], <16 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 16) to ptr), align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 32) to ptr), align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 48) to ptr), align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 64) to ptr), align 4
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF0]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP2]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF0]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP4]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <16 x i8> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP17]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK:       18:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP6]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       19:
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <16 x i8> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP20]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP21:%.*]], label [[TMP22:%.*]], !prof [[PROF0]]
+; CHECK:       21:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP8]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       22:
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF0]]
+; CHECK:       23:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP10]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       24:
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> [[A]], <16 x i8> [[B]], <16 x i8> [[C]], <16 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, ptr %P)
+  ret void
+}
+
+declare void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8>, <16 x i8>, ptr) nounwind sanitize_memory readonly
+declare void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8>, <16 x i8>, <16 x i8>, ptr) nounwind sanitize_memory readonly
+declare void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, ptr) nounwind sanitize_memory readonly

From a0662176a9b40462aafbb17cd8eb8cf6a65e940e Mon Sep 17 00:00:00 2001
From: Iuri Chaer <iuri.chaer@gmail.com>
Date: Thu, 18 Jul 2024 21:11:24 +0100
Subject: [PATCH 074/486]  [libc++] Speed up set_intersection() by
 fast-forwarding over ranges of non-matching elements with one-sided binary
 search. (#75230)

One-sided binary search, aka meta binary search, has been in the public
domain for decades, and has the general advantage of being constant time
in the best case, with the downside of executing at most 2*log(N)
comparisons vs classic binary search's exact log(N). There are two
scenarios in which it really shines: the first one is when operating
over non-random-access iterators, because the classic algorithm requires
knowing the container's size upfront, which adds N iterator increments
to the complexity. The second one is when traversing the container in
order, trying to fast-forward to the next value: in that case the
classic algorithm requires at least O(N*log(N)) comparisons and, for
non-random-access iterators, O(N^2) iterator increments, whereas the
one-sided version will yield O(N) operations on both counts, with a
best-case of O(log(N)) comparisons which is very common in practice.
---
 libcxx/benchmarks/CMakeLists.txt              |   1 +
 .../algorithms/set_intersection.bench.cpp     | 184 ++++++++
 libcxx/docs/ReleaseNotes/19.rst               |   4 +
 .../include/__algorithm/iterator_operations.h |  54 +++
 libcxx/include/__algorithm/lower_bound.h      |  54 ++-
 libcxx/include/__algorithm/set_intersection.h | 122 +++++-
 .../ranges_set_intersection.pass.cpp          |  71 +--
 .../set_intersection_complexity.pass.cpp      | 404 ++++++++++++++++++
 .../iterator_count.pass.cpp                   |  11 +-
 .../iterator_count_sentinel.pass.cpp          |  63 +--
 .../iterator_sentinel.pass.cpp                |  28 +-
 .../range.adaptors/range.drop/begin.pass.cpp  |   8 +-
 .../ranges/range.adaptors/range.drop/types.h  |  10 +-
 .../range.adaptors/range.transform/types.h    |   6 -
 libcxx/test/support/test_iterators.h          | 184 ++++----
 15 files changed, 985 insertions(+), 219 deletions(-)
 create mode 100644 libcxx/benchmarks/algorithms/set_intersection.bench.cpp
 create mode 100644 libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection_complexity.pass.cpp

diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt
index 110672600213a..d96ccc1e49f66 100644
--- a/libcxx/benchmarks/CMakeLists.txt
+++ b/libcxx/benchmarks/CMakeLists.txt
@@ -135,6 +135,7 @@ set(BENCHMARK_TESTS
     algorithms/ranges_sort.bench.cpp
     algorithms/ranges_sort_heap.bench.cpp
     algorithms/ranges_stable_sort.bench.cpp
+    algorithms/set_intersection.bench.cpp
     algorithms/sort.bench.cpp
     algorithms/sort_heap.bench.cpp
     algorithms/stable_sort.bench.cpp
diff --git a/libcxx/benchmarks/algorithms/set_intersection.bench.cpp b/libcxx/benchmarks/algorithms/set_intersection.bench.cpp
new file mode 100644
index 0000000000000..b3fb15fc77b31
--- /dev/null
+++ b/libcxx/benchmarks/algorithms/set_intersection.bench.cpp
@@ -0,0 +1,184 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <algorithm>
+#include <cstdlib>
+#include <iterator>
+#include <set>
+#include <vector>
+
+#include "common.h"
+#include "test_iterators.h"
+
+namespace {
+
+// types of containers we'll want to test, covering interesting iterator types
+struct VectorContainer {
+  template <typename... Args>
+  using type = std::vector<Args...>;
+
+  static constexpr const char* Name = "Vector";
+};
+
+struct SetContainer {
+  template <typename... Args>
+  using type = std::set<Args...>;
+
+  static constexpr const char* Name = "Set";
+};
+
+using AllContainerTypes = std::tuple<VectorContainer, SetContainer>;
+
+// set_intersection performance may depend on where matching values lie
+enum class OverlapPosition {
+  None,
+  Front,
+  // performance-wise, matches at the back are identical to ones at the front
+  Interlaced,
+};
+
+struct AllOverlapPositions : EnumValuesAsTuple<AllOverlapPositions, OverlapPosition, 3> {
+  static constexpr const char* Names[] = {"None", "Front", "Interlaced"};
+};
+
+// forward_iterator wrapping which, for each increment, moves the underlying iterator forward Stride elements
+template <typename Wrapped>
+struct StridedFwdIt {
+  Wrapped base_;
+  unsigned stride_;
+
+  using iterator_category = std::forward_iterator_tag;
+  using difference_type   = typename Wrapped::difference_type;
+  using value_type        = typename Wrapped::value_type;
+  using pointer           = typename Wrapped::pointer;
+  using reference         = typename Wrapped::reference;
+
+  StridedFwdIt(Wrapped base, unsigned stride) : base_(base), stride_(stride) { assert(stride_ != 0); }
+
+  StridedFwdIt operator++() {
+    for (unsigned i = 0; i < stride_; ++i)
+      ++base_;
+    return *this;
+  }
+  StridedFwdIt operator++(int) {
+    auto tmp = *this;
+    ++*this;
+    return tmp;
+  }
+  value_type& operator*() { return *base_; }
+  const value_type& operator*() const { return *base_; }
+  value_type& operator->() { return *base_; }
+  const value_type& operator->() const { return *base_; }
+  bool operator==(const StridedFwdIt& o) const { return base_ == o.base_; }
+  bool operator!=(const StridedFwdIt& o) const { return !operator==(o); }
+};
+template <typename Wrapped>
+StridedFwdIt(Wrapped, unsigned) -> StridedFwdIt<Wrapped>;
+
+template <typename T>
+std::vector<T> getVectorOfRandom(size_t N) {
+  std::vector<T> v;
+  fillValues(v, N, Order::Random);
+  sortValues(v, Order::Random);
+  return std::vector<T>(v);
+}
+
+// Realistically, data won't all be nicely contiguous in a container,
+// we'll go through some effort to ensure that it's shuffled through memory
+// this is especially important for containers with non-contiguous element
+// storage, but it will affect even a std::vector, because when you copy a
+// std::vector<std::string> the underlying data storage position for the char
+// arrays of the copy are likely to have high locality
+template <class Container>
+std::pair<Container, Container> genCacheUnfriendlyData(size_t size1, size_t size2, OverlapPosition pos) {
+  using ValueType = typename Container::value_type;
+  auto move_into  = [](auto first, auto last) {
+    Container out;
+    std::move(first, last, std::inserter(out, out.begin()));
+    return out;
+  };
+  const auto src_size        = pos == OverlapPosition::None ? size1 + size2 : std::max(size1, size2);
+  std::vector<ValueType> src = getVectorOfRandom<ValueType>(src_size);
+
+  if (pos == OverlapPosition::None) {
+    std::sort(src.begin(), src.end());
+    return std::make_pair(move_into(src.begin(), src.begin() + size1), move_into(src.begin() + size1, src.end()));
+  }
+
+  // All other overlap types will have to copy some part of the data, but if
+  // we copy after sorting it will likely have high locality, so we sort
+  // each copy separately
+  auto copy = src;
+  std::sort(src.begin(), src.end());
+  std::sort(copy.begin(), copy.end());
+
+  switch (pos) {
+  case OverlapPosition::None:
+    // we like -Wswitch :)
+    break;
+
+  case OverlapPosition::Front:
+    return std::make_pair(move_into(src.begin(), src.begin() + size1), move_into(copy.begin(), copy.begin() + size2));
+
+  case OverlapPosition::Interlaced:
+    const auto stride1 = size1 < size2 ? size2 / size1 : 1;
+    const auto stride2 = size2 < size1 ? size1 / size2 : 1;
+    return std::make_pair(move_into(StridedFwdIt(src.begin(), stride1), StridedFwdIt(src.end(), stride1)),
+                          move_into(StridedFwdIt(copy.begin(), stride2), StridedFwdIt(copy.end(), stride2)));
+  }
+  std::abort(); // would be std::unreachable() if it could
+  return std::pair<Container, Container>();
+}
+
+template <class ValueType, class Container, class Overlap>
+struct SetIntersection {
+  using ContainerType = typename Container::template type<Value<ValueType>>;
+  size_t size1_;
+  size_t size2_;
+
+  SetIntersection(size_t size1, size_t size2) : size1_(size1), size2_(size2) {}
+
+  bool skip() const noexcept {
+    // let's save some time and skip simmetrical runs
+    return size1_ < size2_;
+  }
+
+  void run(benchmark::State& state) const {
+    auto input = genCacheUnfriendlyData<ContainerType>(size1_, size2_, Overlap());
+    std::vector<Value<ValueType>> out(std::min(size1_, size2_));
+
+    const auto BATCH_SIZE = std::max(size_t{512}, (2 * TestSetElements) / (size1_ + size2_));
+    for (const auto& _ : state) {
+      while (state.KeepRunningBatch(BATCH_SIZE)) {
+        for (unsigned i = 0; i < BATCH_SIZE; ++i) {
+          const auto& [c1, c2] = input;
+          auto res             = std::set_intersection(c1.begin(), c1.end(), c2.begin(), c2.end(), out.begin());
+          benchmark::DoNotOptimize(res);
+        }
+      }
+    }
+  }
+
+  std::string name() const {
+    return std::string("SetIntersection") + Overlap::name() + '_' + Container::Name + ValueType::name() + '_' +
+           std::to_string(size1_) + '_' + std::to_string(size2_);
+  }
+};
+
+} // namespace
+
+int main(int argc, char** argv) { /**/
+  benchmark::Initialize(&argc, argv);
+  if (benchmark::ReportUnrecognizedArguments(argc, argv))
+    return 1;
+
+  makeCartesianProductBenchmark<SetIntersection, AllValueTypes, AllContainerTypes, AllOverlapPositions>(
+      Quantities, Quantities);
+  benchmark::RunSpecifiedBenchmarks();
+  return 0;
+}
diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 624550f998858..ccafa1a1456e0 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -71,6 +71,10 @@ Improvements and New Features
 - The ``std::ranges::minmax`` algorithm has been optimized for integral types, resulting in a performance increase of
   up to 100x.
 
+- The ``std::set_intersection`` and ``std::ranges::set_intersection`` algorithms have been optimized to fast-forward over
+  contiguous ranges of non-matching values, reducing the number of comparisons from linear to 
+  logarithmic growth with the number of elements in best-case scenarios.
+
 - The ``_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM`` macro has been added to make the declarations in ``<strstream>`` available.
 
 - The ``_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT`` macro has been added to make the declarations in ``<locale>``
diff --git a/libcxx/include/__algorithm/iterator_operations.h b/libcxx/include/__algorithm/iterator_operations.h
index 5cf13f0a3f292..8ced989233bc4 100644
--- a/libcxx/include/__algorithm/iterator_operations.h
+++ b/libcxx/include/__algorithm/iterator_operations.h
@@ -11,6 +11,7 @@
 
 #include <__algorithm/iter_swap.h>
 #include <__algorithm/ranges_iterator_concept.h>
+#include <__assert>
 #include <__config>
 #include <__iterator/advance.h>
 #include <__iterator/distance.h>
@@ -160,6 +161,59 @@ struct _IterOps<_ClassicAlgPolicy> {
   _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 void __advance_to(_Iter& __first, _Iter __last) {
     __first = __last;
   }
+
+  // advance with sentinel, a la std::ranges::advance
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static __difference_type<_Iter>
+  __advance_to(_Iter& __iter, __difference_type<_Iter> __count, const _Iter& __sentinel) {
+    return _IterOps::__advance_to(__iter, __count, __sentinel, typename iterator_traits<_Iter>::iterator_category());
+  }
+
+private:
+  // advance with sentinel, a la std::ranges::advance -- InputIterator specialization
+  template <class _InputIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static __difference_type<_InputIter> __advance_to(
+      _InputIter& __iter, __difference_type<_InputIter> __count, const _InputIter& __sentinel, input_iterator_tag) {
+    __difference_type<_InputIter> __dist = 0;
+    for (; __dist < __count && __iter != __sentinel; ++__dist)
+      ++__iter;
+    return __count - __dist;
+  }
+
+  // advance with sentinel, a la std::ranges::advance -- BidirectionalIterator specialization
+  template <class _BiDirIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static __difference_type<_BiDirIter>
+  __advance_to(_BiDirIter& __iter,
+               __difference_type<_BiDirIter> __count,
+               const _BiDirIter& __sentinel,
+               bidirectional_iterator_tag) {
+    __difference_type<_BiDirIter> __dist = 0;
+    if (__count >= 0)
+      for (; __dist < __count && __iter != __sentinel; ++__dist)
+        ++__iter;
+    else
+      for (__count = -__count; __dist < __count && __iter != __sentinel; ++__dist)
+        --__iter;
+    return __count - __dist;
+  }
+
+  // advance with sentinel, a la std::ranges::advance -- RandomIterator specialization
+  template <class _RandIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static __difference_type<_RandIter>
+  __advance_to(_RandIter& __iter,
+               __difference_type<_RandIter> __count,
+               const _RandIter& __sentinel,
+               random_access_iterator_tag) {
+    auto __dist = _IterOps::distance(__iter, __sentinel);
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(
+        __count == 0 || (__dist < 0) == (__count < 0), "__sentinel must precede __iter when __count < 0");
+    if (__count < 0)
+      __dist = __dist > __count ? __dist : __count;
+    else
+      __dist = __dist < __count ? __dist : __count;
+    __iter += __dist;
+    return __count - __dist;
+  }
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__algorithm/lower_bound.h b/libcxx/include/__algorithm/lower_bound.h
index 8fd355a7cfc4a..c417d84835497 100644
--- a/libcxx/include/__algorithm/lower_bound.h
+++ b/libcxx/include/__algorithm/lower_bound.h
@@ -27,11 +27,13 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _AlgPolicy, class _Iter, class _Sent, class _Type, class _Proj, class _Comp>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter
-__lower_bound(_Iter __first, _Sent __last, const _Type& __value, _Comp& __comp, _Proj& __proj) {
-  auto __len = _IterOps<_AlgPolicy>::distance(__first, __last);
-
+template <class _AlgPolicy, class _Iter, class _Type, class _Proj, class _Comp>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter __lower_bound_bisecting(
+    _Iter __first,
+    const _Type& __value,
+    typename iterator_traits<_Iter>::difference_type __len,
+    _Comp& __comp,
+    _Proj& __proj) {
   while (__len != 0) {
     auto __l2 = std::__half_positive(__len);
     _Iter __m = __first;
@@ -46,6 +48,48 @@ __lower_bound(_Iter __first, _Sent __last, const _Type& __value, _Comp& __comp,
   return __first;
 }
 
+// One-sided binary search, aka meta binary search, has been in the public domain for decades, and has the general
+// advantage of being \Omega(1) rather than the classic algorithm's \Omega(log(n)), with the downside of executing at
+// most 2*log(n) comparisons vs the classic algorithm's exact log(n). There are two scenarios in which it really shines:
+// the first one is when operating over non-random-access iterators, because the classic algorithm requires knowing the
+// container's size upfront, which adds \Omega(n) iterator increments to the complexity. The second one is when you're
+// traversing the container in order, trying to fast-forward to the next value: in that case, the classic algorithm
+// would yield \Omega(n*log(n)) comparisons and, for non-random-access iterators, \Omega(n^2) iterator increments,
+// whereas the one-sided version will yield O(n) operations on both counts, with a \Omega(log(n)) bound on the number of
+// comparisons.
+template <class _AlgPolicy, class _ForwardIterator, class _Sent, class _Type, class _Proj, class _Comp>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+__lower_bound_onesided(_ForwardIterator __first, _Sent __last, const _Type& __value, _Comp& __comp, _Proj& __proj) {
+  // step = 0, ensuring we can always short-circuit when distance is 1 later on
+  if (__first == __last || !std::__invoke(__comp, std::__invoke(__proj, *__first), __value))
+    return __first;
+
+  using _Distance = typename iterator_traits<_ForwardIterator>::difference_type;
+  for (_Distance __step = 1; __first != __last; __step <<= 1) {
+    auto __it   = __first;
+    auto __dist = __step - _IterOps<_AlgPolicy>::__advance_to(__it, __step, __last);
+    // once we reach the last range where needle can be we must start
+    // looking inwards, bisecting that range
+    if (__it == __last || !std::__invoke(__comp, std::__invoke(__proj, *__it), __value)) {
+      // we've already checked the previous value and it was less, we can save
+      // one comparison by skipping bisection
+      if (__dist == 1)
+        return __it;
+      return std::__lower_bound_bisecting<_AlgPolicy>(__first, __value, __dist, __comp, __proj);
+    }
+    // range not found, move forward!
+    __first = __it;
+  }
+  return __first;
+}
+
+template <class _AlgPolicy, class _ForwardIterator, class _Sent, class _Type, class _Proj, class _Comp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+__lower_bound(_ForwardIterator __first, _Sent __last, const _Type& __value, _Comp& __comp, _Proj& __proj) {
+  const auto __dist = _IterOps<_AlgPolicy>::distance(__first, __last);
+  return std::__lower_bound_bisecting<_AlgPolicy>(__first, __value, __dist, __comp, __proj);
+}
+
 template <class _ForwardIterator, class _Tp, class _Compare>
 _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
 lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, _Compare __comp) {
diff --git a/libcxx/include/__algorithm/set_intersection.h b/libcxx/include/__algorithm/set_intersection.h
index 73d888d1b0384..bb0d86cd0f58d 100644
--- a/libcxx/include/__algorithm/set_intersection.h
+++ b/libcxx/include/__algorithm/set_intersection.h
@@ -12,10 +12,15 @@
 #include <__algorithm/comp.h>
 #include <__algorithm/comp_ref_type.h>
 #include <__algorithm/iterator_operations.h>
+#include <__algorithm/lower_bound.h>
 #include <__config>
+#include <__functional/identity.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/next.h>
+#include <__type_traits/is_same.h>
+#include <__utility/exchange.h>
 #include <__utility/move.h>
+#include <__utility/swap.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -38,10 +43,103 @@ struct __set_intersection_result {
       : __in1_(std::move(__in_iter1)), __in2_(std::move(__in_iter2)), __out_(std::move(__out_iter)) {}
 };
 
-template <class _AlgPolicy, class _Compare, class _InIter1, class _Sent1, class _InIter2, class _Sent2, class _OutIter>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __set_intersection_result<_InIter1, _InIter2, _OutIter>
+// Helper for __set_intersection() with one-sided binary search: populate result and advance input iterators if they
+// are found to potentially contain the same value in two consecutive calls. This function is very intimately related to
+// the way it is used and doesn't attempt to abstract that, it's not appropriate for general usage outside of its
+// context.
+template <class _InForwardIter1, class _InForwardIter2, class _OutIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __set_intersection_add_output_if_equal(
+    bool __may_be_equal,
+    _InForwardIter1& __first1,
+    _InForwardIter2& __first2,
+    _OutIter& __result,
+    bool& __prev_may_be_equal) {
+  if (__may_be_equal && __prev_may_be_equal) {
+    *__result = *__first1;
+    ++__result;
+    ++__first1;
+    ++__first2;
+    __prev_may_be_equal = false;
+  } else {
+    __prev_may_be_equal = __may_be_equal;
+  }
+}
+
+// With forward iterators we can make multiple passes over the data, allowing the use of one-sided binary search to
+// reduce best-case complexity to log(N). Understanding how we can use binary search and still respect complexity
+// guarantees is _not_ straightforward: the guarantee is "at most 2*(N+M)-1 comparisons", and one-sided binary search
+// will necessarily overshoot depending on the position of the needle in the haystack -- for instance, if we're
+// searching for 3 in (1, 2, 3, 4), we'll check if 3<1, then 3<2, then 3<4, and, finally, 3<3, for a total of 4
+// comparisons, when linear search would have yielded 3. However, because we won't need to perform the intervening
+// reciprocal comparisons (ie 1<3, 2<3, 4<3), that extra comparison doesn't run afoul of the guarantee. Additionally,
+// this type of scenario can only happen for match distances of up to 5 elements, because 2*log2(8) is 6, and we'll
+// still be worse-off at position 5 of an 8-element set. From then onwards these scenarios can't happen. TL;DR: we'll be
+// 1 comparison worse-off compared to the classic linear-searching algorithm if matching position 3 of a set with 4
+// elements, or position 5 if the set has 7 or 8 elements, but we'll never exceed the complexity guarantees from the
+// standard.
+template <class _AlgPolicy,
+          class _Compare,
+          class _InForwardIter1,
+          class _Sent1,
+          class _InForwardIter2,
+          class _Sent2,
+          class _OutIter>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX20 __set_intersection_result<_InForwardIter1, _InForwardIter2, _OutIter>
 __set_intersection(
-    _InIter1 __first1, _Sent1 __last1, _InIter2 __first2, _Sent2 __last2, _OutIter __result, _Compare&& __comp) {
+    _InForwardIter1 __first1,
+    _Sent1 __last1,
+    _InForwardIter2 __first2,
+    _Sent2 __last2,
+    _OutIter __result,
+    _Compare&& __comp,
+    std::forward_iterator_tag,
+    std::forward_iterator_tag) {
+  _LIBCPP_CONSTEXPR std::__identity __proj;
+  bool __prev_may_be_equal = false;
+
+  while (__first2 != __last2) {
+    _InForwardIter1 __first1_next =
+        std::__lower_bound_onesided<_AlgPolicy>(__first1, __last1, *__first2, __comp, __proj);
+    std::swap(__first1_next, __first1);
+    // keeping in mind that a==b iff !(a<b) && !(b<a):
+    // if we can't advance __first1, that means !(*__first1 < *_first2), therefore __may_be_equal==true
+    std::__set_intersection_add_output_if_equal(
+        __first1 == __first1_next, __first1, __first2, __result, __prev_may_be_equal);
+    if (__first1 == __last1)
+      break;
+
+    _InForwardIter2 __first2_next =
+        std::__lower_bound_onesided<_AlgPolicy>(__first2, __last2, *__first1, __comp, __proj);
+    std::swap(__first2_next, __first2);
+    std::__set_intersection_add_output_if_equal(
+        __first2 == __first2_next, __first1, __first2, __result, __prev_may_be_equal);
+  }
+  return __set_intersection_result<_InForwardIter1, _InForwardIter2, _OutIter>(
+      _IterOps<_AlgPolicy>::next(std::move(__first1), std::move(__last1)),
+      _IterOps<_AlgPolicy>::next(std::move(__first2), std::move(__last2)),
+      std::move(__result));
+}
+
+// input iterators are not suitable for multipass algorithms, so we stick to the classic single-pass version
+template <class _AlgPolicy,
+          class _Compare,
+          class _InInputIter1,
+          class _Sent1,
+          class _InInputIter2,
+          class _Sent2,
+          class _OutIter>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX20 __set_intersection_result<_InInputIter1, _InInputIter2, _OutIter>
+__set_intersection(
+    _InInputIter1 __first1,
+    _Sent1 __last1,
+    _InInputIter2 __first2,
+    _Sent2 __last2,
+    _OutIter __result,
+    _Compare&& __comp,
+    std::input_iterator_tag,
+    std::input_iterator_tag) {
   while (__first1 != __last1 && __first2 != __last2) {
     if (__comp(*__first1, *__first2))
       ++__first1;
@@ -55,12 +153,28 @@ __set_intersection(
     }
   }
 
-  return __set_intersection_result<_InIter1, _InIter2, _OutIter>(
+  return __set_intersection_result<_InInputIter1, _InInputIter2, _OutIter>(
       _IterOps<_AlgPolicy>::next(std::move(__first1), std::move(__last1)),
       _IterOps<_AlgPolicy>::next(std::move(__first2), std::move(__last2)),
       std::move(__result));
 }
 
+template <class _AlgPolicy, class _Compare, class _InIter1, class _Sent1, class _InIter2, class _Sent2, class _OutIter>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX20 __set_intersection_result<_InIter1, _InIter2, _OutIter>
+__set_intersection(
+    _InIter1 __first1, _Sent1 __last1, _InIter2 __first2, _Sent2 __last2, _OutIter __result, _Compare&& __comp) {
+  return std::__set_intersection<_AlgPolicy>(
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      std::move(__result),
+      std::forward<_Compare>(__comp),
+      typename std::_IterOps<_AlgPolicy>::template __iterator_category<_InIter1>(),
+      typename std::_IterOps<_AlgPolicy>::template __iterator_category<_InIter2>());
+}
+
 template <class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator set_intersection(
     _InputIterator1 __first1,
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/ranges_set_intersection.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/ranges_set_intersection.pass.cpp
index 5323bb1bc1193..f7870485cfefc 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/ranges_set_intersection.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/ranges_set_intersection.pass.cpp
@@ -28,6 +28,8 @@
 #include <algorithm>
 #include <array>
 #include <concepts>
+#include <cstddef>
+#include <ranges>
 
 #include "almost_satisfies_types.h"
 #include "MoveOnly.h"
@@ -463,75 +465,6 @@ constexpr bool test() {
     }
   }
 
-  // Complexity: At most 2 * ((last1 - first1) + (last2 - first2)) - 1 comparisons and applications of each projection.
-  {
-    std::array<Data, 5> r1{{{1}, {3}, {5}, {7}, {9}}};
-    std::array<Data, 5> r2{{{2}, {4}, {6}, {8}, {10}}};
-    std::array<int, 0> expected{};
-
-    const std::size_t maxOperation = 2 * (r1.size() + r2.size()) - 1;
-
-    // iterator overload
-    {
-      std::array<Data, 0> out{};
-      std::size_t numberOfComp  = 0;
-      std::size_t numberOfProj1 = 0;
-      std::size_t numberOfProj2 = 0;
-
-      const auto comp = [&numberOfComp](int x, int y) {
-        ++numberOfComp;
-        return x < y;
-      };
-
-      const auto proj1 = [&numberOfProj1](const Data& d) {
-        ++numberOfProj1;
-        return d.data;
-      };
-
-      const auto proj2 = [&numberOfProj2](const Data& d) {
-        ++numberOfProj2;
-        return d.data;
-      };
-
-      std::ranges::set_intersection(r1.begin(), r1.end(), r2.begin(), r2.end(), out.data(), comp, proj1, proj2);
-
-      assert(std::ranges::equal(out, expected, {}, &Data::data));
-      assert(numberOfComp < maxOperation);
-      assert(numberOfProj1 < maxOperation);
-      assert(numberOfProj2 < maxOperation);
-    }
-
-    // range overload
-    {
-      std::array<Data, 0> out{};
-      std::size_t numberOfComp  = 0;
-      std::size_t numberOfProj1 = 0;
-      std::size_t numberOfProj2 = 0;
-
-      const auto comp = [&numberOfComp](int x, int y) {
-        ++numberOfComp;
-        return x < y;
-      };
-
-      const auto proj1 = [&numberOfProj1](const Data& d) {
-        ++numberOfProj1;
-        return d.data;
-      };
-
-      const auto proj2 = [&numberOfProj2](const Data& d) {
-        ++numberOfProj2;
-        return d.data;
-      };
-
-      std::ranges::set_intersection(r1, r2, out.data(), comp, proj1, proj2);
-
-      assert(std::ranges::equal(out, expected, {}, &Data::data));
-      assert(numberOfComp < maxOperation);
-      assert(numberOfProj1 < maxOperation);
-      assert(numberOfProj2 < maxOperation);
-    }
-  }
-
   // Comparator convertible to bool
   {
     struct ConvertibleToBool {
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection_complexity.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection_complexity.pass.cpp
new file mode 100644
index 0000000000000..ddf4087ddd6cd
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection_complexity.pass.cpp
@@ -0,0 +1,404 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <algorithm>
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// Algorithmic complexity tests for both std::set_intersection and std::ranges::set_intersection
+
+// template<InputIterator InIter1, InputIterator InIter2, typename OutIter>
+//   requires OutputIterator<OutIter, InIter1::reference>
+//         && OutputIterator<OutIter, InIter2::reference>
+//         && HasLess<InIter2::value_type, InIter1::value_type>
+//         && HasLess<InIter1::value_type, InIter2::value_type>
+//   constexpr OutIter       // constexpr after C++17
+//   set_intersection(InIter1 first1, InIter1 last1, InIter2 first2, InIter2 last2,
+//                    OutIter result);
+//
+// template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+//          weakly_incrementable O, class Comp = ranges::less,
+//          class Proj1 = identity, class Proj2 = identity>
+//   requires mergeable<I1, I2, O, Comp, Proj1, Proj2>
+//   constexpr set_intersection_result<I1, I2, O>
+//     set_intersection(I1 first1, S1 last1, I2 first2, S2 last2, O result,
+//                      Comp comp = {}, Proj1 proj1 = {}, Proj2 proj2 = {});                         // since C++20
+//
+// template<input_range R1, input_range R2, weakly_incrementable O,
+//          class Comp = ranges::less, class Proj1 = identity, class Proj2 = identity>
+//   requires mergeable<iterator_t<R1>, iterator_t<R2>, O, Comp, Proj1, Proj2>
+//   constexpr set_intersection_result<borrowed_iterator_t<R1>, borrowed_iterator_t<R2>, O>
+//     set_intersection(R1&& r1, R2&& r2, O result,
+//                      Comp comp = {}, Proj1 proj1 = {}, Proj2 proj2 = {});                         // since C++20
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <ranges>
+
+#include "test_iterators.h"
+
+namespace {
+
+// __debug_less will perform an additional comparison in an assertion
+static constexpr unsigned std_less_comparison_count_multiplier() noexcept {
+#if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
+  return 2;
+#else
+  return 1;
+#endif
+}
+
+struct [[nodiscard]] OperationCounts {
+  std::size_t comparisons{};
+  struct PerInput {
+    std::size_t proj{};
+    IteratorOpCounts iterops;
+
+    [[nodiscard]] constexpr bool isNotBetterThan(const PerInput& other) {
+      return proj >= other.proj && iterops.increments + iterops.decrements + iterops.zero_moves >=
+                                       other.iterops.increments + other.iterops.decrements + other.iterops.zero_moves;
+    }
+  };
+  std::array<PerInput, 2> in;
+
+  [[nodiscard]] constexpr bool isNotBetterThan(const OperationCounts& expect) {
+    return std_less_comparison_count_multiplier() * comparisons >= expect.comparisons &&
+           in[0].isNotBetterThan(expect.in[0]) && in[1].isNotBetterThan(expect.in[1]);
+  }
+};
+
+template <std::size_t ResultSize>
+struct counted_set_intersection_result {
+  std::array<int, ResultSize> result;
+  OperationCounts opcounts;
+
+  constexpr counted_set_intersection_result() = default;
+
+  constexpr explicit counted_set_intersection_result(std::array<int, ResultSize>&& contents) : result{contents} {}
+
+  constexpr void assertNotBetterThan(const counted_set_intersection_result& other) {
+    assert(result == other.result);
+    assert(opcounts.isNotBetterThan(other.opcounts));
+  }
+};
+
+template <std::size_t ResultSize>
+counted_set_intersection_result(std::array<int, ResultSize>) -> counted_set_intersection_result<ResultSize>;
+
+template <template <class...> class InIterType1,
+          template <class...>
+          class InIterType2,
+          class OutIterType,
+          std::size_t ResultSize,
+          std::ranges::input_range R1,
+          std::ranges::input_range R2>
+constexpr counted_set_intersection_result<ResultSize> counted_set_intersection(const R1& in1, const R2& in2) {
+  counted_set_intersection_result<ResultSize> out;
+
+  const auto comp = [&out](int x, int y) {
+    ++out.opcounts.comparisons;
+    return x < y;
+  };
+
+  operation_counting_iterator b1(InIterType1<decltype(in1.begin())>(in1.begin()), &out.opcounts.in[0].iterops);
+  operation_counting_iterator e1(InIterType1<decltype(in1.end()) >(in1.end()), &out.opcounts.in[0].iterops);
+
+  operation_counting_iterator b2(InIterType2<decltype(in2.begin())>(in2.begin()), &out.opcounts.in[1].iterops);
+  operation_counting_iterator e2(InIterType2<decltype(in2.end()) >(in2.end()), &out.opcounts.in[1].iterops);
+
+  std::set_intersection(b1, e1, b2, e2, OutIterType(out.result.data()), comp);
+
+  return out;
+}
+
+template <template <class...> class InIterType1,
+          template <class...>
+          class InIterType2,
+          class OutIterType,
+          std::size_t ResultSize,
+          std::ranges::input_range R1,
+          std::ranges::input_range R2>
+constexpr counted_set_intersection_result<ResultSize> counted_ranges_set_intersection(const R1& in1, const R2& in2) {
+  counted_set_intersection_result<ResultSize> out;
+
+  const auto comp = [&out](int x, int y) {
+    ++out.opcounts.comparisons;
+    return x < y;
+  };
+
+  const auto proj1 = [&out](const int& i) {
+    ++out.opcounts.in[0].proj;
+    return i;
+  };
+
+  const auto proj2 = [&out](const int& i) {
+    ++out.opcounts.in[1].proj;
+    return i;
+  };
+
+  operation_counting_iterator b1(InIterType1<decltype(in1.begin())>(in1.begin()), &out.opcounts.in[0].iterops);
+  operation_counting_iterator e1(InIterType1<decltype(in1.end()) >(in1.end()), &out.opcounts.in[0].iterops);
+
+  operation_counting_iterator b2(InIterType2<decltype(in2.begin())>(in2.begin()), &out.opcounts.in[1].iterops);
+  operation_counting_iterator e2(InIterType2<decltype(in2.end()) >(in2.end()), &out.opcounts.in[1].iterops);
+
+  std::ranges::subrange r1{b1, sentinel_wrapper<decltype(e1)>{e1}};
+  std::ranges::subrange r2{b2, sentinel_wrapper<decltype(e2)>{e2}};
+  std::same_as<std::ranges::set_intersection_result<decltype(e1), decltype(e2), OutIterType>> decltype(auto) result =
+      std::ranges::set_intersection(r1, r2, OutIterType{out.result.data()}, comp, proj1, proj2);
+  assert(base(result.in1) == base(e1));
+  assert(base(result.in2) == base(e2));
+  assert(base(result.out) == out.result.data() + out.result.size());
+
+  return out;
+}
+
+template <template <typename...> class In1, template <typename...> class In2, class Out>
+constexpr void testComplexityParameterizedIter() {
+  // Worst-case complexity:
+  // Let N=(last1 - first1) and M=(last2 - first2)
+  // At most 2*(N+M) - 1 comparisons and applications of each projection.
+  // At most 2*(N+M) iterator mutations.
+  {
+    std::array r1{1, 3, 5, 7, 9, 11, 13, 15, 17, 19};
+    std::array r2{2, 4, 6, 8, 10, 12, 14, 16, 18, 20};
+
+    counted_set_intersection_result<0> expected;
+    expected.opcounts.comparisons              = 37;
+    expected.opcounts.in[0].proj               = 37;
+    expected.opcounts.in[0].iterops.increments = 30;
+    expected.opcounts.in[0].iterops.decrements = 0;
+    expected.opcounts.in[1]                    = expected.opcounts.in[0];
+
+    expected.assertNotBetterThan(counted_set_intersection<In1, In2, Out, expected.result.size()>(r1, r2));
+    expected.assertNotBetterThan(counted_ranges_set_intersection<In1, In2, Out, expected.result.size()>(r1, r2));
+  }
+
+  {
+    std::array r1{1, 3, 5, 7, 9, 11, 13, 15, 17, 19};
+    std::array r2{1, 3, 5, 7, 9, 11, 13, 15, 17, 19};
+
+    counted_set_intersection_result expected(std::array{1, 3, 5, 7, 9, 11, 13, 15, 17, 19});
+    expected.opcounts.comparisons              = 38;
+    expected.opcounts.in[0].proj               = 38;
+    expected.opcounts.in[0].iterops.increments = 30;
+    expected.opcounts.in[0].iterops.decrements = 0;
+    expected.opcounts.in[1]                    = expected.opcounts.in[0];
+
+    expected.assertNotBetterThan(counted_set_intersection<In1, In2, Out, expected.result.size()>(r1, r2));
+    expected.assertNotBetterThan(counted_ranges_set_intersection<In1, In2, Out, expected.result.size()>(r1, r2));
+  }
+
+  // Lower complexity when there is low overlap between ranges: we can make 2*log(X) comparisons when one range
+  // has X elements that can be skipped over (and then 1 more to confirm that the value we found is equal).
+  {
+    std::array r1{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+    std::array r2{15};
+
+    counted_set_intersection_result expected(std::array{15});
+    expected.opcounts.comparisons              = 9;
+    expected.opcounts.in[0].proj               = 9;
+    expected.opcounts.in[0].iterops.increments = 23;
+    expected.opcounts.in[0].iterops.decrements = 0;
+    expected.opcounts.in[1].proj               = 9;
+    expected.opcounts.in[1].iterops.increments = 1;
+    expected.opcounts.in[1].iterops.decrements = 0;
+
+    expected.assertNotBetterThan(counted_set_intersection<In1, In2, Out, expected.result.size()>(r1, r2));
+    expected.assertNotBetterThan(counted_ranges_set_intersection<In1, In2, Out, expected.result.size()>(r1, r2));
+  }
+
+  {
+    std::array r1{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+    std::array r2{0, 16};
+    counted_set_intersection_result<0> expected;
+
+    expected.opcounts.comparisons              = 10;
+    expected.opcounts.in[0].proj               = 10;
+    expected.opcounts.in[0].iterops.increments = 24;
+    expected.opcounts.in[0].iterops.decrements = 0;
+    expected.opcounts.in[1].proj               = 10;
+    expected.opcounts.in[1].iterops.increments = 4;
+    expected.opcounts.in[1].iterops.decrements = 0;
+
+    expected.assertNotBetterThan(counted_set_intersection<In1, In2, Out, expected.result.size()>(r1, r2));
+    expected.assertNotBetterThan(counted_ranges_set_intersection<In1, In2, Out, expected.result.size()>(r1, r2));
+  }
+}
+
+template <template <typename...> class In2, class Out>
+constexpr void testComplexityParameterizedIterPermutateIn1() {
+  //common_input_iterator
+  testComplexityParameterizedIter<forward_iterator, In2, Out>();
+  testComplexityParameterizedIter<bidirectional_iterator, In2, Out>();
+  testComplexityParameterizedIter<random_access_iterator, In2, Out>();
+}
+
+template <class Out>
+constexpr void testComplexityParameterizedIterPermutateIn1In2() {
+  testComplexityParameterizedIterPermutateIn1<forward_iterator, Out>();
+  testComplexityParameterizedIterPermutateIn1<bidirectional_iterator, Out>();
+  testComplexityParameterizedIterPermutateIn1<random_access_iterator, Out>();
+}
+
+constexpr bool testComplexity() {
+  testComplexityParameterizedIterPermutateIn1In2<forward_iterator<int*>>();
+  testComplexityParameterizedIterPermutateIn1In2<bidirectional_iterator<int*>>();
+  testComplexityParameterizedIterPermutateIn1In2<random_access_iterator<int*>>();
+  return true;
+}
+
+template <template <typename...> class In1, template <typename...> class In2, class Out>
+constexpr void testComplexityGuaranteesParameterizedIter() {
+  // now a more generic validation of the complexity guarantees when searching for a single value
+  for (unsigned range_size = 1; range_size < 20; ++range_size) {
+    std::ranges::iota_view<int, int> r1(0, range_size);
+    for (int i : r1) {
+      // At most 2 * ((last1 - first1) + (last2 - first2)) - 1 comparisons
+      counted_set_intersection_result<1> expected(std::array{i});
+      expected.opcounts.comparisons              = 2 * (r1.size() + 1) - 1;
+      expected.opcounts.in[0].proj               = expected.opcounts.comparisons;
+      expected.opcounts.in[1].proj               = expected.opcounts.comparisons;
+      expected.opcounts.in[0].iterops.increments = 2 * r1.size();
+      expected.opcounts.in[1].iterops.increments = 2;
+      expected.opcounts.in[0].iterops.decrements = expected.opcounts.in[0].iterops.increments;
+      expected.opcounts.in[1].iterops.decrements = expected.opcounts.in[1].iterops.increments;
+
+      expected.assertNotBetterThan(
+          counted_set_intersection<In1, In2, Out, expected.result.size()>(r1, expected.result));
+      expected.assertNotBetterThan(
+          counted_ranges_set_intersection<In1, In2, Out, expected.result.size()>(r1, expected.result));
+    }
+  }
+}
+
+template <template <typename...> class In2, class Out>
+constexpr void testComplexityGuaranteesParameterizedIterPermutateIn1() {
+  //common_input_iterator
+  testComplexityGuaranteesParameterizedIter<forward_iterator, In2, Out>();
+  testComplexityGuaranteesParameterizedIter<bidirectional_iterator, In2, Out>();
+  testComplexityGuaranteesParameterizedIter<random_access_iterator, In2, Out>();
+}
+
+template <class Out>
+constexpr void testComplexityGuaranteesParameterizedIterPermutateIn1In2() {
+  testComplexityGuaranteesParameterizedIterPermutateIn1<forward_iterator, Out>();
+  testComplexityGuaranteesParameterizedIterPermutateIn1<bidirectional_iterator, Out>();
+  testComplexityGuaranteesParameterizedIterPermutateIn1<random_access_iterator, Out>();
+}
+
+constexpr bool testComplexityGuarantees() {
+  testComplexityGuaranteesParameterizedIterPermutateIn1In2<forward_iterator<int*>>();
+  testComplexityGuaranteesParameterizedIterPermutateIn1In2<bidirectional_iterator<int*>>();
+  testComplexityGuaranteesParameterizedIterPermutateIn1In2<random_access_iterator<int*>>();
+  return true;
+}
+
+constexpr bool testComplexityBasic() {
+  // Complexity: At most 2 * ((last1 - first1) + (last2 - first2)) - 1 comparisons and applications of each projection.
+  std::array<int, 5> r1{1, 3, 5, 7, 9};
+  std::array<int, 5> r2{2, 4, 6, 8, 10};
+  std::array<int, 0> expected{};
+
+  const std::size_t maxOperation = std_less_comparison_count_multiplier() * (2 * (r1.size() + r2.size()) - 1);
+
+  // std::set_intersection
+  {
+    std::array<int, 0> out{};
+    std::size_t numberOfComp = 0;
+
+    const auto comp = [&numberOfComp](int x, int y) {
+      ++numberOfComp;
+      return x < y;
+    };
+
+    std::set_intersection(r1.begin(), r1.end(), r2.begin(), r2.end(), out.data(), comp);
+
+    assert(std::ranges::equal(out, expected));
+    assert(numberOfComp <= maxOperation);
+  }
+
+  // ranges::set_intersection iterator overload
+  {
+    std::array<int, 0> out{};
+    std::size_t numberOfComp  = 0;
+    std::size_t numberOfProj1 = 0;
+    std::size_t numberOfProj2 = 0;
+
+    const auto comp = [&numberOfComp](int x, int y) {
+      ++numberOfComp;
+      return x < y;
+    };
+
+    const auto proj1 = [&numberOfProj1](int d) {
+      ++numberOfProj1;
+      return d;
+    };
+
+    const auto proj2 = [&numberOfProj2](int d) {
+      ++numberOfProj2;
+      return d;
+    };
+
+    std::ranges::set_intersection(r1.begin(), r1.end(), r2.begin(), r2.end(), out.data(), comp, proj1, proj2);
+
+    assert(std::ranges::equal(out, expected));
+    assert(numberOfComp <= maxOperation);
+    assert(numberOfProj1 <= maxOperation);
+    assert(numberOfProj2 <= maxOperation);
+  }
+
+  // ranges::set_intersection range overload
+  {
+    std::array<int, 0> out{};
+    std::size_t numberOfComp  = 0;
+    std::size_t numberOfProj1 = 0;
+    std::size_t numberOfProj2 = 0;
+
+    const auto comp = [&numberOfComp](int x, int y) {
+      ++numberOfComp;
+      return x < y;
+    };
+
+    const auto proj1 = [&numberOfProj1](int d) {
+      ++numberOfProj1;
+      return d;
+    };
+
+    const auto proj2 = [&numberOfProj2](int d) {
+      ++numberOfProj2;
+      return d;
+    };
+
+    std::ranges::set_intersection(r1, r2, out.data(), comp, proj1, proj2);
+
+    assert(std::ranges::equal(out, expected));
+    assert(numberOfComp < maxOperation);
+    assert(numberOfProj1 < maxOperation);
+    assert(numberOfProj2 < maxOperation);
+  }
+  return true;
+}
+
+} // unnamed namespace
+
+int main(int, char**) {
+  testComplexityBasic();
+  testComplexity();
+  testComplexityGuarantees();
+
+  static_assert(testComplexityBasic());
+  static_assert(testComplexity());
+
+  // we hit maximum constexpr evaluation step limit even if we split this into
+  // the 3 types of the first type layer, so let's skip the constexpr validation
+  // static_assert(testComplexityGuarantees());
+
+  return 0;
+}
diff --git a/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_count.pass.cpp b/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_count.pass.cpp
index cda49acad985b..18f8d15335ec6 100644
--- a/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_count.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_count.pass.cpp
@@ -12,6 +12,7 @@
 
 #include <iterator>
 
+#include <algorithm>
 #include <cassert>
 
 #include "test_iterators.h"
@@ -32,12 +33,16 @@ constexpr void check(int* first, std::iter_difference_t<It> n, int* expected) {
 
   // Count operations
   if constexpr (Count) {
-    auto it = stride_counting_iterator(It(first));
+    IteratorOpCounts ops;
+    auto it = operation_counting_iterator(It(first), &ops);
     std::ranges::advance(it, n);
     if constexpr (std::random_access_iterator<It>) {
-      assert(it.stride_count() <= 1);
+      assert(ops.increments + ops.decrements <= 1);
     } else {
-      assert(it.stride_count() == abs(M));
+      const auto big   = std::max(ops.increments, ops.decrements);
+      const auto small = std::min(ops.increments, ops.decrements);
+      assert(big == std::size_t(abs(M)));
+      assert(small == 0);
     }
   }
 }
diff --git a/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_count_sentinel.pass.cpp b/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_count_sentinel.pass.cpp
index 76439ef93a607..d613105805dd5 100644
--- a/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_count_sentinel.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_count_sentinel.pass.cpp
@@ -38,14 +38,16 @@ check_forward(int* first, int* last, std::iter_difference_t<It> n, int* expected
 
   // Count operations
   if constexpr (Count) {
-    auto it = stride_counting_iterator(It(first));
-    auto sent = sentinel_wrapper(stride_counting_iterator(It(last)));
+    IteratorOpCounts ops;
+    auto it   = operation_counting_iterator(It(first), &ops);
+    auto sent = sentinel_wrapper(operation_counting_iterator(It(last), &ops));
     (void)std::ranges::advance(it, n, sent);
     // We don't have a sized sentinel, so we have to increment one-by-one
     // regardless of the iterator category.
-    assert(it.stride_count() == M);
-    assert(it.stride_displacement() == M);
-    assert(it.equals_count() == expected_equals_count);
+    assert(static_cast<Difference>(ops.increments) == M);
+    assert(static_cast<Difference>(ops.decrements) == 0);
+    assert(ops.zero_moves == 0);
+    assert(ops.equal_cmps == static_cast<std::size_t>(expected_equals_count));
   }
 }
 
@@ -65,28 +67,24 @@ constexpr void check_forward_sized_sentinel(int* first, int* last, std::iter_dif
 
   // Count operations
   {
-    auto it = stride_counting_iterator(It(first));
+    IteratorOpCounts ops;
+    auto it   = operation_counting_iterator(It(first), &ops);
     auto sent = distance_apriori_sentinel(size);
     (void)std::ranges::advance(it, n, sent);
     if constexpr (std::random_access_iterator<It>) {
-      assert(it.stride_count() <= 1);
-      assert(it.stride_displacement() <= 1);
+      assert(ops.increments + ops.zero_moves == 1);
+      assert(ops.decrements == 0);
     } else {
-      assert(it.stride_count() == M);
-      assert(it.stride_displacement() == M);
+      assert(static_cast<Difference>(ops.increments) == M);
+      assert(ops.decrements == 0);
+      assert(ops.zero_moves == 0);
     }
   }
 }
 
-struct Expected {
-  int stride_count;
-  int stride_displacement;
-  int equals_count;
-};
-
 template <bool Count, typename It>
 constexpr void
-check_backward(int* first, int* last, std::iter_difference_t<It> n, int* expected, Expected expected_counts) {
+check_backward(int* first, int* last, std::iter_difference_t<It> n, int* expected, IteratorOpCounts expected_counts) {
   // Check preconditions for `advance` when called with negative `n`
   // (see [range.iter.op.advance]). In addition, allow `n == 0`.
   assert(n <= 0);
@@ -105,16 +103,18 @@ check_backward(int* first, int* last, std::iter_difference_t<It> n, int* expecte
 
   // Count operations
   {
-    auto it = stride_counting_iterator(It(last));
-    auto sent = stride_counting_iterator(It(first));
-    static_assert(std::bidirectional_iterator<stride_counting_iterator<It>>);
+    IteratorOpCounts ops;
+    auto it   = operation_counting_iterator(It(last), &ops);
+    auto sent = operation_counting_iterator(It(first), &ops);
+    static_assert(std::bidirectional_iterator<operation_counting_iterator<It>>);
     static_assert(Count == !std::sized_sentinel_for<It, It>);
 
     (void)std::ranges::advance(it, n, sent);
 
-    assert(it.stride_count() == expected_counts.stride_count);
-    assert(it.stride_displacement() == expected_counts.stride_displacement);
-    assert(it.equals_count() == expected_counts.equals_count);
+    assert(ops.increments == expected_counts.increments);
+    assert(ops.decrements == expected_counts.decrements);
+    assert(ops.zero_moves == expected_counts.zero_moves);
+    assert(ops.equal_cmps == expected_counts.equal_cmps);
   }
 }
 
@@ -217,21 +217,22 @@ constexpr bool test() {
       {
         int* expected = n > size ? range : range + size - n;
         {
-          Expected expected_counts = {
-              .stride_count        = static_cast<int>(range + size - expected),
-              .stride_displacement = -expected_counts.stride_count,
-              .equals_count        = n > size ? size + 1 : n,
+          IteratorOpCounts expected_counts = {
+              .increments = 0,
+              .decrements = static_cast<std::size_t>(range + size - expected),
+              .equal_cmps = static_cast<std::size_t>(n > size ? size + 1 : n),
           };
 
           check_backward<true, bidirectional_iterator<int*>>(range, range + size, -n, expected, expected_counts);
         }
         {
-          Expected expected_counts = {
+          IteratorOpCounts expected_counts = {
               // If `n >= size`, the algorithm can just do `it = std::move(sent);`
               // instead of doing iterator arithmetic.
-              .stride_count        = (n >= size) ? 0 : 1,
-              .stride_displacement = (n >= size) ? 0 : 1,
-              .equals_count        = 0,
+              .increments = 0,
+              .decrements = static_cast<std::size_t>((n == 0 || n >= size) ? 0 : 1),
+              .zero_moves = static_cast<std::size_t>(n == 0 && size != 0 ? 1 : 0),
+              .equal_cmps = 0,
           };
 
           check_backward<false, random_access_iterator<int*>>(range, range + size, -n, expected, expected_counts);
diff --git a/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_sentinel.pass.cpp b/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_sentinel.pass.cpp
index 2e9a28e4ad395..147c26e429063 100644
--- a/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_sentinel.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.advance/iterator_sentinel.pass.cpp
@@ -12,6 +12,7 @@
 
 #include <iterator>
 
+#include <algorithm>
 #include <cassert>
 #include <cstddef>
 
@@ -31,11 +32,12 @@ constexpr void check_assignable(int* first, int* last, int* expected) {
 
   // Count operations
   if constexpr (Count) {
-    auto it = stride_counting_iterator(It(first));
-    auto sent = assignable_sentinel(stride_counting_iterator(It(last)));
+    IteratorOpCounts ops;
+    auto it   = operation_counting_iterator(It(first), &ops);
+    auto sent = assignable_sentinel(operation_counting_iterator(It(last), &ops));
     std::ranges::advance(it, sent);
     assert(base(base(it)) == expected);
-    assert(it.stride_count() == 0); // because we got here by assigning from last, not by incrementing
+    assert(ops.increments + ops.decrements == 0); // because we got here by assigning from last, not by incrementing
   }
 }
 
@@ -53,13 +55,17 @@ constexpr void check_sized_sentinel(int* first, int* last, int* expected) {
 
   // Count operations
   if constexpr (Count) {
-    auto it = stride_counting_iterator(It(first));
+    IteratorOpCounts ops;
+    auto it   = operation_counting_iterator(It(first), &ops);
     auto sent = distance_apriori_sentinel(size);
     std::ranges::advance(it, sent);
     if constexpr (std::random_access_iterator<It>) {
-      assert(it.stride_count() == 1);
+      assert(ops.increments + ops.decrements + ops.zero_moves == 1);
     } else {
-      assert(it.stride_count() == size);
+      const auto big   = std::max(ops.increments, ops.decrements);
+      const auto small = std::min(ops.increments, ops.decrements);
+      assert(big == static_cast<size_t>(size > 0 ? size : -size));
+      assert(small == 0);
     }
   }
 }
@@ -78,10 +84,14 @@ constexpr void check_sentinel(int* first, int* last, int* expected) {
 
   // Count operations
   if constexpr (Count) {
-    auto it = stride_counting_iterator(It(first));
-    auto sent = sentinel_wrapper(stride_counting_iterator(It(last)));
+    IteratorOpCounts ops;
+    auto it   = operation_counting_iterator(It(first), &ops);
+    auto sent = sentinel_wrapper(operation_counting_iterator(It(last), &ops));
     std::ranges::advance(it, sent);
-    assert(it.stride_count() == size);
+    const auto big   = std::max(ops.increments, ops.decrements);
+    const auto small = std::min(ops.increments, ops.decrements);
+    assert(big == static_cast<size_t>(size > 0 ? size : -size));
+    assert(small == 0);
   }
 }
 
diff --git a/libcxx/test/std/ranges/range.adaptors/range.drop/begin.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.drop/begin.pass.cpp
index 28ac53c2445d4..7e01b5884728f 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.drop/begin.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.drop/begin.pass.cpp
@@ -73,12 +73,12 @@ constexpr bool test() {
   std::ranges::drop_view dropView7(MoveOnlyView(), 10);
   assert(dropView7.begin() == globalBuff + 8);
 
-  CountedView view8;
+  IteratorOpCounts opcounts;
+  CountedView view8(&opcounts);
+  ;
   std::ranges::drop_view dropView8(view8, 5);
   assert(base(base(dropView8.begin())) == globalBuff + 5);
-  assert(dropView8.begin().stride_count() == 5);
-  assert(base(base(dropView8.begin())) == globalBuff + 5);
-  assert(dropView8.begin().stride_count() == 5);
+  assert(opcounts.increments == 5);
 
   static_assert(!BeginInvocable<const ForwardView>);
 
diff --git a/libcxx/test/std/ranges/range.adaptors/range.drop/types.h b/libcxx/test/std/ranges/range.adaptors/range.drop/types.h
index ae861bce40f1e..73d1e5045ad22 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.drop/types.h
+++ b/libcxx/test/std/ranges/range.adaptors/range.drop/types.h
@@ -120,10 +120,14 @@ struct Range {
   int *end() const;
 };
 
-using CountedIter = stride_counting_iterator<forward_iterator<int*>>;
+using CountedIter = operation_counting_iterator<forward_iterator<int*>>;
 struct CountedView : std::ranges::view_base {
-  constexpr CountedIter begin() const { return CountedIter(ForwardIter(globalBuff)); }
-  constexpr CountedIter end() const { return CountedIter(ForwardIter(globalBuff + 8)); }
+  explicit constexpr CountedView(IteratorOpCounts* opcounts) noexcept : opcounts_(opcounts) {}
+  constexpr CountedIter begin() const { return CountedIter(ForwardIter(globalBuff), opcounts_); }
+  constexpr CountedIter end() const { return CountedIter(ForwardIter(globalBuff + 8), opcounts_); }
+
+private:
+  IteratorOpCounts* opcounts_;
 };
 
 struct View : std::ranges::view_base {
diff --git a/libcxx/test/std/ranges/range.adaptors/range.transform/types.h b/libcxx/test/std/ranges/range.adaptors/range.transform/types.h
index 14f85722a8c19..cc5679f229de2 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.transform/types.h
+++ b/libcxx/test/std/ranges/range.adaptors/range.transform/types.h
@@ -119,12 +119,6 @@ struct Range {
   int *end() const;
 };
 
-using CountedIter = stride_counting_iterator<forward_iterator<int*>>;
-struct CountedView : std::ranges::view_base {
-  constexpr CountedIter begin() const { return CountedIter(ForwardIter(globalBuff)); }
-  constexpr CountedIter end() const { return CountedIter(ForwardIter(globalBuff + 8)); }
-};
-
 struct TimesTwo {
   constexpr int operator()(int x) const { return x * 2; }
 };
diff --git a/libcxx/test/support/test_iterators.h b/libcxx/test/support/test_iterators.h
index bb3ba2182f557..31564a3977317 100644
--- a/libcxx/test/support/test_iterators.h
+++ b/libcxx/test/support/test_iterators.h
@@ -725,17 +725,18 @@ struct common_input_iterator {
 
 #  endif // TEST_STD_VER >= 20
 
-// Iterator adaptor that counts the number of times the iterator has had a successor/predecessor
-// operation or an equality comparison operation called. Has three recorders:
-// * `stride_count`, which records the total number of calls to an op++, op--, op+=, or op-=.
-// * `stride_displacement`, which records the displacement of the calls. This means that both
-//   op++/op+= will increase the displacement counter by 1, and op--/op-= will decrease the
-//   displacement counter by 1.
-// * `equals_count`, which records the total number of calls to an op== or op!=. If compared
-//   against a sentinel object, that sentinel object must call the `record_equality_comparison`
-//   function so that the comparison is counted correctly.
+struct IteratorOpCounts {
+  std::size_t increments = 0; ///< Number of times the iterator moved forward (++it, it++, it+=positive, it-=negative).
+  std::size_t decrements = 0; ///< Number of times the iterator moved backward (--it, it--, it-=positive, it+=negative).
+  std::size_t zero_moves = 0; ///< Number of times a call was made to move the iterator by 0 positions (it+=0, it-=0).
+  std::size_t equal_cmps = 0; ///< Total number of calls to op== or op!=. If compared against a sentinel object, that
+                              ///  sentinel object must call the `record_equality_comparison` function so that the
+                              ///  comparison is counted correctly.
+};
+
+// Iterator adaptor that records its operation counts in a IteratorOpCounts
 template <class It>
-class stride_counting_iterator {
+class operation_counting_iterator {
 public:
     using value_type = typename iter_value_or_void<It>::type;
     using difference_type = std::iter_difference_t<It>;
@@ -747,147 +748,160 @@ class stride_counting_iterator {
         std::conditional_t<std::input_iterator<It>,         std::input_iterator_tag,
         /* else */                                          std::output_iterator_tag
     >>>>>;
+    using iterator_category = iterator_concept;
 
-    stride_counting_iterator() requires std::default_initializable<It> = default;
-
-    constexpr explicit stride_counting_iterator(It const& it) : base_(base(it)) { }
+    operation_counting_iterator()
+      requires std::default_initializable<It>
+    = default;
 
-    friend constexpr It base(stride_counting_iterator const& it) { return It(it.base_); }
+    constexpr explicit operation_counting_iterator(It const& it, IteratorOpCounts* counts = nullptr)
+        : base_(base(it)), counts_(counts) {}
 
-    constexpr difference_type stride_count() const { return stride_count_; }
+    constexpr operation_counting_iterator(const operation_counting_iterator& o) { *this = o; }
+    constexpr operation_counting_iterator(operation_counting_iterator&& o) { *this = o; }
 
-    constexpr difference_type stride_displacement() const { return stride_displacement_; }
+    constexpr operation_counting_iterator& operator=(const operation_counting_iterator& o) = default;
+    constexpr operation_counting_iterator& operator=(operation_counting_iterator&& o) { return *this = o; }
 
-    constexpr difference_type equals_count() const { return equals_count_; }
+    friend constexpr It base(operation_counting_iterator const& it) { return It(it.base_); }
 
     constexpr decltype(auto) operator*() const { return *It(base_); }
 
     constexpr decltype(auto) operator[](difference_type n) const { return It(base_)[n]; }
 
-    constexpr stride_counting_iterator& operator++() {
-        It tmp(base_);
-        base_ = base(++tmp);
-        ++stride_count_;
-        ++stride_displacement_;
-        return *this;
+    constexpr operation_counting_iterator& operator++() {
+      It tmp(base_);
+      base_ = base(++tmp);
+      moved_by(1);
+      return *this;
     }
 
     constexpr void operator++(int) { ++*this; }
 
-    constexpr stride_counting_iterator operator++(int)
-        requires std::forward_iterator<It>
+    constexpr operation_counting_iterator operator++(int)
+      requires std::forward_iterator<It>
     {
-        auto temp = *this;
-        ++*this;
-        return temp;
+      auto temp = *this;
+      ++*this;
+      return temp;
     }
 
-    constexpr stride_counting_iterator& operator--()
-        requires std::bidirectional_iterator<It>
+    constexpr operation_counting_iterator& operator--()
+      requires std::bidirectional_iterator<It>
     {
-        It tmp(base_);
-        base_ = base(--tmp);
-        ++stride_count_;
-        --stride_displacement_;
-        return *this;
+      It tmp(base_);
+      base_ = base(--tmp);
+      moved_by(-1);
+      return *this;
     }
 
-    constexpr stride_counting_iterator operator--(int)
-        requires std::bidirectional_iterator<It>
+    constexpr operation_counting_iterator operator--(int)
+      requires std::bidirectional_iterator<It>
     {
-        auto temp = *this;
-        --*this;
-        return temp;
+      auto temp = *this;
+      --*this;
+      return temp;
     }
 
-    constexpr stride_counting_iterator& operator+=(difference_type const n)
-        requires std::random_access_iterator<It>
+    constexpr operation_counting_iterator& operator+=(difference_type const n)
+      requires std::random_access_iterator<It>
     {
-        It tmp(base_);
-        base_ = base(tmp += n);
-        ++stride_count_;
-        ++stride_displacement_;
-        return *this;
+      It tmp(base_);
+      base_ = base(tmp += n);
+      moved_by(n);
+      return *this;
     }
 
-    constexpr stride_counting_iterator& operator-=(difference_type const n)
-        requires std::random_access_iterator<It>
+    constexpr operation_counting_iterator& operator-=(difference_type const n)
+      requires std::random_access_iterator<It>
     {
-        It tmp(base_);
-        base_ = base(tmp -= n);
-        ++stride_count_;
-        --stride_displacement_;
-        return *this;
+      It tmp(base_);
+      base_ = base(tmp -= n);
+      moved_by(-n);
+      return *this;
     }
 
-    friend constexpr stride_counting_iterator operator+(stride_counting_iterator it, difference_type n)
-        requires std::random_access_iterator<It>
+    friend constexpr operation_counting_iterator operator+(operation_counting_iterator it, difference_type n)
+      requires std::random_access_iterator<It>
     {
-        return it += n;
+      return it += n;
     }
 
-    friend constexpr stride_counting_iterator operator+(difference_type n, stride_counting_iterator it)
-        requires std::random_access_iterator<It>
+    friend constexpr operation_counting_iterator operator+(difference_type n, operation_counting_iterator it)
+      requires std::random_access_iterator<It>
     {
-        return it += n;
+      return it += n;
     }
 
-    friend constexpr stride_counting_iterator operator-(stride_counting_iterator it, difference_type n)
-        requires std::random_access_iterator<It>
+    friend constexpr operation_counting_iterator operator-(operation_counting_iterator it, difference_type n)
+      requires std::random_access_iterator<It>
     {
-        return it -= n;
+      return it -= n;
     }
 
-    friend constexpr difference_type operator-(stride_counting_iterator const& x, stride_counting_iterator const& y)
-        requires std::sized_sentinel_for<It, It>
+    friend constexpr difference_type
+    operator-(operation_counting_iterator const& x, operation_counting_iterator const& y)
+      requires std::sized_sentinel_for<It, It>
     {
-        return base(x) - base(y);
+      return base(x) - base(y);
     }
 
-    constexpr void record_equality_comparison() const { ++equals_count_; }
+    constexpr void record_equality_comparison() const {
+      if (counts_ != nullptr)
+        ++counts_->equal_cmps;
+    }
 
-    constexpr bool operator==(stride_counting_iterator const& other) const
-        requires std::sentinel_for<It, It>
+    constexpr bool operator==(operation_counting_iterator const& other) const
+      requires std::sentinel_for<It, It>
     {
       record_equality_comparison();
       return It(base_) == It(other.base_);
     }
 
-    friend constexpr bool operator<(stride_counting_iterator const& x, stride_counting_iterator const& y)
-        requires std::random_access_iterator<It>
+    friend constexpr bool operator<(operation_counting_iterator const& x, operation_counting_iterator const& y)
+      requires std::random_access_iterator<It>
     {
-        return It(x.base_) < It(y.base_);
+      return It(x.base_) < It(y.base_);
     }
 
-    friend constexpr bool operator>(stride_counting_iterator const& x, stride_counting_iterator const& y)
-        requires std::random_access_iterator<It>
+    friend constexpr bool operator>(operation_counting_iterator const& x, operation_counting_iterator const& y)
+      requires std::random_access_iterator<It>
     {
-        return It(x.base_) > It(y.base_);
+      return It(x.base_) > It(y.base_);
     }
 
-    friend constexpr bool operator<=(stride_counting_iterator const& x, stride_counting_iterator const& y)
-        requires std::random_access_iterator<It>
+    friend constexpr bool operator<=(operation_counting_iterator const& x, operation_counting_iterator const& y)
+      requires std::random_access_iterator<It>
     {
-        return It(x.base_) <= It(y.base_);
+      return It(x.base_) <= It(y.base_);
     }
 
-    friend constexpr bool operator>=(stride_counting_iterator const& x, stride_counting_iterator const& y)
-        requires std::random_access_iterator<It>
+    friend constexpr bool operator>=(operation_counting_iterator const& x, operation_counting_iterator const& y)
+      requires std::random_access_iterator<It>
     {
-        return It(x.base_) >= It(y.base_);
+      return It(x.base_) >= It(y.base_);
     }
 
     template <class T>
     void operator,(T const &) = delete;
 
 private:
+  constexpr void moved_by(difference_type n) {
+    if (counts_ == nullptr)
+      return;
+    if (n > 0)
+      ++counts_->increments;
+    else if (n < 0)
+      ++counts_->decrements;
+    else
+      ++counts_->zero_moves;
+  }
+
     decltype(base(std::declval<It>())) base_;
-    difference_type stride_count_ = 0;
-    difference_type stride_displacement_ = 0;
-    mutable difference_type equals_count_ = 0;
+    IteratorOpCounts* counts_ = nullptr;
 };
 template <class It>
-stride_counting_iterator(It) -> stride_counting_iterator<It>;
+operation_counting_iterator(It) -> operation_counting_iterator<It>;
 
 #endif // TEST_STD_VER > 17
 

From b1fd6f0996a9d6e6ebfa0cc3df0fe499c5ccdf65 Mon Sep 17 00:00:00 2001
From: aaryanshukla <53713108+aaryanshukla@users.noreply.github.com>
Date: Thu, 18 Jul 2024 13:16:00 -0700
Subject: [PATCH 075/486] [libc] newheadergen: cmake config newhdrgen (#99543)

- revert to revert for patch
https://github.com/llvm/llvm-project/pull/98828
- revert to revert https://github.com/llvm/llvm-project/pull/99410
- revert to revert https://github.com/llvm/llvm-project/pull/99413
---
 libc/CMakeLists.txt                          |   2 +
 libc/cmake/modules/LLVMLibCHeaderRules.cmake | 101 ++++-
 libc/include/CMakeLists.txt                  | 381 +++++++++++--------
 3 files changed, 329 insertions(+), 155 deletions(-)

diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt
index 6ba54475d0fd1..6e0760724d963 100644
--- a/libc/CMakeLists.txt
+++ b/libc/CMakeLists.txt
@@ -73,6 +73,8 @@ if(LIBC_BUILD_GPU_LOADER OR (LLVM_LIBC_GPU_BUILD AND NOT LLVM_RUNTIMES_BUILD))
   add_subdirectory(utils/gpu)
 endif()
 
+option(LIBC_USE_NEW_HEADER_GEN "Generate header files using new headergen instead of the old one" OFF)
+
 set(NEED_LIBC_HDRGEN FALSE)
 if(NOT LLVM_RUNTIMES_BUILD)
   if("libc" IN_LIST LLVM_ENABLE_RUNTIMES)
diff --git a/libc/cmake/modules/LLVMLibCHeaderRules.cmake b/libc/cmake/modules/LLVMLibCHeaderRules.cmake
index 7fc6860f23eb2..91054810f5ec5 100644
--- a/libc/cmake/modules/LLVMLibCHeaderRules.cmake
+++ b/libc/cmake/modules/LLVMLibCHeaderRules.cmake
@@ -66,7 +66,106 @@ function(add_header target_name)
   )
 endfunction(add_header)
 
-# A rule for generated header file targets.
+function(add_gen_header2 target_name)
+  cmake_parse_arguments(
+    "ADD_GEN_HDR2"
+    "PUBLIC" # No optional arguments
+    "YAML_FILE;DEF_FILE;GEN_HDR" # Single value arguments
+    "DEPENDS"     # Multi value arguments
+    ${ARGN}
+  )
+  get_fq_target_name(${target_name} fq_target_name)
+  if(NOT LLVM_LIBC_FULL_BUILD)
+    add_library(${fq_target_name} INTERFACE)
+    return()
+  endif()
+  if(NOT ADD_GEN_HDR2_DEF_FILE)
+    message(FATAL_ERROR "`add_gen_hdr2` rule requires DEF_FILE to be specified.")
+  endif()
+  if(NOT ADD_GEN_HDR2_GEN_HDR)
+    message(FATAL_ERROR "`add_gen_hdr2` rule requires GEN_HDR to be specified.")
+  endif()
+  if(NOT ADD_GEN_HDR2_YAML_FILE)
+    message(FATAL_ERROR "`add_gen_hdr2` rule requires YAML_FILE to be specified.")
+  endif()
+
+  set(absolute_path ${CMAKE_CURRENT_SOURCE_DIR}/${ADD_GEN_HDR2_GEN_HDR})
+  file(RELATIVE_PATH relative_path ${LIBC_INCLUDE_SOURCE_DIR} ${absolute_path})
+  set(out_file ${LIBC_INCLUDE_DIR}/${relative_path})
+  set(yaml_file ${CMAKE_SOURCE_DIR}/${ADD_GEN_HDR2_YAML_FILE})
+  set(def_file ${CMAKE_CURRENT_SOURCE_DIR}/${ADD_GEN_HDR2_DEF_FILE})
+
+  set(fq_data_files "")
+  if(ADD_GEN_HDR2_DATA_FILES)
+    foreach(data_file IN LISTS ADD_GEN_HDR2_DATA_FILES)
+      list(APPEND fq_data_files "${CMAKE_CURRENT_SOURCE_DIR}/${data_file}")
+    endforeach(data_file)
+  endif()
+
+  set(entry_points "${TARGET_ENTRYPOINT_NAME_LIST}")
+  list(TRANSFORM entry_points PREPEND "--e=")
+
+  add_custom_command(
+    OUTPUT ${out_file}
+    COMMAND ${Python3_EXECUTABLE} ${LIBC_SOURCE_DIR}/newhdrgen/yaml_to_classes.py
+            ${yaml_file}
+            --h_def_file ${def_file}
+            ${entry_points}
+            --output_dir ${out_file}
+    DEPENDS ${yaml_file} ${def_file} ${fq_data_files}
+    COMMENT "Generating header ${ADD_GEN_HDR2_GE2N_HDR} from ${yaml_file} and ${def_file}"
+  )
+  if(LIBC_TARGET_OS_IS_GPU)
+    file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/llvm-libc-decls)
+    file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/llvm-libc-decls/gpu)
+    set(decl_out_file ${LIBC_INCLUDE_DIR}/llvm-libc-decls/${relative_path})
+    add_custom_command(
+      OUTPUT ${decl_out_file}
+      COMMAND ${Python3_EXECUTABLE} ${LIBC_SOURCE_DIR}/newhdrgen/yaml_to_classes.py
+              ${yaml_file}
+              --export-decls
+              ${entry_points}
+              --output_dir ${decl_out_file}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      DEPENDS ${yaml_file} ${fq_data_files}
+    )
+  endif()
+  
+  if(ADD_GEN_HDR2_DEPENDS)
+    get_fq_deps_list(fq_deps_list ${ADD_GEN_HDR2_DEPENDS})
+    # Dependencies of a add_header target can only be another add_gen_header target
+    # or an add_header target.
+    foreach(dep IN LISTS fq_deps_list)
+      get_target_property(header_file ${dep} HEADER_FILE_PATH)
+      if(NOT header_file)
+        message(FATAL_ERROR "Invalid dependency '${dep}' for '${fq_target_name}'.")
+      endif()
+    endforeach()
+  endif()
+  set(generated_hdr_target ${fq_target_name}.__generated_hdr__)
+  add_custom_target(
+    ${generated_hdr_target}
+    DEPENDS ${out_file} ${fq_deps_list} ${decl_out_file}
+  )
+
+  add_header_library(
+    ${target_name}
+    HDRS
+      ${out_file}
+  )
+
+  add_dependencies(${fq_target_name} ${generated_hdr_target})
+
+  set_target_properties(
+    ${fq_target_name}
+    PROPERTIES
+      HEADER_FILE_PATH ${out_file}
+      DEPS "${fq_deps_list}"
+  )
+
+
+endfunction(add_gen_header2)
+
 # Usage:
 #     add_gen_header(
 #       <target name>
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 2cf7206f3a625..df01172854871 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -17,18 +17,41 @@ add_header(
     __llvm-libc-common.h
 )
 
-add_gen_header(
+macro(add_header_macro TARGET_NAME YAML_FILE DEF_FILE GEN_HDR DEPENDS)
+  if (LIBC_USE_NEW_HEADER_GEN)
+    add_gen_header2(
+      ${TARGET_NAME}
+      YAML_FILE ${YAML_FILE}
+      DEF_FILE ${DEF_FILE}
+      GEN_HDR ${GEN_HDR}
+      ${DEPENDS}
+      ${ARGN}
+    )
+  else()
+    add_gen_header(
+      ${TARGET_NAME}
+      DEF_FILE ${DEF_FILE}
+      GEN_HDR ${GEN_HDR}
+      ${DEPENDS}
+      ${ARGN}
+    )
+  endif()
+endmacro()
+
+add_header_macro(
   ctype
-  DEF_FILE ctype.h.def
-  GEN_HDR ctype.h
+  ../libc/newhdrgen/yaml/ctype.yaml
+  ctype.h.def
+  ctype.h
   DEPENDS
     .llvm_libc_common_h
 )
 
-add_gen_header(
+add_header_macro(
   dirent
-  DEF_FILE dirent.h.def
-  GEN_HDR dirent.h
+  ../libc/newhdrgen/yaml/dirent.yaml
+  dirent.h.def
+  dirent.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.ino_t
@@ -36,10 +59,11 @@ add_gen_header(
     .llvm-libc-types.struct_dirent
 )
 
-add_gen_header(
+add_header_macro(
   fcntl
-  DEF_FILE fcntl.h.def
-  GEN_HDR fcntl.h
+  ../libc/newhdrgen/yaml/fcntl.yaml
+  fcntl.h.def
+  fcntl.h
   DEPENDS
     .llvm-libc-macros.fcntl_macros
     .llvm-libc-types.mode_t
@@ -51,28 +75,31 @@ add_gen_header(
     .llvm_libc_common_h
 )
 
-add_gen_header(
+add_header_macro(
   dlfcn
-  DEF_FILE dlfcn.h.def
-  GEN_HDR dlfcn.h
+  ../libc/newhdrgen/yaml/dlfcn.yaml
+  dlfcn.h.def
+  dlfcn.h
   DEPENDS
     .llvm-libc-macros.dlfcn_macros
     .llvm_libc_common_h
 )
 
-add_gen_header(
+add_header_macro(
   features
-  DEF_FILE features.h.def
-  GEN_HDR features.h
+  ../libc/newhdrgen/yaml/features.yaml
+  features.h.def
+  features.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.features_macros
 )
 
-add_gen_header(
+add_header_macro(
   fenv
-  DEF_FILE fenv.h.def
-  GEN_HDR fenv.h
+  ../libc/newhdrgen/yaml/fenv.yaml
+  fenv.h.def
+  fenv.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.fenv_macros
@@ -80,44 +107,49 @@ add_gen_header(
     .llvm-libc-types.fexcept_t
 )
 
-add_gen_header(
+add_header_macro(
   inttypes
-  DEF_FILE inttypes.h.def
-  GEN_HDR inttypes.h
+  ../libc/newhdrgen/yaml/inttypes.yaml
+  inttypes.h.def
+  inttypes.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.imaxdiv_t
     .llvm-libc-macros.inttypes_macros
 )
 
-add_gen_header(
+add_header_macro(
   float
-  DEF_FILE float.h.def
-  GEN_HDR float.h
+  ../libc/newhdrgen/yaml/float.yaml
+  float.h.def
+  float.h
   DEPENDS
     .llvm-libc-macros.float_macros
 )
 
-add_gen_header(
+add_header_macro(
   stdint
-  DEF_FILE stdint.h.def
-  GEN_HDR stdint.h
+  ../libc/newhdrgen/yaml/stdint.yaml
+  stdint.h.def
+  stdint.h
   DEPENDS
     .llvm-libc-macros.stdint_macros
 )
 
-add_gen_header(
+add_header_macro(
   limits
-  DEF_FILE limits.h.def
-  GEN_HDR limits.h
+  ../libc/newhdrgen/yaml/limits.yaml
+  limits.h.def
+  limits.h
   DEPENDS
     .llvm-libc-macros.limits_macros
 )
 
-add_gen_header(
+add_header_macro(
   math
-  DEF_FILE math.h.def
-  GEN_HDR math.h
+  ../libc/newhdrgen/yaml/math.yaml
+  math.h.def
+  math.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.float16_macros
@@ -128,10 +160,11 @@ add_gen_header(
     .llvm-libc-types.float128
 )
 
-add_gen_header(
+add_header_macro(
   stdfix
-  DEF_FILE stdfix.h.def
-  GEN_HDR stdfix.h
+  ../libc/newhdrgen/yaml/stdfix.yaml
+  stdfix.h.def
+  stdfix.h
   DEPENDS
     .llvm-libc-macros.stdfix_macros
 )
@@ -139,55 +172,61 @@ add_gen_header(
 # TODO: This should be conditional on POSIX networking being included.
 file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/arpa)
 
-add_gen_header(
+add_header_macro(
   arpa_inet
-  DEF_FILE arpa/inet.h.def
-  GEN_HDR arpa/inet.h
+  ../libc/newhdrgen/yaml/arpa/arpa_inet.yaml
+  arpa/inet.h.def
+  arpa/inet.h
   DEPENDS
     .llvm_libc_common_h
 )
 
-add_gen_header(
+add_header_macro(
   assert
-  DEF_FILE assert.h.def
-  GEN_HDR assert.h
+  ../libc/newhdrgen/yaml/assert.yaml
+  assert.h.def
+  assert.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.assert_macros
 )
 
-add_gen_header(
+add_header_macro(
   setjmp
-  DEF_FILE setjmp.h.def
-  GEN_HDR setjmp.h
+  ../libc/newhdrgen/yaml/setjmp.yaml
+  setjmp.h.def
+  setjmp.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.jmp_buf
 )
 
-add_gen_header(
+add_header_macro(
   string
-  DEF_FILE string.h.def
-  GEN_HDR string.h
+  ../libc/newhdrgen/yaml/string.yaml
+  string.h.def
+  string.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.null_macro
     .llvm-libc-types.size_t
 )
 
-add_gen_header(
+add_header_macro(
   strings
-  DEF_FILE strings.h.def
-  GEN_HDR strings.h
+  ../libc/newhdrgen/yaml/strings.yaml
+  strings.h.def
+  strings.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.size_t
 )
 
-add_gen_header(
+add_header_macro(
   search
-  DEF_FILE search.h.def
-  GEN_HDR search.h
+  ../libc/newhdrgen/yaml/search.yaml
+  search.h.def
+  search.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.ACTION
@@ -196,10 +235,11 @@ add_gen_header(
     .llvm-libc-types.size_t
 )
 
-add_gen_header(
+add_header_macro(
   time
-  DEF_FILE time.h.def
-  GEN_HDR time.h
+  ../libc/newhdrgen/yaml/time.yaml
+  time.h.def
+  time.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.time_macros
@@ -211,10 +251,11 @@ add_gen_header(
     .llvm-libc-types.clockid_t
 )
 
-add_gen_header(
+add_header_macro(
   threads
-  DEF_FILE threads.h.def
-  GEN_HDR threads.h
+  ../libc/newhdrgen/yaml/threads.yaml
+  threads.h.def
+  threads.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.__call_once_func_t
@@ -227,19 +268,21 @@ add_gen_header(
     .llvm-libc-types.tss_dtor_t
 )
 
-add_gen_header(
+add_header_macro(
   errno
-  DEF_FILE errno.h.def
-  GEN_HDR errno.h
+  ../libc/newhdrgen/yaml/errno.yaml
+  errno.h.def
+  errno.h
   DEPENDS
     .llvm-libc-macros.generic_error_number_macros
     .llvm-libc-macros.error_number_macros
 )
 
-add_gen_header(
+add_header_macro(
   signal
-  DEF_FILE signal.h.def
-  GEN_HDR signal.h
+  ../libc/newhdrgen/yaml/signal.yaml
+  signal.h.def
+  signal.h
   DEPENDS
     .llvm-libc-macros.signal_macros
     .llvm-libc-types.sig_atomic_t
@@ -251,28 +294,31 @@ add_gen_header(
     .llvm-libc-types.pid_t
 )
 
-add_gen_header(
+add_header_macro(
   stdbit
-  DEF_FILE stdbit.h.def
-  GEN_HDR stdbit.h
+  ../libc/newhdrgen/yaml/stdbit.yaml
+  stdbit.h.def
+  stdbit.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.stdbit_macros
 )
 
-add_gen_header(
+add_header_macro(
   stdckdint
-  DEF_FILE stdckdint.h.def
-  GEN_HDR stdckdint.h
+  ../libc/newhdrgen/yaml/stdckdint.yaml
+  stdckdint.h.def
+  stdckdint.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.stdckdint_macros
 )
 
-add_gen_header(
+add_header_macro(
   stdio
-  DEF_FILE stdio.h.def
-  GEN_HDR stdio.h
+  ../libc/newhdrgen/yaml/stdio.yaml
+  stdio.h.def
+  stdio.h
   DEPENDS
     .llvm-libc-macros.file_seek_macros
     .llvm-libc-macros.stdio_macros
@@ -284,10 +330,11 @@ add_gen_header(
     .llvm_libc_common_h
 )
 
-add_gen_header(
+add_header_macro(
   stdlib
-  DEF_FILE stdlib.h.def
-  GEN_HDR stdlib.h
+  ../libc/newhdrgen/yaml/stdlib.yaml
+  stdlib.h.def
+  stdlib.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.stdlib_macros
@@ -301,10 +348,11 @@ add_gen_header(
     .llvm-libc-types.__atexithandler_t
 )
 
-add_gen_header(
+add_header_macro(
   unistd
-  DEF_FILE unistd.h.def
-  GEN_HDR unistd.h
+  ../libc/newhdrgen/yaml/unistd.yaml
+  unistd.h.def
+  unistd.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.file_seek_macros
@@ -319,10 +367,11 @@ add_gen_header(
     .llvm-libc-types.__getoptargv_t
 )
 
-add_gen_header(
+add_header_macro(
   pthread
-  DEF_FILE pthread.h.def
-  GEN_HDR pthread.h
+  ../libc/newhdrgen/yaml/pthread.yaml
+  pthread.h.def
+  pthread.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.__atfork_callback_t
@@ -340,10 +389,11 @@ add_gen_header(
     .llvm-libc-types.pthread_t
 )
 
-add_gen_header(
+add_header_macro(
   sched
-  DEF_FILE sched.h.def
-  GEN_HDR sched.h
+  ../libc/newhdrgen/yaml/sched.yaml
+  sched.h.def
+  sched.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sched_macros
@@ -356,10 +406,11 @@ add_gen_header(
     .llvm-libc-types.struct_timespec
 )
 
-add_gen_header(
+add_header_macro(
   spawn
-  DEF_FILE spawn.h.def
-  GEN_HDR spawn.h
+  ../libc/newhdrgen/yaml/spawn.yaml
+  spawn.h.def
+  spawn.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.mode_t
@@ -373,19 +424,21 @@ add_gen_header(
 # them.
 file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/sys)
 
-add_gen_header(
+add_header_macro(
   sys_auxv
-  DEF_FILE sys/auxv.h.def
-  GEN_HDR sys/auxv.h
+  ../libc/newhdrgen/yaml/sys/sys_auxv.yaml
+  sys/auxv.h.def
+  sys/auxv.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_auxv_macros
 )
 
-add_gen_header(
+add_header_macro(
   sys_epoll
-  DEF_FILE sys/epoll.h.def
-  GEN_HDR sys/epoll.h
+  ../libc/newhdrgen/yaml/sys/sys_epoll.yaml
+  sys/epoll.h.def
+  sys/epoll.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.struct_epoll_event
@@ -394,19 +447,21 @@ add_gen_header(
     .llvm-libc-macros.sys_epoll_macros
 )
 
-add_gen_header(
+add_header_macro(
   sys_ioctl
-  DEF_FILE sys/ioctl.h.def
-  GEN_HDR sys/ioctl.h
+  ../libc/newhdrgen/yaml/sys/sys_ioctl.yaml
+  sys/ioctl.h.def
+  sys/ioctl.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_ioctl_macros
 )
 
-add_gen_header(
+add_header_macro(
   sys_mman
-  DEF_FILE sys/mman.h.def
-  GEN_HDR sys/mman.h
+  ../libc/newhdrgen/yaml/sys/sys_mman.yaml
+  sys/mman.h.def
+  sys/mman.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_mman_macros
@@ -415,10 +470,11 @@ add_gen_header(
     .llvm-libc-types.ssize_t
 )
 
-add_gen_header(
+add_header_macro(
   sys_prctl
-  DEF_FILE sys/prctl.h.def
-  GEN_HDR sys/prctl.h
+  ../libc/newhdrgen/yaml/sys/sys_prctl.yaml
+  sys/prctl.h.def
+  sys/prctl.h
   DEPENDS
     .llvm_libc_common_h
 )
@@ -431,10 +487,11 @@ add_header(
     .llvm-libc-macros.sys_queue_macros
 )
 
-add_gen_header(
+add_header_macro(
   sys_random
-  DEF_FILE sys/random.h.def
-  GEN_HDR sys/random.h
+  ../libc/newhdrgen/yaml/sys/sys_random.yaml
+  sys/random.h.def
+  sys/random.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_random_macros
@@ -442,10 +499,11 @@ add_gen_header(
     .llvm-libc-types.ssize_t
 )
 
-add_gen_header(
+add_header_macro(
   sys_resource
-  DEF_FILE sys/resource.h.def
-  GEN_HDR sys/resource.h
+  ../libc/newhdrgen/yaml/sys/sys_resource.yaml
+  sys/resource.h.def
+  sys/resource.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_resource_macros
@@ -453,10 +511,11 @@ add_gen_header(
     .llvm-libc-types.struct_rlimit
 )
 
-add_gen_header(
+add_header_macro(
   sys_stat
-  DEF_FILE sys/stat.h.def
-  GEN_HDR sys/stat.h
+  ../libc/newhdrgen/yaml/sys/sys_stat.yaml
+  sys/stat.h.def
+  sys/stat.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_stat_macros
@@ -474,10 +533,11 @@ add_gen_header(
     .llvm-libc-types.struct_stat
 )
 
-add_gen_header(
+add_header_macro(
   sys_select
-  DEF_FILE sys/select.h.def
-  GEN_HDR sys/select.h
+  ../libc/newhdrgen/yaml/sys/sys_select.yaml
+  sys/select.h.def
+  sys/select.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_select_macros
@@ -489,10 +549,11 @@ add_gen_header(
     .llvm-libc-types.struct_timeval
 )
 
-add_gen_header(
+add_header_macro(
   sys_sendfile
-  DEF_FILE sys/sendfile.h.def
-  GEN_HDR sys/sendfile.h
+  ../libc/newhdrgen/yaml/sys/sys_sendfile.yaml
+  sys/sendfile.h.def
+  sys/sendfile.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.off_t
@@ -500,10 +561,11 @@ add_gen_header(
     .llvm-libc-types.ssize_t
 )
 
-add_gen_header(
+add_header_macro(
   sys_socket
-  DEF_FILE sys/socket.h.def
-  GEN_HDR sys/socket.h
+  ../libc/newhdrgen/yaml/sys/sys_socket.yaml
+  sys/socket.h.def
+  sys/socket.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_socket_macros
@@ -513,35 +575,40 @@ add_gen_header(
     .llvm-libc-types.struct_sockaddr_un
 )
 
-add_gen_header(
+add_header_macro(
   sys_statvfs
-  DEF_FILE sys/statvfs.h.def
-  GEN_HDR sys/statvfs.h
+  ../libc/newhdrgen/yaml/sys/sys_statvfs.yaml
+  sys/statvfs.h.def
+  sys/statvfs.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.struct_statvfs
 )
 
-add_gen_header(
+add_header_macro(
   sys_syscall
-  DEF_FILE sys/syscall.h.def
-  GEN_HDR sys/syscall.h
+  ../libc/newhdrgen/yaml/sys/sys_syscall.yaml
+  sys/syscall.h.def
+  sys/syscall.h
+  DEPENDS
 )
 
-add_gen_header(
+add_header_macro(
   sys_time
-  DEF_FILE sys/time.h.def
-  GEN_HDR sys/time.h
+  ../libc/newhdrgen/yaml/sys/sys_time.yaml
+  sys/time.h.def
+  sys/time.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.struct_timeval
     .llvm-libc-macros.sys_time_macros
 )
 
-add_gen_header(
+add_header_macro(
   sys_types
-  DEF_FILE sys/types.h.def
-  GEN_HDR sys/types.h
+  ../libc/newhdrgen/yaml/sys/sys_types.yaml
+  sys/types.h.def
+  sys/types.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.blkcnt_t
@@ -567,19 +634,21 @@ add_gen_header(
     .llvm-libc-types.uid_t
 )
 
-add_gen_header(
+add_header_macro(
   sys_utsname
-  DEF_FILE sys/utsname.h.def
-  GEN_HDR sys/utsname.h
+  ../libc/newhdrgen/yaml/sys/sys_utsname.yaml
+  sys/utsname.h.def
+  sys/utsname.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.struct_utsname
 )
 
-add_gen_header(
+add_header_macro(
   sys_wait
-  DEF_FILE sys/wait.h.def
-  GEN_HDR sys/wait.h
+  ../libc/newhdrgen/yaml/sys/sys_wait.yaml
+  sys/wait.h.def
+  sys/wait.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_wait_macros
@@ -588,10 +657,11 @@ add_gen_header(
     .llvm-libc-types.siginfo_t
 )
 
-add_gen_header(
+add_header_macro(
   termios
-  DEF_FILE termios.h.def
-  GEN_HDR termios.h
+  ../libc/newhdrgen/yaml/termios.yaml
+  termios.h.def
+  termios.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.termios_macros
@@ -602,10 +672,11 @@ add_gen_header(
     .llvm-libc-types.tcflag_t
 )
 
-add_gen_header(
+add_header_macro(
   uchar
-  DEF_FILE uchar.h.def
-  GEN_HDR uchar.h
+  ../libc/newhdrgen/yaml/uchar.yaml
+  uchar.h.def
+  uchar.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-types.mbstate_t
@@ -614,10 +685,11 @@ add_gen_header(
     .llvm-libc-types.char32_t
 )
 
-add_gen_header(
+add_header_macro(
   wchar
-  DEF_FILE wchar.h.def
-  GEN_HDR wchar.h
+  ../libc/newhdrgen/yaml/wchar.yaml
+  wchar.h.def
+  wchar.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.wchar_macros
@@ -630,10 +702,11 @@ add_gen_header(
 if(LIBC_TARGET_OS_IS_GPU)
   file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/gpu)
 
-  add_gen_header(
+  add_header_macro(
     gpu_rpc
-    DEF_FILE gpu/rpc.h.def
-    GEN_HDR gpu/rpc.h
+    ../libc/newhdrgen/yaml/gpu/rpc.yaml
+    gpu/rpc.h.def
+    gpu/rpc.h
     DEPENDS
       .llvm_libc_common_h
       .llvm-libc-types.rpc_opcodes_t
@@ -703,4 +776,4 @@ if(LLVM_LIBC_FULL_BUILD)
                             -P "${CMAKE_BINARY_DIR}/cmake_install.cmake")
   # Stripping is a no-op for headers
   add_custom_target(install-libc-headers-stripped DEPENDS install-libc-headers)
-endif()
+endif()
\ No newline at end of file

From 05275b05ca58e4d015eea1503f120e6967ef1b91 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Thu, 18 Jul 2024 13:25:17 -0700
Subject: [PATCH 076/486] fixup! [clang][test] Split AArch64 target feature
 checks across multiple lines. NFC (#99365)

Looks like sed on the ppc64-aix bot does not support the '\n' literal. Let's
try using `tr` to perform that substitution. Failing that, we can revert.
---
 .../Preprocessor/aarch64-target-features.c    | 52 +++++++++----------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index d811cb36e28d8..5ac1ba21988a4 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -291,30 +291,30 @@
 // RUN: %clang -target aarch64 -mtune=CYCLONE -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MTUNE-CYCLONE %s
 // CHECK-MTUNE-CYCLONE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a"
 
-// RUN: %clang -target aarch64 -mcpu=apple-a7 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a8 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a9 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a10 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A10 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a11 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A11 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a12 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a13 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A13 %s
-// RUN: %clang -target aarch64 -mcpu=apple-s4 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
-// RUN: %clang -target aarch64 -mcpu=apple-s5 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
-// RUN: %clang -target aarch64 -mcpu=cyclone -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a34 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A34 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a35 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A35 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a53 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A53 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a57 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A57 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a72 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A72 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a73 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-CORTEX-A73 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-r82 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-CORTEX-R82 %s
-// RUN: %clang -target aarch64 -mcpu=exynos-m3 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-M3 %s
-// RUN: %clang -target aarch64 -mcpu=exynos-m4 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-M4 %s
-// RUN: %clang -target aarch64 -mcpu=exynos-m5 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-M4 %s
-// RUN: %clang -target aarch64 -mcpu=kryo -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-KRYO %s
-// RUN: %clang -target aarch64 -mcpu=thunderx2t99 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-THUNDERX2T99 %s
-// RUN: %clang -target aarch64 -mcpu=a64fx -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-A64FX %s
-// RUN: %clang -target aarch64 -mcpu=carmel -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck -check-prefix=CHECK-MCPU-CARMEL %s
+// RUN: %clang -target aarch64 -mcpu=apple-a7 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a8 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a9 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a10 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A10 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a11 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A11 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a12 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a13 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A13 %s
+// RUN: %clang -target aarch64 -mcpu=apple-s4 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
+// RUN: %clang -target aarch64 -mcpu=apple-s5 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
+// RUN: %clang -target aarch64 -mcpu=cyclone -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a34 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A34 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a35 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A35 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a53 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A53 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a57 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A57 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a72 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A72 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a73 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-CORTEX-A73 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-r82 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-CORTEX-R82 %s
+// RUN: %clang -target aarch64 -mcpu=exynos-m3 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-M3 %s
+// RUN: %clang -target aarch64 -mcpu=exynos-m4 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-M4 %s
+// RUN: %clang -target aarch64 -mcpu=exynos-m5 -### -c %s 2>&1 || sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-M4 %s
+// RUN: %clang -target aarch64 -mcpu=kryo -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-KRYO %s
+// RUN: %clang -target aarch64 -mcpu=thunderx2t99 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-THUNDERX2T99 %s
+// RUN: %clang -target aarch64 -mcpu=a64fx -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A64FX %s
+// RUN: %clang -target aarch64 -mcpu=carmel -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-CARMEL %s
 // CHECK-MCPU-APPLE-A7-LABEL: "-target-cpu" "apple-a7"
 // CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+zcm"
 // CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+zcz"
@@ -543,7 +543,7 @@
 // CHECK-MCPU-CARMEL-NEXT: "-target-abi"
 
 
-// RUN: %clang -target x86_64-apple-macosx -arch arm64 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck --check-prefix=CHECK-ARCH-ARM64 %s
+// RUN: %clang -target x86_64-apple-macosx -arch arm64 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck --check-prefix=CHECK-ARCH-ARM64 %s
 // CHECK-ARCH-ARM64-LABEL: "-target-cpu" "apple-m1"
 // CHECK-ARCH-ARM64-NEXT: "-target-feature" "+zcm"
 // CHECK-ARCH-ARM64-NEXT: "-target-feature" "+zcz"
@@ -574,7 +574,7 @@
 // CHECK-ARCH-ARM64-NEXT: "-target-feature" "+ssbs"
 // CHECK-ARCH-ARM64-NEXT: "-target-abi"
 
-// RUN: %clang -target x86_64-apple-macosx -arch arm64_32 -### -c %s 2>&1 | sed -e 's/"-/\n"-/g' | FileCheck --check-prefix=CHECK-ARCH-ARM64_32 %s
+// RUN: %clang -target x86_64-apple-macosx -arch arm64_32 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck --check-prefix=CHECK-ARCH-ARM64_32 %s
 // CHECK-ARCH-ARM64_32-LABEL: "-target-cpu" "apple-s4"
 // CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+zcm"
 // CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+zcz"

From 5c9fc3cdd7acae4ede998f2983e6107f3c0ea36a Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Thu, 18 Jul 2024 13:27:50 -0700
Subject: [PATCH 077/486] [libc] implement cached process/thread identity
 (#98989)

migrated from https://github.com/llvm/llvm-project/pull/95965 due to
corrupted git history
---
 libc/config/config.json                       | 10 +++++
 libc/config/linux/aarch64/entrypoints.txt     |  1 +
 libc/config/linux/riscv/entrypoints.txt       |  1 +
 libc/config/linux/x86_64/entrypoints.txt      |  1 +
 libc/docs/configure.rst                       |  3 ++
 libc/docs/dev/undefined_behavior.rst          | 23 +++++++++++
 libc/spec/posix.td                            | 15 +++----
 libc/src/__support/OSUtil/CMakeLists.txt      | 17 ++++++++
 .../src/__support/OSUtil/linux/CMakeLists.txt | 13 ++++++
 libc/src/__support/OSUtil/linux/pid.cpp       | 20 +++++++++
 libc/src/__support/OSUtil/pid.h               | 41 +++++++++++++++++++
 libc/src/__support/threads/CMakeLists.txt     | 27 ++++++++++++
 .../__support/threads/linux/CMakeLists.txt    |  1 +
 libc/src/__support/threads/linux/rwlock.h     |  9 ++--
 libc/src/__support/threads/linux/thread.cpp   |  2 +
 libc/src/__support/threads/thread.h           | 27 +++++++++++-
 libc/src/__support/threads/tid.h              | 34 +++++++++++++++
 libc/src/unistd/CMakeLists.txt                | 10 +++++
 libc/src/unistd/getpid.h                      |  4 +-
 libc/src/unistd/gettid.cpp                    | 17 ++++++++
 libc/src/unistd/gettid.h                      | 21 ++++++++++
 libc/src/unistd/linux/CMakeLists.txt          |  4 +-
 libc/src/unistd/linux/fork.cpp                | 32 ++++++++++-----
 libc/src/unistd/linux/getpid.cpp              | 11 +----
 libc/startup/linux/CMakeLists.txt             |  1 +
 libc/startup/linux/do_start.cpp               |  5 +++
 .../integration/src/unistd/CMakeLists.txt     |  4 ++
 .../test/integration/src/unistd/fork_test.cpp | 24 ++++++++++-
 libc/test/src/unistd/CMakeLists.txt           | 10 +++++
 libc/test/src/unistd/gettid_test.cpp          | 15 +++++++
 30 files changed, 363 insertions(+), 40 deletions(-)
 create mode 100644 libc/src/__support/OSUtil/linux/pid.cpp
 create mode 100644 libc/src/__support/OSUtil/pid.h
 create mode 100644 libc/src/__support/threads/tid.h
 create mode 100644 libc/src/unistd/gettid.cpp
 create mode 100644 libc/src/unistd/gettid.h
 create mode 100644 libc/test/src/unistd/gettid_test.cpp

diff --git a/libc/config/config.json b/libc/config/config.json
index 94bfed894c173..0fc88e2b8dbd5 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -75,6 +75,16 @@
     "LIBC_CONF_FREELIST_MALLOC_BUFFER_SIZE": {
       "value": 1073741824,
       "doc": "Default size for the constinit freelist buffer used for the freelist malloc implementation (default 1o 1GB)."
+    },
+  },
+  "unistd": {
+    "LIBC_CONF_ENABLE_TID_CACHE": {
+      "value": true,
+      "doc": "Enable caching mechanism for gettid to avoid syscall (only effective in fullbuild mode, default to true). Please refer to Undefined Behavior documentation for implications."
+    },
+    "LIBC_CONF_ENABLE_PID_CACHE": {
+      "value": true,
+      "doc": "Enable caching mechanism for getpid to avoid syscall (default to true). Please refer to Undefined Behavior documentation for implications."
     }
   },
   "math": {
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 208889ba34a59..fc2b61226c3fc 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -297,6 +297,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.geteuid
     libc.src.unistd.getpid
     libc.src.unistd.getppid
+    libc.src.unistd.gettid
     libc.src.unistd.getuid
     libc.src.unistd.isatty
     libc.src.unistd.link
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 266c94d54a9df..b2e68fdaa86e9 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -296,6 +296,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.geteuid
     libc.src.unistd.getpid
     libc.src.unistd.getppid
+    libc.src.unistd.gettid
     libc.src.unistd.getuid
     libc.src.unistd.isatty
     libc.src.unistd.link
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index cbdee084aa199..9074bd8aea1f1 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -315,6 +315,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.geteuid
     libc.src.unistd.getpid
     libc.src.unistd.getppid
+    libc.src.unistd.gettid
     libc.src.unistd.getuid
     libc.src.unistd.isatty
     libc.src.unistd.link
diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst
index dfb35f6a6611a..5c55e4ab0f181 100644
--- a/libc/docs/configure.rst
+++ b/libc/docs/configure.rst
@@ -52,3 +52,6 @@ to learn about the defaults for your platform and target.
 * **"string" options**
     - ``LIBC_CONF_MEMSET_X86_USE_SOFTWARE_PREFETCHING``: Inserts prefetch for write instructions (PREFETCHW) for memset on x86 to recover performance when hardware prefetcher is disabled.
     - ``LIBC_CONF_STRING_UNSAFE_WIDE_READ``: Read more than a byte at a time to perform byte-string operations like strlen.
+* **"unistd" options**
+    - ``LIBC_CONF_ENABLE_PID_CACHE``: Enable caching mechanism for getpid to avoid syscall (default to true). Please refer to Undefined Behavior documentation for implications.
+    - ``LIBC_CONF_ENABLE_TID_CACHE``: Enable caching mechanism for gettid to avoid syscall (only effective in fullbuild mode, default to true). Please refer to Undefined Behavior documentation for implications.
diff --git a/libc/docs/dev/undefined_behavior.rst b/libc/docs/dev/undefined_behavior.rst
index 3faae3134ce2a..b712780222aa3 100644
--- a/libc/docs/dev/undefined_behavior.rst
+++ b/libc/docs/dev/undefined_behavior.rst
@@ -93,3 +93,26 @@ direction in this case.
 Non-const Constant Return Values
 --------------------------------
 Some libc functions, like ``dlerror()``, return ``char *`` instead of ``const char *`` and then tell the caller they promise not to to modify this value. Any modification of this value is undefined behavior.
+
+Cached ``getpid/gettid``
+------------------------
+Since version ``2.25``, glibc removes its cache mechanism for ``getpid/gettid`` 
+(See the history section in https://man7.org/linux/man-pages/man2/getpid.2.html).
+LLVM's libc still implements the cache as it is useful for fast deadlock detection.
+The cache mechanism is also implemented in MUSL and bionic. The tid/pid cache can 
+be disabled by setting ``LIBC_CONF_ENABLE_TID_CACHE`` and ``LIBC_CONF_ENABLE_PID_CACHE``
+to ``false`` respectively.
+
+Unwrapped ``SYS_clone/SYS_fork/SYS_vfork``
+------------------------------------------
+It is highly discouraged to use unwrapped ``SYS_clone/SYS_fork/SYS_vfork``. 
+First, calling such syscalls without provided libc wrappers ignores 
+all the ``pthread_atfork`` entries as libc can no longer detect the ``fork``. 
+Second, libc relies on the ``fork/clone`` wrappers to correctly maintain cache for
+process id and thread id, and other important process-specific states such as the list 
+of robust mutexes. Third, even if the user is to call ``exec*`` functions immediately, 
+there can still be other unexpected issues. For instance, there can be signal handlers 
+inherited from parent process triggered inside the instruction window between ``fork`` 
+and ``exec*``. As libc failed to maintain its internal states correctly, even though the
+functions used inside the signal handlers are marked as ``async-signal-safe`` (such as
+``getpid``), they will still return wrong values or lead to other even worse situations.
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index 1878b1ee2ae41..48f743dff4e6f 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -546,6 +546,11 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<PidT>,
           [ArgSpec<VoidType>]
         >,
+        FunctionSpec<
+          "gettid",
+          RetValSpec<PidT>,
+          [ArgSpec<VoidType>]
+        >,
         FunctionSpec<
           "getuid",
           RetValSpec<UidT>,
@@ -601,16 +606,6 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<IntType>,
           [ArgSpec<ConstCharPtr>]
         >,
-        FunctionSpec<
-          "getpid",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "getppid",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidType>]
-        >,
         FunctionSpec<
           "link",
           RetValSpec<IntType>,
diff --git a/libc/src/__support/OSUtil/CMakeLists.txt b/libc/src/__support/OSUtil/CMakeLists.txt
index 94d1042ccbb4a..517f888178718 100644
--- a/libc/src/__support/OSUtil/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/CMakeLists.txt
@@ -15,3 +15,20 @@ add_object_library(
   DEPENDS
     ${target_os_util}
 )
+
+if (LIBC_CONF_ENABLE_PID_CACHE)
+  set(libc_copt_enable_pid_cache 1)
+else()
+  set(libc_copt_enable_pid_cache 0)
+endif()
+
+if(TARGET libc.src.__support.OSUtil.${LIBC_TARGET_OS}.pid)
+  add_object_library(
+    pid
+    ALIAS
+    DEPENDS
+      .${LIBC_TARGET_OS}.pid
+    COMPILE_OPTIONS
+      -DLIBC_COPT_ENABLE_PID_CACHE=${libc_copt_enable_pid_cache}
+  )
+endif()
diff --git a/libc/src/__support/OSUtil/linux/CMakeLists.txt b/libc/src/__support/OSUtil/linux/CMakeLists.txt
index 089cad454d534..95a83d77d0257 100644
--- a/libc/src/__support/OSUtil/linux/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/CMakeLists.txt
@@ -23,3 +23,16 @@ add_object_library(
     libc.hdr.types.struct_f_owner_ex
     libc.hdr.types.off_t
 )
+
+add_object_library(
+  pid
+  SRCS
+    pid.cpp
+  HDRS
+    ../pid.h
+  DEPENDS
+    libc.src.__support.OSUtil.osutil
+    libc.src.__support.common
+    libc.hdr.types.pid_t
+    libc.include.sys_syscall
+)
diff --git a/libc/src/__support/OSUtil/linux/pid.cpp b/libc/src/__support/OSUtil/linux/pid.cpp
new file mode 100644
index 0000000000000..a8499af596229
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/pid.cpp
@@ -0,0 +1,20 @@
+//===------------ pid_t utilities implementation ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/OSUtil/pid.h"
+#include "src/__support/OSUtil/syscall.h"
+#include <sys/syscall.h>
+
+namespace LIBC_NAMESPACE_DECL {
+
+pid_t ProcessIdentity::cache = -1;
+pid_t ProcessIdentity::get_uncached() {
+  return syscall_impl<pid_t>(SYS_getpid);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/OSUtil/pid.h b/libc/src/__support/OSUtil/pid.h
new file mode 100644
index 0000000000000..d723abe728569
--- /dev/null
+++ b/libc/src/__support/OSUtil/pid.h
@@ -0,0 +1,41 @@
+//===------------ pid_t utilities -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_PID_H
+#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_PID_H
+#include "hdr/types/pid_t.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/optimization.h"
+
+#ifndef LIBC_COPT_ENABLE_PID_CACHE
+#define LIBC_COPT_ENABLE_PID_CACHE 1
+#endif
+
+namespace LIBC_NAMESPACE_DECL {
+
+class ProcessIdentity {
+  static LIBC_INLINE_VAR thread_local bool fork_inflight = true;
+  static pid_t cache;
+  static pid_t get_uncached();
+
+public:
+  LIBC_INLINE static void start_fork() { fork_inflight = true; }
+  LIBC_INLINE static void end_fork() { fork_inflight = false; }
+  LIBC_INLINE static void refresh_cache() { cache = get_uncached(); }
+  LIBC_INLINE static pid_t get() {
+#if LIBC_COPT_ENABLE_PID_CACHE
+    if (LIBC_LIKELY(!fork_inflight))
+      return cache;
+#endif
+    return get_uncached();
+  }
+};
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_PID_H
diff --git a/libc/src/__support/threads/CMakeLists.txt b/libc/src/__support/threads/CMakeLists.txt
index d2e46b8e2574e..f1a2f162acfc2 100644
--- a/libc/src/__support/threads/CMakeLists.txt
+++ b/libc/src/__support/threads/CMakeLists.txt
@@ -44,6 +44,12 @@ if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.mutex)
   )
 endif()
 
+if (LIBC_CONF_ENABLE_TID_CACHE)
+  set(libc_copt_enable_tid_cache 1)
+else()
+  set(libc_copt_enable_tid_cache 0)
+endif()
+
 add_header_library(
   thread_common
   HDRS
@@ -54,6 +60,9 @@ add_header_library(
     libc.src.__support.CPP.optional
     libc.src.__support.CPP.string_view
     libc.src.__support.CPP.stringstream
+    libc.hdr.types.pid_t
+  COMPILE_OPTIONS
+    -DLIBC_COPT_ENABLE_TID_CACHE=${libc_copt_enable_tid_cache}
 )
 
 if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.thread)
@@ -89,3 +98,21 @@ if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.CndVar)
     .${LIBC_TARGET_OS}.CndVar
   )
 endif()
+
+set(tid_dep)
+if (LLVM_LIBC_FULL_BUILD)
+  list(APPEND tid_dep libc.src.__support.thread)
+else()
+  list(APPEND tid_dep libc.src.__support.OSUtil.osutil)
+  list(APPEND tid_dep libc.include.sys_syscall)
+endif()
+
+add_header_library(
+  tid
+  HDRS
+    tid.h
+  DEPENDS
+    libc.src.__support.common
+    libc.hdr.types.pid_t
+    ${tid_dep}
+)
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index 8b7971584e77e..d86441dd67cd7 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -55,6 +55,7 @@ add_header_library(
     libc.src.__support.common
     libc.src.__support.OSUtil.osutil
     libc.src.__support.CPP.limits
+    libc.src.__support.threads.tid
   COMPILE_OPTIONS
     -DLIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT=${LIBC_CONF_RWLOCK_DEFAULT_SPIN_COUNT}
     ${monotonicity_flags}
diff --git a/libc/src/__support/threads/linux/rwlock.h b/libc/src/__support/threads/linux/rwlock.h
index d2fb0ce1a3c08..cae8aa6410686 100644
--- a/libc/src/__support/threads/linux/rwlock.h
+++ b/libc/src/__support/threads/linux/rwlock.h
@@ -23,6 +23,7 @@
 #include "src/__support/threads/linux/futex_word.h"
 #include "src/__support/threads/linux/raw_mutex.h"
 #include "src/__support/threads/sleep.h"
+#include "src/__support/threads/tid.h"
 
 #ifndef LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT
 #define LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT 100
@@ -336,8 +337,6 @@ class RwLock {
   LIBC_INLINE Role get_preference() const {
     return static_cast<Role>(preference);
   }
-  // TODO: use cached thread id once implemented.
-  LIBC_INLINE static pid_t gettid() { return syscall_impl<pid_t>(SYS_gettid); }
 
   template <Role role> LIBC_INLINE LockResult try_lock(RwState &old) {
     if constexpr (role == Role::Reader) {
@@ -359,7 +358,7 @@ class RwLock {
         if (LIBC_LIKELY(old.compare_exchange_weak_with(
                 state, old.set_writer_bit(), cpp::MemoryOrder::ACQUIRE,
                 cpp::MemoryOrder::RELAXED))) {
-          writer_tid.store(gettid(), cpp::MemoryOrder::RELAXED);
+          writer_tid.store(gettid_inline(), cpp::MemoryOrder::RELAXED);
           return LockResult::Success;
         }
         // Notice that old is updated by the compare_exchange_weak_with
@@ -394,7 +393,7 @@ class RwLock {
             unsigned spin_count = LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT) {
     // Phase 1: deadlock detection.
     // A deadlock happens if this is a RAW/WAW lock in the same thread.
-    if (writer_tid.load(cpp::MemoryOrder::RELAXED) == gettid())
+    if (writer_tid.load(cpp::MemoryOrder::RELAXED) == gettid_inline())
       return LockResult::Deadlock;
 
 #if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
@@ -520,7 +519,7 @@ class RwLock {
     if (old.has_active_writer()) {
       // The lock is held by a writer.
       // Check if we are the owner of the lock.
-      if (writer_tid.load(cpp::MemoryOrder::RELAXED) != gettid())
+      if (writer_tid.load(cpp::MemoryOrder::RELAXED) != gettid_inline())
         return LockResult::PermissionDenied;
       // clear writer tid.
       writer_tid.store(0, cpp::MemoryOrder::RELAXED);
diff --git a/libc/src/__support/threads/linux/thread.cpp b/libc/src/__support/threads/linux/thread.cpp
index 36b4a88eba9b4..c8ad086f3d1cb 100644
--- a/libc/src/__support/threads/linux/thread.cpp
+++ b/libc/src/__support/threads/linux/thread.cpp
@@ -518,4 +518,6 @@ void thread_exit(ThreadReturnValue retval, ThreadStyle style) {
   __builtin_unreachable();
 }
 
+pid_t Thread::get_uncached_tid() { return syscall_impl<pid_t>(SYS_gettid); }
+
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/threads/thread.h b/libc/src/__support/threads/thread.h
index ce23a880e048a..1805b6fd6182a 100644
--- a/libc/src/__support/threads/thread.h
+++ b/libc/src/__support/threads/thread.h
@@ -9,6 +9,11 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_THREAD_H
 #define LLVM_LIBC_SRC___SUPPORT_THREADS_THREAD_H
 
+#ifndef LIBC_COPT_ENABLE_TID_CACHE
+#define LIBC_COPT_ENABLE_TID_CACHE 1
+#endif
+
+#include "hdr/types/pid_t.h"
 #include "src/__support/CPP/atomic.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/CPP/string_view.h"
@@ -103,7 +108,7 @@ struct alignas(STACK_ALIGNMENT) ThreadAttributes {
   uintptr_t tls;                // Address to the thread TLS memory
   uintptr_t tls_size;           // The size of area pointed to by |tls|.
   unsigned char owned_stack; // Indicates if the thread owns this stack memory
-  int tid;
+  pid_t tid;
   ThreadStyle style;
   ThreadReturnValue retval;
   ThreadAtExitCallbackMgr *atexit_callback_mgr;
@@ -228,6 +233,26 @@ struct Thread {
 
   // Return the name of the thread in |name|. Return the error number of error.
   int get_name(cpp::StringStream &name) const;
+
+  static pid_t get_uncached_tid();
+
+  LIBC_INLINE void refresh_tid(pid_t cached = -1) {
+    if (cached >= 0)
+      this->attrib->tid = cached;
+    else
+      this->attrib->tid = get_uncached_tid();
+  }
+  LIBC_INLINE void invalidate_tid() { this->attrib->tid = -1; }
+
+  LIBC_INLINE pid_t get_tid() {
+#if LIBC_COPT_ENABLE_TID_CACHE
+    if (LIBC_UNLIKELY(this->attrib->tid < 0))
+      return get_uncached_tid();
+    return this->attrib->tid;
+#else
+    return get_uncached_tid();
+#endif
+  }
 };
 
 extern LIBC_THREAD_LOCAL Thread self;
diff --git a/libc/src/__support/threads/tid.h b/libc/src/__support/threads/tid.h
new file mode 100644
index 0000000000000..a575cff508a0f
--- /dev/null
+++ b/libc/src/__support/threads/tid.h
@@ -0,0 +1,34 @@
+//===--- Tid wrapper --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_TID_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_TID_H
+
+// This header is for internal usage which automatically dispatches full build
+// and overlay build behaviors.
+
+#include "hdr/types/pid_t.h"
+#include "src/__support/common.h"
+#ifdef LIBC_FULL_BUILD
+#include "src/__support/threads/thread.h"
+#else
+#include "src/__support/OSUtil/syscall.h"
+#include <sys/syscall.h>
+#endif // LIBC_FULL_BUILD
+
+namespace LIBC_NAMESPACE_DECL {
+LIBC_INLINE pid_t gettid_inline() {
+#ifdef LIBC_FULL_BUILD
+  return self.get_tid();
+#else
+  return syscall_impl<pid_t>(SYS_gettid);
+#endif
+}
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_TID_H
diff --git a/libc/src/unistd/CMakeLists.txt b/libc/src/unistd/CMakeLists.txt
index ddafcd7c92f21..ec767128588f6 100644
--- a/libc/src/unistd/CMakeLists.txt
+++ b/libc/src/unistd/CMakeLists.txt
@@ -333,3 +333,13 @@ add_entrypoint_external(
 add_entrypoint_external(
   opterr
 )
+
+add_entrypoint_object(
+  gettid
+  SRCS
+    gettid.cpp
+  HDRS
+    gettid.h
+  DEPENDS
+    libc.src.__support.threads.tid
+)
diff --git a/libc/src/unistd/getpid.h b/libc/src/unistd/getpid.h
index c3c55b0c06b10..5812df0dfecd6 100644
--- a/libc/src/unistd/getpid.h
+++ b/libc/src/unistd/getpid.h
@@ -9,12 +9,12 @@
 #ifndef LLVM_LIBC_SRC_UNISTD_GETPID_H
 #define LLVM_LIBC_SRC_UNISTD_GETPID_H
 
+#include "hdr/types/pid_t.h"
 #include "src/__support/macros/config.h"
-#include <unistd.h>
 
 namespace LIBC_NAMESPACE_DECL {
 
-pid_t getpid();
+pid_t getpid(void);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/unistd/gettid.cpp b/libc/src/unistd/gettid.cpp
new file mode 100644
index 0000000000000..6d8ed65fb753d
--- /dev/null
+++ b/libc/src/unistd/gettid.cpp
@@ -0,0 +1,17 @@
+//===-- Implementation file for gettid --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/unistd/gettid.h"
+#include "src/__support/common.h"
+#include "src/__support/threads/tid.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(pid_t, gettid, (void)) { return gettid_inline(); }
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/unistd/gettid.h b/libc/src/unistd/gettid.h
new file mode 100644
index 0000000000000..42283191be49b
--- /dev/null
+++ b/libc/src/unistd/gettid.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for gettid ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_UNISTD_GETTID_H
+#define LLVM_LIBC_SRC_UNISTD_GETTID_H
+
+#include "hdr/types/pid_t.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+pid_t gettid(void);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_UNISTD_GETTID_H
diff --git a/libc/src/unistd/linux/CMakeLists.txt b/libc/src/unistd/linux/CMakeLists.txt
index 7e733d7f002c3..651ea60d07a30 100644
--- a/libc/src/unistd/linux/CMakeLists.txt
+++ b/libc/src/unistd/linux/CMakeLists.txt
@@ -101,6 +101,7 @@ add_entrypoint_object(
     libc.include.sys_syscall
     libc.src.__support.threads.fork_callbacks
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.OSUtil.pid
     libc.src.__support.threads.thread
     libc.src.errno.errno
 )
@@ -204,8 +205,7 @@ add_entrypoint_object(
     ../getpid.h
   DEPENDS
     libc.include.unistd
-    libc.include.sys_syscall
-    libc.src.__support.OSUtil.osutil
+    libc.src.__support.OSUtil.pid
 )
 
 add_entrypoint_object(
diff --git a/libc/src/unistd/linux/fork.cpp b/libc/src/unistd/linux/fork.cpp
index 7d47665b16d3f..8fe1881733f34 100644
--- a/libc/src/unistd/linux/fork.cpp
+++ b/libc/src/unistd/linux/fork.cpp
@@ -8,13 +8,14 @@
 
 #include "src/unistd/fork.h"
 
+#include "src/__support/OSUtil/pid.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/fork_callbacks.h"
 #include "src/__support/threads/thread.h" // For thread self object
-
 #include "src/errno/libc_errno.h"
+
 #include <signal.h>      // For SIGCHLD
 #include <sys/syscall.h> // For syscall numbers.
 
@@ -25,6 +26,14 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(pid_t, fork, (void)) {
   invoke_prepare_callbacks();
+
+  // Invalidate tid/pid cache before fork to avoid post fork signal handler from
+  // getting wrong values. gettid() is not async-signal-safe, but let's provide
+  // our best efforts here.
+  pid_t parent_tid = self.get_tid();
+  self.invalidate_tid();
+  ProcessIdentity::start_fork();
+
 #ifdef SYS_fork
   pid_t ret = LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_fork);
 #elif defined(SYS_clone)
@@ -32,15 +41,6 @@ LLVM_LIBC_FUNCTION(pid_t, fork, (void)) {
 #else
 #error "fork and clone syscalls not available."
 #endif
-  if (ret == 0) {
-    // Return value is 0 in the child process.
-    // The child is created with a single thread whose self object will be a
-    // copy of parent process' thread which called fork. So, we have to fix up
-    // the child process' self object with the new process' tid.
-    self.attrib->tid = LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_gettid);
-    invoke_child_callbacks();
-    return 0;
-  }
 
   if (ret < 0) {
     // Error case, a child process was not created.
@@ -48,6 +48,18 @@ LLVM_LIBC_FUNCTION(pid_t, fork, (void)) {
     return -1;
   }
 
+  // Child process
+  if (ret == 0) {
+    self.refresh_tid();
+    ProcessIdentity::refresh_cache();
+    ProcessIdentity::end_fork();
+    invoke_child_callbacks();
+    return 0;
+  }
+
+  // Parent process
+  self.refresh_tid(parent_tid);
+  ProcessIdentity::end_fork();
   invoke_parent_callbacks();
   return ret;
 }
diff --git a/libc/src/unistd/linux/getpid.cpp b/libc/src/unistd/linux/getpid.cpp
index b24c86a15990f..65d6c8a3bea95 100644
--- a/libc/src/unistd/linux/getpid.cpp
+++ b/libc/src/unistd/linux/getpid.cpp
@@ -7,17 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/unistd/getpid.h"
-
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/OSUtil/pid.h"
 #include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-
-#include <sys/syscall.h> // For syscall numbers.
-
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(pid_t, getpid, ()) {
-  return LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_getpid);
-}
+LLVM_LIBC_FUNCTION(pid_t, getpid, (void)) { return ProcessIdentity::get(); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/startup/linux/CMakeLists.txt b/libc/startup/linux/CMakeLists.txt
index 336c5d0f6bfa2..585edf20f65be 100644
--- a/libc/startup/linux/CMakeLists.txt
+++ b/libc/startup/linux/CMakeLists.txt
@@ -101,6 +101,7 @@ add_object_library(
     libc.include.llvm-libc-macros.link_macros
     libc.src.__support.threads.thread
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.OSUtil.pid
     libc.src.stdlib.exit
     libc.src.stdlib.atexit
     libc.src.unistd.environ
diff --git a/libc/startup/linux/do_start.cpp b/libc/startup/linux/do_start.cpp
index 824c0e1cf8f26..4047c06ff25c1 100644
--- a/libc/startup/linux/do_start.cpp
+++ b/libc/startup/linux/do_start.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 #include "startup/linux/do_start.h"
 #include "include/llvm-libc-macros/link-macros.h"
+#include "src/__support/OSUtil/pid.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/thread.h"
@@ -127,6 +128,10 @@ static ThreadAttributes main_thread_attrib;
   if (tls.size != 0 && !set_thread_ptr(tls.tp))
     syscall_impl<long>(SYS_exit, 1);
 
+  // Validate process identity cache (TLS needed).
+  ProcessIdentity::refresh_cache();
+  ProcessIdentity::end_fork();
+
   self.attrib = &main_thread_attrib;
   main_thread_attrib.atexit_callback_mgr =
       internal::get_thread_atexit_callback_mgr();
diff --git a/libc/test/integration/src/unistd/CMakeLists.txt b/libc/test/integration/src/unistd/CMakeLists.txt
index 3f18231209512..f50405d0925e2 100644
--- a/libc/test/integration/src/unistd/CMakeLists.txt
+++ b/libc/test/integration/src/unistd/CMakeLists.txt
@@ -31,6 +31,10 @@ add_integration_test(
     libc.src.sys.wait.wait4
     libc.src.sys.wait.waitpid
     libc.src.unistd.fork
+    libc.src.unistd.getpid
+    libc.src.unistd.gettid
+    libc.src.stdlib.exit
+    libc.include.sys_syscall
 )
 
 if((${LIBC_TARGET_OS} STREQUAL "linux") AND (${LIBC_TARGET_ARCHITECTURE_IS_X86}))
diff --git a/libc/test/integration/src/unistd/fork_test.cpp b/libc/test/integration/src/unistd/fork_test.cpp
index 9c9213ed46316..4b82d5f195627 100644
--- a/libc/test/integration/src/unistd/fork_test.cpp
+++ b/libc/test/integration/src/unistd/fork_test.cpp
@@ -6,17 +6,21 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/OSUtil/syscall.h"
 #include "src/pthread/pthread_atfork.h"
 #include "src/signal/raise.h"
+#include "src/stdlib/exit.h"
 #include "src/sys/wait/wait.h"
 #include "src/sys/wait/wait4.h"
 #include "src/sys/wait/waitpid.h"
 #include "src/unistd/fork.h"
-
+#include "src/unistd/getpid.h"
+#include "src/unistd/gettid.h"
 #include "test/IntegrationTest/test.h"
 
 #include <errno.h>
 #include <signal.h>
+#include <sys/syscall.h>
 #include <sys/wait.h>
 #include <unistd.h>
 
@@ -140,7 +144,25 @@ void fork_with_atfork_callbacks() {
   ASSERT_NE(child, DONE);
 }
 
+void fork_pid_tid_test() {
+  pid_t pid = fork();
+  ASSERT_TRUE(pid >= 0);
+  ASSERT_EQ(LIBC_NAMESPACE::gettid(),
+            LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_gettid));
+  ASSERT_EQ(LIBC_NAMESPACE::getpid(),
+            LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_getpid));
+
+  if (pid == 0) {
+    LIBC_NAMESPACE::exit(0);
+  } else {
+    int status;
+    LIBC_NAMESPACE::waitpid(pid, &status, 0);
+    ASSERT_EQ(status, 0);
+  }
+}
+
 TEST_MAIN(int argc, char **argv, char **envp) {
+  fork_pid_tid_test();
   fork_and_wait_normal_exit();
   fork_and_wait4_normal_exit();
   fork_and_waitpid_normal_exit();
diff --git a/libc/test/src/unistd/CMakeLists.txt b/libc/test/src/unistd/CMakeLists.txt
index 332455b791aee..f8292653081f1 100644
--- a/libc/test/src/unistd/CMakeLists.txt
+++ b/libc/test/src/unistd/CMakeLists.txt
@@ -378,6 +378,16 @@ add_libc_unittest(
     libc.src.unistd.getpid
 )
 
+add_libc_unittest(
+  gettid_test
+  SUITE
+    libc_unistd_unittests
+  SRCS
+    gettid_test.cpp
+  DEPENDS
+    libc.src.unistd.gettid
+)
+
 add_libc_unittest(
   getppid_test
   SUITE
diff --git a/libc/test/src/unistd/gettid_test.cpp b/libc/test/src/unistd/gettid_test.cpp
new file mode 100644
index 0000000000000..c2330f4002279
--- /dev/null
+++ b/libc/test/src/unistd/gettid_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for gettid ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/unistd/gettid.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcGetTidTest, SmokeTest) {
+  // gettid always succeeds. So, we just call it as a smoke test.
+  ASSERT_GT(LIBC_NAMESPACE::gettid(), 0);
+}

From 415ca24f8e392bca6f6295e667be2f02211fc303 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Thu, 18 Jul 2024 13:31:04 -0700
Subject: [PATCH 078/486] Revert "[libc] implement cached process/thread
 identity" (#99559)

Reverts llvm/llvm-project#98989
---
 libc/config/config.json                       | 10 -----
 libc/config/linux/aarch64/entrypoints.txt     |  1 -
 libc/config/linux/riscv/entrypoints.txt       |  1 -
 libc/config/linux/x86_64/entrypoints.txt      |  1 -
 libc/docs/configure.rst                       |  3 --
 libc/docs/dev/undefined_behavior.rst          | 23 -----------
 libc/spec/posix.td                            | 15 ++++---
 libc/src/__support/OSUtil/CMakeLists.txt      | 17 --------
 .../src/__support/OSUtil/linux/CMakeLists.txt | 13 ------
 libc/src/__support/OSUtil/linux/pid.cpp       | 20 ---------
 libc/src/__support/OSUtil/pid.h               | 41 -------------------
 libc/src/__support/threads/CMakeLists.txt     | 27 ------------
 .../__support/threads/linux/CMakeLists.txt    |  1 -
 libc/src/__support/threads/linux/rwlock.h     |  9 ++--
 libc/src/__support/threads/linux/thread.cpp   |  2 -
 libc/src/__support/threads/thread.h           | 27 +-----------
 libc/src/__support/threads/tid.h              | 34 ---------------
 libc/src/unistd/CMakeLists.txt                | 10 -----
 libc/src/unistd/getpid.h                      |  4 +-
 libc/src/unistd/gettid.cpp                    | 17 --------
 libc/src/unistd/gettid.h                      | 21 ----------
 libc/src/unistd/linux/CMakeLists.txt          |  4 +-
 libc/src/unistd/linux/fork.cpp                | 32 +++++----------
 libc/src/unistd/linux/getpid.cpp              | 11 ++++-
 libc/startup/linux/CMakeLists.txt             |  1 -
 libc/startup/linux/do_start.cpp               |  5 ---
 .../integration/src/unistd/CMakeLists.txt     |  4 --
 .../test/integration/src/unistd/fork_test.cpp | 24 +----------
 libc/test/src/unistd/CMakeLists.txt           | 10 -----
 libc/test/src/unistd/gettid_test.cpp          | 15 -------
 30 files changed, 40 insertions(+), 363 deletions(-)
 delete mode 100644 libc/src/__support/OSUtil/linux/pid.cpp
 delete mode 100644 libc/src/__support/OSUtil/pid.h
 delete mode 100644 libc/src/__support/threads/tid.h
 delete mode 100644 libc/src/unistd/gettid.cpp
 delete mode 100644 libc/src/unistd/gettid.h
 delete mode 100644 libc/test/src/unistd/gettid_test.cpp

diff --git a/libc/config/config.json b/libc/config/config.json
index 0fc88e2b8dbd5..94bfed894c173 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -75,16 +75,6 @@
     "LIBC_CONF_FREELIST_MALLOC_BUFFER_SIZE": {
       "value": 1073741824,
       "doc": "Default size for the constinit freelist buffer used for the freelist malloc implementation (default 1o 1GB)."
-    },
-  },
-  "unistd": {
-    "LIBC_CONF_ENABLE_TID_CACHE": {
-      "value": true,
-      "doc": "Enable caching mechanism for gettid to avoid syscall (only effective in fullbuild mode, default to true). Please refer to Undefined Behavior documentation for implications."
-    },
-    "LIBC_CONF_ENABLE_PID_CACHE": {
-      "value": true,
-      "doc": "Enable caching mechanism for getpid to avoid syscall (default to true). Please refer to Undefined Behavior documentation for implications."
     }
   },
   "math": {
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index fc2b61226c3fc..208889ba34a59 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -297,7 +297,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.geteuid
     libc.src.unistd.getpid
     libc.src.unistd.getppid
-    libc.src.unistd.gettid
     libc.src.unistd.getuid
     libc.src.unistd.isatty
     libc.src.unistd.link
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index b2e68fdaa86e9..266c94d54a9df 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -296,7 +296,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.geteuid
     libc.src.unistd.getpid
     libc.src.unistd.getppid
-    libc.src.unistd.gettid
     libc.src.unistd.getuid
     libc.src.unistd.isatty
     libc.src.unistd.link
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 9074bd8aea1f1..cbdee084aa199 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -315,7 +315,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.geteuid
     libc.src.unistd.getpid
     libc.src.unistd.getppid
-    libc.src.unistd.gettid
     libc.src.unistd.getuid
     libc.src.unistd.isatty
     libc.src.unistd.link
diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst
index 5c55e4ab0f181..dfb35f6a6611a 100644
--- a/libc/docs/configure.rst
+++ b/libc/docs/configure.rst
@@ -52,6 +52,3 @@ to learn about the defaults for your platform and target.
 * **"string" options**
     - ``LIBC_CONF_MEMSET_X86_USE_SOFTWARE_PREFETCHING``: Inserts prefetch for write instructions (PREFETCHW) for memset on x86 to recover performance when hardware prefetcher is disabled.
     - ``LIBC_CONF_STRING_UNSAFE_WIDE_READ``: Read more than a byte at a time to perform byte-string operations like strlen.
-* **"unistd" options**
-    - ``LIBC_CONF_ENABLE_PID_CACHE``: Enable caching mechanism for getpid to avoid syscall (default to true). Please refer to Undefined Behavior documentation for implications.
-    - ``LIBC_CONF_ENABLE_TID_CACHE``: Enable caching mechanism for gettid to avoid syscall (only effective in fullbuild mode, default to true). Please refer to Undefined Behavior documentation for implications.
diff --git a/libc/docs/dev/undefined_behavior.rst b/libc/docs/dev/undefined_behavior.rst
index b712780222aa3..3faae3134ce2a 100644
--- a/libc/docs/dev/undefined_behavior.rst
+++ b/libc/docs/dev/undefined_behavior.rst
@@ -93,26 +93,3 @@ direction in this case.
 Non-const Constant Return Values
 --------------------------------
 Some libc functions, like ``dlerror()``, return ``char *`` instead of ``const char *`` and then tell the caller they promise not to to modify this value. Any modification of this value is undefined behavior.
-
-Cached ``getpid/gettid``
-------------------------
-Since version ``2.25``, glibc removes its cache mechanism for ``getpid/gettid`` 
-(See the history section in https://man7.org/linux/man-pages/man2/getpid.2.html).
-LLVM's libc still implements the cache as it is useful for fast deadlock detection.
-The cache mechanism is also implemented in MUSL and bionic. The tid/pid cache can 
-be disabled by setting ``LIBC_CONF_ENABLE_TID_CACHE`` and ``LIBC_CONF_ENABLE_PID_CACHE``
-to ``false`` respectively.
-
-Unwrapped ``SYS_clone/SYS_fork/SYS_vfork``
-------------------------------------------
-It is highly discouraged to use unwrapped ``SYS_clone/SYS_fork/SYS_vfork``. 
-First, calling such syscalls without provided libc wrappers ignores 
-all the ``pthread_atfork`` entries as libc can no longer detect the ``fork``. 
-Second, libc relies on the ``fork/clone`` wrappers to correctly maintain cache for
-process id and thread id, and other important process-specific states such as the list 
-of robust mutexes. Third, even if the user is to call ``exec*`` functions immediately, 
-there can still be other unexpected issues. For instance, there can be signal handlers 
-inherited from parent process triggered inside the instruction window between ``fork`` 
-and ``exec*``. As libc failed to maintain its internal states correctly, even though the
-functions used inside the signal handlers are marked as ``async-signal-safe`` (such as
-``getpid``), they will still return wrong values or lead to other even worse situations.
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index 48f743dff4e6f..1878b1ee2ae41 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -546,11 +546,6 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<PidT>,
           [ArgSpec<VoidType>]
         >,
-        FunctionSpec<
-          "gettid",
-          RetValSpec<PidT>,
-          [ArgSpec<VoidType>]
-        >,
         FunctionSpec<
           "getuid",
           RetValSpec<UidT>,
@@ -606,6 +601,16 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<IntType>,
           [ArgSpec<ConstCharPtr>]
         >,
+        FunctionSpec<
+          "getpid",
+          RetValSpec<IntType>,
+          [ArgSpec<VoidType>]
+        >,
+        FunctionSpec<
+          "getppid",
+          RetValSpec<IntType>,
+          [ArgSpec<VoidType>]
+        >,
         FunctionSpec<
           "link",
           RetValSpec<IntType>,
diff --git a/libc/src/__support/OSUtil/CMakeLists.txt b/libc/src/__support/OSUtil/CMakeLists.txt
index 517f888178718..94d1042ccbb4a 100644
--- a/libc/src/__support/OSUtil/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/CMakeLists.txt
@@ -15,20 +15,3 @@ add_object_library(
   DEPENDS
     ${target_os_util}
 )
-
-if (LIBC_CONF_ENABLE_PID_CACHE)
-  set(libc_copt_enable_pid_cache 1)
-else()
-  set(libc_copt_enable_pid_cache 0)
-endif()
-
-if(TARGET libc.src.__support.OSUtil.${LIBC_TARGET_OS}.pid)
-  add_object_library(
-    pid
-    ALIAS
-    DEPENDS
-      .${LIBC_TARGET_OS}.pid
-    COMPILE_OPTIONS
-      -DLIBC_COPT_ENABLE_PID_CACHE=${libc_copt_enable_pid_cache}
-  )
-endif()
diff --git a/libc/src/__support/OSUtil/linux/CMakeLists.txt b/libc/src/__support/OSUtil/linux/CMakeLists.txt
index 95a83d77d0257..089cad454d534 100644
--- a/libc/src/__support/OSUtil/linux/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/CMakeLists.txt
@@ -23,16 +23,3 @@ add_object_library(
     libc.hdr.types.struct_f_owner_ex
     libc.hdr.types.off_t
 )
-
-add_object_library(
-  pid
-  SRCS
-    pid.cpp
-  HDRS
-    ../pid.h
-  DEPENDS
-    libc.src.__support.OSUtil.osutil
-    libc.src.__support.common
-    libc.hdr.types.pid_t
-    libc.include.sys_syscall
-)
diff --git a/libc/src/__support/OSUtil/linux/pid.cpp b/libc/src/__support/OSUtil/linux/pid.cpp
deleted file mode 100644
index a8499af596229..0000000000000
--- a/libc/src/__support/OSUtil/linux/pid.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-//===------------ pid_t utilities implementation ----------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/__support/OSUtil/pid.h"
-#include "src/__support/OSUtil/syscall.h"
-#include <sys/syscall.h>
-
-namespace LIBC_NAMESPACE_DECL {
-
-pid_t ProcessIdentity::cache = -1;
-pid_t ProcessIdentity::get_uncached() {
-  return syscall_impl<pid_t>(SYS_getpid);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/OSUtil/pid.h b/libc/src/__support/OSUtil/pid.h
deleted file mode 100644
index d723abe728569..0000000000000
--- a/libc/src/__support/OSUtil/pid.h
+++ /dev/null
@@ -1,41 +0,0 @@
-//===------------ pid_t utilities -------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_PID_H
-#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_PID_H
-#include "hdr/types/pid_t.h"
-#include "src/__support/macros/attributes.h"
-#include "src/__support/macros/optimization.h"
-
-#ifndef LIBC_COPT_ENABLE_PID_CACHE
-#define LIBC_COPT_ENABLE_PID_CACHE 1
-#endif
-
-namespace LIBC_NAMESPACE_DECL {
-
-class ProcessIdentity {
-  static LIBC_INLINE_VAR thread_local bool fork_inflight = true;
-  static pid_t cache;
-  static pid_t get_uncached();
-
-public:
-  LIBC_INLINE static void start_fork() { fork_inflight = true; }
-  LIBC_INLINE static void end_fork() { fork_inflight = false; }
-  LIBC_INLINE static void refresh_cache() { cache = get_uncached(); }
-  LIBC_INLINE static pid_t get() {
-#if LIBC_COPT_ENABLE_PID_CACHE
-    if (LIBC_LIKELY(!fork_inflight))
-      return cache;
-#endif
-    return get_uncached();
-  }
-};
-
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_PID_H
diff --git a/libc/src/__support/threads/CMakeLists.txt b/libc/src/__support/threads/CMakeLists.txt
index f1a2f162acfc2..d2e46b8e2574e 100644
--- a/libc/src/__support/threads/CMakeLists.txt
+++ b/libc/src/__support/threads/CMakeLists.txt
@@ -44,12 +44,6 @@ if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.mutex)
   )
 endif()
 
-if (LIBC_CONF_ENABLE_TID_CACHE)
-  set(libc_copt_enable_tid_cache 1)
-else()
-  set(libc_copt_enable_tid_cache 0)
-endif()
-
 add_header_library(
   thread_common
   HDRS
@@ -60,9 +54,6 @@ add_header_library(
     libc.src.__support.CPP.optional
     libc.src.__support.CPP.string_view
     libc.src.__support.CPP.stringstream
-    libc.hdr.types.pid_t
-  COMPILE_OPTIONS
-    -DLIBC_COPT_ENABLE_TID_CACHE=${libc_copt_enable_tid_cache}
 )
 
 if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.thread)
@@ -98,21 +89,3 @@ if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.CndVar)
     .${LIBC_TARGET_OS}.CndVar
   )
 endif()
-
-set(tid_dep)
-if (LLVM_LIBC_FULL_BUILD)
-  list(APPEND tid_dep libc.src.__support.thread)
-else()
-  list(APPEND tid_dep libc.src.__support.OSUtil.osutil)
-  list(APPEND tid_dep libc.include.sys_syscall)
-endif()
-
-add_header_library(
-  tid
-  HDRS
-    tid.h
-  DEPENDS
-    libc.src.__support.common
-    libc.hdr.types.pid_t
-    ${tid_dep}
-)
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index d86441dd67cd7..8b7971584e77e 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -55,7 +55,6 @@ add_header_library(
     libc.src.__support.common
     libc.src.__support.OSUtil.osutil
     libc.src.__support.CPP.limits
-    libc.src.__support.threads.tid
   COMPILE_OPTIONS
     -DLIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT=${LIBC_CONF_RWLOCK_DEFAULT_SPIN_COUNT}
     ${monotonicity_flags}
diff --git a/libc/src/__support/threads/linux/rwlock.h b/libc/src/__support/threads/linux/rwlock.h
index cae8aa6410686..d2fb0ce1a3c08 100644
--- a/libc/src/__support/threads/linux/rwlock.h
+++ b/libc/src/__support/threads/linux/rwlock.h
@@ -23,7 +23,6 @@
 #include "src/__support/threads/linux/futex_word.h"
 #include "src/__support/threads/linux/raw_mutex.h"
 #include "src/__support/threads/sleep.h"
-#include "src/__support/threads/tid.h"
 
 #ifndef LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT
 #define LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT 100
@@ -337,6 +336,8 @@ class RwLock {
   LIBC_INLINE Role get_preference() const {
     return static_cast<Role>(preference);
   }
+  // TODO: use cached thread id once implemented.
+  LIBC_INLINE static pid_t gettid() { return syscall_impl<pid_t>(SYS_gettid); }
 
   template <Role role> LIBC_INLINE LockResult try_lock(RwState &old) {
     if constexpr (role == Role::Reader) {
@@ -358,7 +359,7 @@ class RwLock {
         if (LIBC_LIKELY(old.compare_exchange_weak_with(
                 state, old.set_writer_bit(), cpp::MemoryOrder::ACQUIRE,
                 cpp::MemoryOrder::RELAXED))) {
-          writer_tid.store(gettid_inline(), cpp::MemoryOrder::RELAXED);
+          writer_tid.store(gettid(), cpp::MemoryOrder::RELAXED);
           return LockResult::Success;
         }
         // Notice that old is updated by the compare_exchange_weak_with
@@ -393,7 +394,7 @@ class RwLock {
             unsigned spin_count = LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT) {
     // Phase 1: deadlock detection.
     // A deadlock happens if this is a RAW/WAW lock in the same thread.
-    if (writer_tid.load(cpp::MemoryOrder::RELAXED) == gettid_inline())
+    if (writer_tid.load(cpp::MemoryOrder::RELAXED) == gettid())
       return LockResult::Deadlock;
 
 #if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
@@ -519,7 +520,7 @@ class RwLock {
     if (old.has_active_writer()) {
       // The lock is held by a writer.
       // Check if we are the owner of the lock.
-      if (writer_tid.load(cpp::MemoryOrder::RELAXED) != gettid_inline())
+      if (writer_tid.load(cpp::MemoryOrder::RELAXED) != gettid())
         return LockResult::PermissionDenied;
       // clear writer tid.
       writer_tid.store(0, cpp::MemoryOrder::RELAXED);
diff --git a/libc/src/__support/threads/linux/thread.cpp b/libc/src/__support/threads/linux/thread.cpp
index c8ad086f3d1cb..36b4a88eba9b4 100644
--- a/libc/src/__support/threads/linux/thread.cpp
+++ b/libc/src/__support/threads/linux/thread.cpp
@@ -518,6 +518,4 @@ void thread_exit(ThreadReturnValue retval, ThreadStyle style) {
   __builtin_unreachable();
 }
 
-pid_t Thread::get_uncached_tid() { return syscall_impl<pid_t>(SYS_gettid); }
-
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/threads/thread.h b/libc/src/__support/threads/thread.h
index 1805b6fd6182a..ce23a880e048a 100644
--- a/libc/src/__support/threads/thread.h
+++ b/libc/src/__support/threads/thread.h
@@ -9,11 +9,6 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_THREAD_H
 #define LLVM_LIBC_SRC___SUPPORT_THREADS_THREAD_H
 
-#ifndef LIBC_COPT_ENABLE_TID_CACHE
-#define LIBC_COPT_ENABLE_TID_CACHE 1
-#endif
-
-#include "hdr/types/pid_t.h"
 #include "src/__support/CPP/atomic.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/CPP/string_view.h"
@@ -108,7 +103,7 @@ struct alignas(STACK_ALIGNMENT) ThreadAttributes {
   uintptr_t tls;                // Address to the thread TLS memory
   uintptr_t tls_size;           // The size of area pointed to by |tls|.
   unsigned char owned_stack; // Indicates if the thread owns this stack memory
-  pid_t tid;
+  int tid;
   ThreadStyle style;
   ThreadReturnValue retval;
   ThreadAtExitCallbackMgr *atexit_callback_mgr;
@@ -233,26 +228,6 @@ struct Thread {
 
   // Return the name of the thread in |name|. Return the error number of error.
   int get_name(cpp::StringStream &name) const;
-
-  static pid_t get_uncached_tid();
-
-  LIBC_INLINE void refresh_tid(pid_t cached = -1) {
-    if (cached >= 0)
-      this->attrib->tid = cached;
-    else
-      this->attrib->tid = get_uncached_tid();
-  }
-  LIBC_INLINE void invalidate_tid() { this->attrib->tid = -1; }
-
-  LIBC_INLINE pid_t get_tid() {
-#if LIBC_COPT_ENABLE_TID_CACHE
-    if (LIBC_UNLIKELY(this->attrib->tid < 0))
-      return get_uncached_tid();
-    return this->attrib->tid;
-#else
-    return get_uncached_tid();
-#endif
-  }
 };
 
 extern LIBC_THREAD_LOCAL Thread self;
diff --git a/libc/src/__support/threads/tid.h b/libc/src/__support/threads/tid.h
deleted file mode 100644
index a575cff508a0f..0000000000000
--- a/libc/src/__support/threads/tid.h
+++ /dev/null
@@ -1,34 +0,0 @@
-//===--- Tid wrapper --------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_TID_H
-#define LLVM_LIBC_SRC___SUPPORT_THREADS_TID_H
-
-// This header is for internal usage which automatically dispatches full build
-// and overlay build behaviors.
-
-#include "hdr/types/pid_t.h"
-#include "src/__support/common.h"
-#ifdef LIBC_FULL_BUILD
-#include "src/__support/threads/thread.h"
-#else
-#include "src/__support/OSUtil/syscall.h"
-#include <sys/syscall.h>
-#endif // LIBC_FULL_BUILD
-
-namespace LIBC_NAMESPACE_DECL {
-LIBC_INLINE pid_t gettid_inline() {
-#ifdef LIBC_FULL_BUILD
-  return self.get_tid();
-#else
-  return syscall_impl<pid_t>(SYS_gettid);
-#endif
-}
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_TID_H
diff --git a/libc/src/unistd/CMakeLists.txt b/libc/src/unistd/CMakeLists.txt
index ec767128588f6..ddafcd7c92f21 100644
--- a/libc/src/unistd/CMakeLists.txt
+++ b/libc/src/unistd/CMakeLists.txt
@@ -333,13 +333,3 @@ add_entrypoint_external(
 add_entrypoint_external(
   opterr
 )
-
-add_entrypoint_object(
-  gettid
-  SRCS
-    gettid.cpp
-  HDRS
-    gettid.h
-  DEPENDS
-    libc.src.__support.threads.tid
-)
diff --git a/libc/src/unistd/getpid.h b/libc/src/unistd/getpid.h
index 5812df0dfecd6..c3c55b0c06b10 100644
--- a/libc/src/unistd/getpid.h
+++ b/libc/src/unistd/getpid.h
@@ -9,12 +9,12 @@
 #ifndef LLVM_LIBC_SRC_UNISTD_GETPID_H
 #define LLVM_LIBC_SRC_UNISTD_GETPID_H
 
-#include "hdr/types/pid_t.h"
 #include "src/__support/macros/config.h"
+#include <unistd.h>
 
 namespace LIBC_NAMESPACE_DECL {
 
-pid_t getpid(void);
+pid_t getpid();
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/unistd/gettid.cpp b/libc/src/unistd/gettid.cpp
deleted file mode 100644
index 6d8ed65fb753d..0000000000000
--- a/libc/src/unistd/gettid.cpp
+++ /dev/null
@@ -1,17 +0,0 @@
-//===-- Implementation file for gettid --------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/unistd/gettid.h"
-#include "src/__support/common.h"
-#include "src/__support/threads/tid.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(pid_t, gettid, (void)) { return gettid_inline(); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/unistd/gettid.h b/libc/src/unistd/gettid.h
deleted file mode 100644
index 42283191be49b..0000000000000
--- a/libc/src/unistd/gettid.h
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation header for gettid ------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC_UNISTD_GETTID_H
-#define LLVM_LIBC_SRC_UNISTD_GETTID_H
-
-#include "hdr/types/pid_t.h"
-#include "src/__support/common.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-pid_t gettid(void);
-
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC_UNISTD_GETTID_H
diff --git a/libc/src/unistd/linux/CMakeLists.txt b/libc/src/unistd/linux/CMakeLists.txt
index 651ea60d07a30..7e733d7f002c3 100644
--- a/libc/src/unistd/linux/CMakeLists.txt
+++ b/libc/src/unistd/linux/CMakeLists.txt
@@ -101,7 +101,6 @@ add_entrypoint_object(
     libc.include.sys_syscall
     libc.src.__support.threads.fork_callbacks
     libc.src.__support.OSUtil.osutil
-    libc.src.__support.OSUtil.pid
     libc.src.__support.threads.thread
     libc.src.errno.errno
 )
@@ -205,7 +204,8 @@ add_entrypoint_object(
     ../getpid.h
   DEPENDS
     libc.include.unistd
-    libc.src.__support.OSUtil.pid
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
 )
 
 add_entrypoint_object(
diff --git a/libc/src/unistd/linux/fork.cpp b/libc/src/unistd/linux/fork.cpp
index 8fe1881733f34..7d47665b16d3f 100644
--- a/libc/src/unistd/linux/fork.cpp
+++ b/libc/src/unistd/linux/fork.cpp
@@ -8,14 +8,13 @@
 
 #include "src/unistd/fork.h"
 
-#include "src/__support/OSUtil/pid.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/fork_callbacks.h"
 #include "src/__support/threads/thread.h" // For thread self object
-#include "src/errno/libc_errno.h"
 
+#include "src/errno/libc_errno.h"
 #include <signal.h>      // For SIGCHLD
 #include <sys/syscall.h> // For syscall numbers.
 
@@ -26,14 +25,6 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(pid_t, fork, (void)) {
   invoke_prepare_callbacks();
-
-  // Invalidate tid/pid cache before fork to avoid post fork signal handler from
-  // getting wrong values. gettid() is not async-signal-safe, but let's provide
-  // our best efforts here.
-  pid_t parent_tid = self.get_tid();
-  self.invalidate_tid();
-  ProcessIdentity::start_fork();
-
 #ifdef SYS_fork
   pid_t ret = LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_fork);
 #elif defined(SYS_clone)
@@ -41,6 +32,15 @@ LLVM_LIBC_FUNCTION(pid_t, fork, (void)) {
 #else
 #error "fork and clone syscalls not available."
 #endif
+  if (ret == 0) {
+    // Return value is 0 in the child process.
+    // The child is created with a single thread whose self object will be a
+    // copy of parent process' thread which called fork. So, we have to fix up
+    // the child process' self object with the new process' tid.
+    self.attrib->tid = LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_gettid);
+    invoke_child_callbacks();
+    return 0;
+  }
 
   if (ret < 0) {
     // Error case, a child process was not created.
@@ -48,18 +48,6 @@ LLVM_LIBC_FUNCTION(pid_t, fork, (void)) {
     return -1;
   }
 
-  // Child process
-  if (ret == 0) {
-    self.refresh_tid();
-    ProcessIdentity::refresh_cache();
-    ProcessIdentity::end_fork();
-    invoke_child_callbacks();
-    return 0;
-  }
-
-  // Parent process
-  self.refresh_tid(parent_tid);
-  ProcessIdentity::end_fork();
   invoke_parent_callbacks();
   return ret;
 }
diff --git a/libc/src/unistd/linux/getpid.cpp b/libc/src/unistd/linux/getpid.cpp
index 65d6c8a3bea95..b24c86a15990f 100644
--- a/libc/src/unistd/linux/getpid.cpp
+++ b/libc/src/unistd/linux/getpid.cpp
@@ -7,10 +7,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/unistd/getpid.h"
-#include "src/__support/OSUtil/pid.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+#include <sys/syscall.h> // For syscall numbers.
+
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(pid_t, getpid, (void)) { return ProcessIdentity::get(); }
+LLVM_LIBC_FUNCTION(pid_t, getpid, ()) {
+  return LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_getpid);
+}
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/startup/linux/CMakeLists.txt b/libc/startup/linux/CMakeLists.txt
index 585edf20f65be..336c5d0f6bfa2 100644
--- a/libc/startup/linux/CMakeLists.txt
+++ b/libc/startup/linux/CMakeLists.txt
@@ -101,7 +101,6 @@ add_object_library(
     libc.include.llvm-libc-macros.link_macros
     libc.src.__support.threads.thread
     libc.src.__support.OSUtil.osutil
-    libc.src.__support.OSUtil.pid
     libc.src.stdlib.exit
     libc.src.stdlib.atexit
     libc.src.unistd.environ
diff --git a/libc/startup/linux/do_start.cpp b/libc/startup/linux/do_start.cpp
index 4047c06ff25c1..824c0e1cf8f26 100644
--- a/libc/startup/linux/do_start.cpp
+++ b/libc/startup/linux/do_start.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 #include "startup/linux/do_start.h"
 #include "include/llvm-libc-macros/link-macros.h"
-#include "src/__support/OSUtil/pid.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/thread.h"
@@ -128,10 +127,6 @@ static ThreadAttributes main_thread_attrib;
   if (tls.size != 0 && !set_thread_ptr(tls.tp))
     syscall_impl<long>(SYS_exit, 1);
 
-  // Validate process identity cache (TLS needed).
-  ProcessIdentity::refresh_cache();
-  ProcessIdentity::end_fork();
-
   self.attrib = &main_thread_attrib;
   main_thread_attrib.atexit_callback_mgr =
       internal::get_thread_atexit_callback_mgr();
diff --git a/libc/test/integration/src/unistd/CMakeLists.txt b/libc/test/integration/src/unistd/CMakeLists.txt
index f50405d0925e2..3f18231209512 100644
--- a/libc/test/integration/src/unistd/CMakeLists.txt
+++ b/libc/test/integration/src/unistd/CMakeLists.txt
@@ -31,10 +31,6 @@ add_integration_test(
     libc.src.sys.wait.wait4
     libc.src.sys.wait.waitpid
     libc.src.unistd.fork
-    libc.src.unistd.getpid
-    libc.src.unistd.gettid
-    libc.src.stdlib.exit
-    libc.include.sys_syscall
 )
 
 if((${LIBC_TARGET_OS} STREQUAL "linux") AND (${LIBC_TARGET_ARCHITECTURE_IS_X86}))
diff --git a/libc/test/integration/src/unistd/fork_test.cpp b/libc/test/integration/src/unistd/fork_test.cpp
index 4b82d5f195627..9c9213ed46316 100644
--- a/libc/test/integration/src/unistd/fork_test.cpp
+++ b/libc/test/integration/src/unistd/fork_test.cpp
@@ -6,21 +6,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/__support/OSUtil/syscall.h"
 #include "src/pthread/pthread_atfork.h"
 #include "src/signal/raise.h"
-#include "src/stdlib/exit.h"
 #include "src/sys/wait/wait.h"
 #include "src/sys/wait/wait4.h"
 #include "src/sys/wait/waitpid.h"
 #include "src/unistd/fork.h"
-#include "src/unistd/getpid.h"
-#include "src/unistd/gettid.h"
+
 #include "test/IntegrationTest/test.h"
 
 #include <errno.h>
 #include <signal.h>
-#include <sys/syscall.h>
 #include <sys/wait.h>
 #include <unistd.h>
 
@@ -144,25 +140,7 @@ void fork_with_atfork_callbacks() {
   ASSERT_NE(child, DONE);
 }
 
-void fork_pid_tid_test() {
-  pid_t pid = fork();
-  ASSERT_TRUE(pid >= 0);
-  ASSERT_EQ(LIBC_NAMESPACE::gettid(),
-            LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_gettid));
-  ASSERT_EQ(LIBC_NAMESPACE::getpid(),
-            LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_getpid));
-
-  if (pid == 0) {
-    LIBC_NAMESPACE::exit(0);
-  } else {
-    int status;
-    LIBC_NAMESPACE::waitpid(pid, &status, 0);
-    ASSERT_EQ(status, 0);
-  }
-}
-
 TEST_MAIN(int argc, char **argv, char **envp) {
-  fork_pid_tid_test();
   fork_and_wait_normal_exit();
   fork_and_wait4_normal_exit();
   fork_and_waitpid_normal_exit();
diff --git a/libc/test/src/unistd/CMakeLists.txt b/libc/test/src/unistd/CMakeLists.txt
index f8292653081f1..332455b791aee 100644
--- a/libc/test/src/unistd/CMakeLists.txt
+++ b/libc/test/src/unistd/CMakeLists.txt
@@ -378,16 +378,6 @@ add_libc_unittest(
     libc.src.unistd.getpid
 )
 
-add_libc_unittest(
-  gettid_test
-  SUITE
-    libc_unistd_unittests
-  SRCS
-    gettid_test.cpp
-  DEPENDS
-    libc.src.unistd.gettid
-)
-
 add_libc_unittest(
   getppid_test
   SUITE
diff --git a/libc/test/src/unistd/gettid_test.cpp b/libc/test/src/unistd/gettid_test.cpp
deleted file mode 100644
index c2330f4002279..0000000000000
--- a/libc/test/src/unistd/gettid_test.cpp
+++ /dev/null
@@ -1,15 +0,0 @@
-//===-- Unittests for gettid ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/unistd/gettid.h"
-#include "test/UnitTest/Test.h"
-
-TEST(LlvmLibcGetTidTest, SmokeTest) {
-  // gettid always succeeds. So, we just call it as a smoke test.
-  ASSERT_GT(LIBC_NAMESPACE::gettid(), 0);
-}

From af5352fe8e66e71ae381498745c0e52ca56478c6 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Thu, 18 Jul 2024 16:34:59 -0400
Subject: [PATCH 079/486] [Clang][AMDGPU] Use unsigned data type for
 `__builtin_amdgcn_raw_buffer_store_*` (#99546)

---
 clang/include/clang/Basic/BuiltinsAMDGPU.def  | 12 ++---
 .../builtins-amdgcn-raw-buffer-store.cl       | 48 +++++++++----------
 .../builtins-amdgcn-raw-buffer-store-error.cl | 26 +++++-----
 3 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index 774cbaa74f8af..ab29ef38f7792 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -149,12 +149,12 @@ BUILTIN(__builtin_amdgcn_mqsad_pk_u16_u8, "WUiWUiUiWUi", "nc")
 BUILTIN(__builtin_amdgcn_mqsad_u32_u8, "V4UiWUiUiV4Ui", "nc")
 
 BUILTIN(__builtin_amdgcn_make_buffer_rsrc, "Qbv*sii", "nc")
-BUILTIN(__builtin_amdgcn_raw_buffer_store_b8, "vcQbiiIi", "n")
-BUILTIN(__builtin_amdgcn_raw_buffer_store_b16, "vsQbiiIi", "n")
-BUILTIN(__builtin_amdgcn_raw_buffer_store_b32, "viQbiiIi", "n")
-BUILTIN(__builtin_amdgcn_raw_buffer_store_b64, "vV2iQbiiIi", "n")
-BUILTIN(__builtin_amdgcn_raw_buffer_store_b96, "vV3iQbiiIi", "n")
-BUILTIN(__builtin_amdgcn_raw_buffer_store_b128, "vV4iQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_store_b8, "vUcQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_store_b16, "vUsQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_store_b32, "vUiQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_store_b64, "vV2UiQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_store_b96, "vV3UiQbiiIi", "n")
+BUILTIN(__builtin_amdgcn_raw_buffer_store_b128, "vV4UiQbiiIi", "n")
 BUILTIN(__builtin_amdgcn_raw_buffer_load_b8, "UcQbiiIi", "n")
 BUILTIN(__builtin_amdgcn_raw_buffer_load_b16, "UsQbiiIi", "n")
 BUILTIN(__builtin_amdgcn_raw_buffer_load_b32, "UiQbiiIi", "n")
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl
index 37975d59730c5..097c545917270 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl
@@ -2,19 +2,19 @@
 // REQUIRES: amdgpu-registered-target
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu verde -emit-llvm -o - %s | FileCheck %s
 
-typedef char i8;
-typedef short i16;
-typedef int i32;
-typedef int i64 __attribute__((ext_vector_type(2)));
-typedef int i96 __attribute__((ext_vector_type(3)));
-typedef int i128 __attribute__((ext_vector_type(4)));
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned int v2u32 __attribute__((ext_vector_type(2)));
+typedef unsigned int v3u32 __attribute__((ext_vector_type(3)));
+typedef unsigned int v4u32 __attribute__((ext_vector_type(4)));
 
 // CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b8(i8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b8(u8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b8(vdata, rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -23,7 +23,7 @@ void test_amdgcn_raw_ptr_buffer_store_b8(i8 vdata, __amdgpu_buffer_rsrc_t rsrc,
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b16(i16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b16(u16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b16(vdata, rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -32,7 +32,7 @@ void test_amdgcn_raw_ptr_buffer_store_b16(i16 vdata, __amdgpu_buffer_rsrc_t rsrc
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b32(i32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b32(u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b32(vdata, rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -41,7 +41,7 @@ void test_amdgcn_raw_ptr_buffer_store_b32(i32 vdata, __amdgpu_buffer_rsrc_t rsrc
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b64(i64 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b64(v2u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b64(vdata, rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -50,7 +50,7 @@ void test_amdgcn_raw_ptr_buffer_store_b64(i64 vdata, __amdgpu_buffer_rsrc_t rsrc
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b96(i96 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b96(v3u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b96(vdata, rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -59,7 +59,7 @@ void test_amdgcn_raw_ptr_buffer_store_b96(i96 vdata, __amdgpu_buffer_rsrc_t rsrc
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b128(i128 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b128(v4u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b128(vdata, rsrc, /*offset=*/0, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -68,7 +68,7 @@ void test_amdgcn_raw_ptr_buffer_store_b128(i128 vdata, __amdgpu_buffer_rsrc_t rs
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b8_non_const_offset(i8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b8_non_const_offset(u8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b8(vdata, rsrc, offset, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -77,7 +77,7 @@ void test_amdgcn_raw_ptr_buffer_store_b8_non_const_offset(i8 vdata, __amdgpu_buf
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b16_non_const_offset(i16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b16_non_const_offset(u16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b16(vdata, rsrc, offset, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -86,7 +86,7 @@ void test_amdgcn_raw_ptr_buffer_store_b16_non_const_offset(i16 vdata, __amdgpu_b
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b32_non_const_offset(i32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b32_non_const_offset(u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b32(vdata, rsrc, offset, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -95,7 +95,7 @@ void test_amdgcn_raw_ptr_buffer_store_b32_non_const_offset(i32 vdata, __amdgpu_b
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b64_non_const_offset(i64 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b64_non_const_offset(v2u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b64(vdata, rsrc, offset, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -104,7 +104,7 @@ void test_amdgcn_raw_ptr_buffer_store_b64_non_const_offset(i64 vdata, __amdgpu_b
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b96_non_const_offset(i96 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b96_non_const_offset(v3u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b96(vdata, rsrc, offset, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -113,7 +113,7 @@ void test_amdgcn_raw_ptr_buffer_store_b96_non_const_offset(i96 vdata, __amdgpu_b
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b128_non_const_offset(i128 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b128_non_const_offset(v4u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b128(vdata, rsrc, offset, /*soffset=*/0, /*aux=*/0);
 }
 
@@ -122,7 +122,7 @@ void test_amdgcn_raw_ptr_buffer_store_b128_non_const_offset(i128 vdata, __amdgpu
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b8_non_const_soffset(i8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b8_non_const_soffset(u8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b8(vdata, rsrc, /*offset=*/0, soffset, /*aux=*/0);
 }
 
@@ -131,7 +131,7 @@ void test_amdgcn_raw_ptr_buffer_store_b8_non_const_soffset(i8 vdata, __amdgpu_bu
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b16_non_const_soffset(i16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b16_non_const_soffset(u16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b16(vdata, rsrc, /*offset=*/0, soffset, /*aux=*/0);
 }
 
@@ -140,7 +140,7 @@ void test_amdgcn_raw_ptr_buffer_store_b16_non_const_soffset(i16 vdata, __amdgpu_
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b32_non_const_soffset(i32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b32_non_const_soffset(u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b32(vdata, rsrc, /*offset=*/0, soffset, /*aux=*/0);
 }
 
@@ -149,7 +149,7 @@ void test_amdgcn_raw_ptr_buffer_store_b32_non_const_soffset(i32 vdata, __amdgpu_
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b64_non_const_soffset(i64 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b64_non_const_soffset(v2u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b64(vdata, rsrc, /*offset=*/0, soffset, /*aux=*/0);
 }
 
@@ -158,7 +158,7 @@ void test_amdgcn_raw_ptr_buffer_store_b64_non_const_soffset(i64 vdata, __amdgpu_
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b96_non_const_soffset(i96 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b96_non_const_soffset(v3u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b96(vdata, rsrc, /*offset=*/0, soffset, /*aux=*/0);
 }
 
@@ -167,6 +167,6 @@ void test_amdgcn_raw_ptr_buffer_store_b96_non_const_soffset(i96 vdata, __amdgpu_
 // CHECK-NEXT:    tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-void test_amdgcn_raw_ptr_buffer_store_b128_non_const_soffset(i128 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
+void test_amdgcn_raw_ptr_buffer_store_b128_non_const_soffset(v4u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
   __builtin_amdgcn_raw_buffer_store_b128(vdata, rsrc, /*offset=*/0, soffset, /*aux=*/0);
 }
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-raw-buffer-store-error.cl b/clang/test/SemaOpenCL/builtins-amdgcn-raw-buffer-store-error.cl
index 356c031640483..119adcd6b4e63 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-raw-buffer-store-error.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-raw-buffer-store-error.cl
@@ -3,33 +3,33 @@
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
-typedef char i8;
-typedef short i16;
-typedef int i32;
-typedef int i64 __attribute__((ext_vector_type(2)));
-typedef int i96 __attribute__((ext_vector_type(3)));
-typedef int i128 __attribute__((ext_vector_type(4)));
-
-void test_amdgcn_raw_ptr_buffer_store_b8(i8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned int v2u32 __attribute__((ext_vector_type(2)));
+typedef unsigned int v3u32 __attribute__((ext_vector_type(3)));
+typedef unsigned int v4u32 __attribute__((ext_vector_type(4)));
+
+void test_amdgcn_raw_ptr_buffer_store_b8(u8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
   __builtin_amdgcn_raw_buffer_store_b8(vdata, rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_store_b8' must be a constant integer}}
 }
 
-void test_amdgcn_raw_ptr_buffer_store_b16(i16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+void test_amdgcn_raw_ptr_buffer_store_b16(u16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
   __builtin_amdgcn_raw_buffer_store_b16(vdata, rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_store_b16' must be a constant integer}}
 }
 
-void test_amdgcn_raw_ptr_buffer_store_b32(i32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+void test_amdgcn_raw_ptr_buffer_store_b32(u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
   __builtin_amdgcn_raw_buffer_store_b32(vdata, rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_store_b32' must be a constant integer}}
 }
 
-void test_amdgcn_raw_ptr_buffer_store_b64(i64 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+void test_amdgcn_raw_ptr_buffer_store_b64(v2u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
   __builtin_amdgcn_raw_buffer_store_b64(vdata, rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_store_b64' must be a constant integer}}
 }
 
-void test_amdgcn_raw_ptr_buffer_store_b96(i96 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+void test_amdgcn_raw_ptr_buffer_store_b96(v3u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
   __builtin_amdgcn_raw_buffer_store_b96(vdata, rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_store_b96' must be a constant integer}}
 }
 
-void test_amdgcn_raw_ptr_buffer_store_b128(i128 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
+void test_amdgcn_raw_ptr_buffer_store_b128(v4u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset, int aux) {
   __builtin_amdgcn_raw_buffer_store_b128(vdata, rsrc, /*offset=*/0, /*soffset=*/0, aux); //expected-error{{argument to '__builtin_amdgcn_raw_buffer_store_b128' must be a constant integer}}
 }

From 4272847546225bcf2e83e09e3a620bf286c058fd Mon Sep 17 00:00:00 2001
From: Michael Spencer <bigcheesegs@gmail.com>
Date: Thu, 18 Jul 2024 13:37:01 -0700
Subject: [PATCH 080/486] [clang][deps] Don't treat ObjC method args as module
 directives (#97654)

`import:(type)name` is a method argument decl in ObjC, but the C++20
preprocessing rules say this is a preprocessing line.

Because the dependency directive scanner is not language dependent, this
patch extends the C++20 rule to exclude `module :` (which is never a
valid module decl anyway), and `import :` that is not followed by an
identifier.

This is ok to do because in C++20 mode the compiler will later error on
lines like this anyway, and the dependencies the scanner returns are
still correct.
---
 clang/lib/Lex/DependencyDirectivesScanner.cpp   | 13 ++++++++++++-
 .../Lex/DependencyDirectivesScannerTest.cpp     | 17 +++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Lex/DependencyDirectivesScanner.cpp b/clang/lib/Lex/DependencyDirectivesScanner.cpp
index 57652be8244b4..8a020d0e95fe3 100644
--- a/clang/lib/Lex/DependencyDirectivesScanner.cpp
+++ b/clang/lib/Lex/DependencyDirectivesScanner.cpp
@@ -660,7 +660,18 @@ bool Scanner::lexModule(const char *&First, const char *const End) {
   // an import.
 
   switch (*First) {
-  case ':':
+  case ':': {
+    // `module :` is never the start of a valid module declaration.
+    if (Id == "module") {
+      skipLine(First, End);
+      return false;
+    }
+    // `import:(type)name` is a valid ObjC method decl, so check one more token.
+    (void)lexToken(First, End);
+    if (!tryLexIdentifierOrSkipLine(First, End))
+      return false;
+    break;
+  }
   case '<':
   case '"':
     break;
diff --git a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
index 94af9688a96e2..23304fff950eb 100644
--- a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
+++ b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
@@ -970,6 +970,23 @@ ort \
   EXPECT_EQ(Directives[1].Kind, cxx_export_module_decl);
 }
 
+TEST(MinimizeSourceToDependencyDirectivesTest, ObjCMethodArgs) {
+  SmallVector<char, 128> Out;
+
+  StringRef Source = R"(
+    @interface SomeObjcClass
+      - (void)func:(int)otherData
+              module:(int)module
+              import:(int)import;
+    @end
+  )";
+
+  ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out));
+  // `module :` and `import :` not followed by an identifier are not treated as
+  // directive lines because they can be method argument decls.
+  EXPECT_STREQ("<TokBeforeEOF>\n", Out.data());
+}
+
 TEST(MinimizeSourceToDependencyDirectivesTest, TokensBeforeEOF) {
   SmallString<128> Out;
 

From 06518cea3905556d8d1eea4088132ebb234bfdab Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Thu, 18 Jul 2024 13:47:15 -0700
Subject: [PATCH 081/486] Revert "[clang][test] Split AArch64 target feature
 checks across multiple lines. NFC (#99365)"

This reverts commit c0c4ad5d9a6e05e0b1f5f98ce2e08d479b281be8.

And the fixup broke another bot. Darn.

https://lab.llvm.org/buildbot/#/builders/46/builds/1896
---
 .../Preprocessor/aarch64-target-features.c    | 405 +++---------------
 1 file changed, 48 insertions(+), 357 deletions(-)

diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index 5ac1ba21988a4..85424c6862795 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -291,309 +291,54 @@
 // RUN: %clang -target aarch64 -mtune=CYCLONE -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MTUNE-CYCLONE %s
 // CHECK-MTUNE-CYCLONE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a"
 
-// RUN: %clang -target aarch64 -mcpu=apple-a7 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a8 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a9 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a10 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A10 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a11 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A11 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a12 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
-// RUN: %clang -target aarch64 -mcpu=apple-a13 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A13 %s
-// RUN: %clang -target aarch64 -mcpu=apple-s4 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
-// RUN: %clang -target aarch64 -mcpu=apple-s5 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
-// RUN: %clang -target aarch64 -mcpu=cyclone -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a34 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A34 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a35 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A35 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a53 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A53 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a57 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A57 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a72 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A72 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-a73 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-CORTEX-A73 %s
-// RUN: %clang -target aarch64 -mcpu=cortex-r82 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-CORTEX-R82 %s
-// RUN: %clang -target aarch64 -mcpu=exynos-m3 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-M3 %s
-// RUN: %clang -target aarch64 -mcpu=exynos-m4 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-M4 %s
-// RUN: %clang -target aarch64 -mcpu=exynos-m5 -### -c %s 2>&1 || sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-M4 %s
-// RUN: %clang -target aarch64 -mcpu=kryo -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-KRYO %s
-// RUN: %clang -target aarch64 -mcpu=thunderx2t99 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-THUNDERX2T99 %s
-// RUN: %clang -target aarch64 -mcpu=a64fx -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-A64FX %s
-// RUN: %clang -target aarch64 -mcpu=carmel -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck -check-prefix=CHECK-MCPU-CARMEL %s
-// CHECK-MCPU-APPLE-A7-LABEL: "-target-cpu" "apple-a7"
-// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+zcm"
-// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+zcz"
-// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+v8a"
-// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-APPLE-A7-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A7-NEXT: "-target-abi"
-// CHECK-MCPU-APPLE-A10-LABEL: "-target-cpu" "apple-a10"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+zcm"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+zcz"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+v8a"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+lor"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+pan"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+rdm"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-feature" "+vh"
-// CHECK-MCPU-APPLE-A10-NEXT: "-target-abi"
-// CHECK-MCPU-APPLE-A11-LABEL: "-target-cpu" "apple-a11"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+zcm"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+zcz"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+v8.2a"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+fullfp16"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+lse"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+ras"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+rdm"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A11-NEXT: "-target-abi"
-// CHECK-MCPU-APPLE-A12-LABEL: "-target-cpu" "apple-a12"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+zcm"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+zcz"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+v8.3a"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+complxnum"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+fullfp16"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+jsconv"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+lse"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+pauth"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+ras"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+rcpc"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+rdm"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A12-NEXT: "-target-abi"
-// CHECK-MCPU-A34-LABEL: "-target-cpu" "cortex-a34"
-// CHECK-MCPU-A34-NEXT: "-target-feature" "+v8a"
-// CHECK-MCPU-A34-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-A34-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-A34-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-A34-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-A34-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-A34-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-A34-NEXT: "-target-abi"
-// CHECK-MCPU-APPLE-A13-LABEL: "-target-cpu" "apple-a13"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+zcm"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+zcz"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+v8.4a"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+complxnum"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+dotprod"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+fp16fml"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+fullfp16"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+jsconv"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+lse"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+pauth"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+ras"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+rcpc"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+rdm"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-feature" "+sha3"
-// CHECK-MCPU-APPLE-A13-NEXT: "-target-abi"
-// CHECK-MCPU-A35-LABEL: "-target-cpu" "cortex-a35"
-// CHECK-MCPU-A35-NEXT: "-target-feature" "+v8a"
-// CHECK-MCPU-A35-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-A35-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-A35-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-A35-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-A35-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-A35-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-A35-NEXT: "-target-abi"
-// CHECK-MCPU-A53-LABEL: "-target-cpu" "cortex-a53"
-// CHECK-MCPU-A53-NEXT: "-target-feature" "+v8a"
-// CHECK-MCPU-A53-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-A53-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-A53-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-A53-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-A53-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-A53-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-A53-NEXT: "-target-abi"
-// CHECK-MCPU-A57-LABEL: "-target-cpu" "cortex-a57"
-// CHECK-MCPU-A57-NEXT: "-target-feature" "+v8a"
-// CHECK-MCPU-A57-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-A57-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-A57-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-A57-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-A57-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-A57-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-A57-NEXT: "-target-abi"
-// CHECK-MCPU-A72-LABEL: "-target-cpu" "cortex-a72"
-// CHECK-MCPU-A72-NEXT: "-target-feature" "+v8a"
-// CHECK-MCPU-A72-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-A72-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-A72-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-A72-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-A72-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-A72-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-A72-NEXT: "-target-abi"
-// CHECK-MCPU-CORTEX-A73-LABEL: "-target-cpu" "cortex-a73"
-// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+v8a"
-// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-CORTEX-A73-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-CORTEX-A73-NEXT: "-target-abi"
-// CHECK-MCPU-CORTEX-R82-LABEL: "-target-cpu" "cortex-r82"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+v8r"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+ccdp"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+complxnum"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+dotprod"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+flagm"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+fp16fml"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+fullfp16"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+jsconv"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+lse"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+pauth"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+predres"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+ras"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+rcpc"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+rdm"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+sb"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-feature" "+ssbs"
-// CHECK-MCPU-CORTEX-R82-NEXT: "-target-abi"
-// CHECK-MCPU-M3-LABEL: "-target-cpu" "exynos-m3"
-// CHECK-MCPU-M3-NEXT: "-target-feature" "+v8a"
-// CHECK-MCPU-M3-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-M3-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-M3-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-M3-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-M3-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-M3-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-M3-NEXT: "-target-abi"
-// CHECK-MCPU-M4-LABEL: "-target-cpu" "exynos-m{{[45]}}"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+v8.2a"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+dotprod"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+fullfp16"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+lse"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+ras"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+rdm"
-// CHECK-MCPU-M4-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-M4-NEXT: "-target-abi"
-// CHECK-MCPU-KRYO-LABEL: "-target-cpu" "kryo"
-// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+v8a"
-// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-KRYO-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-KRYO-NEXT: "-target-abi"
-// CHECK-MCPU-THUNDERX2T99-LABEL: "-target-cpu" "thunderx2t99"
-// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+v8.1a"
-// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+lse"
-// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+rdm"
-// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-feature" "+sha2
-// CHECK-MCPU-THUNDERX2T99-NEXT: "-target-abi"
-// CHECK-MCPU-A64FX-LABEL: "-target-cpu" "a64fx"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+v8.2a"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+complxnum"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+fullfp16"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+lse"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+perfmon"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+ras"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+rdm"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-A64FX-NEXT: "-target-feature" "+sve"
-// CHECK-MCPU-A64FX-NEXT: "-target-abi"
-// CHECK-MCPU-CARMEL-LABEL: "-target-cpu" "carmel"
-// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+v8.2a"
-// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+aes"
-// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+crc"
-// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+fullfp16"
-// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+lse"
-// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+neon"
-// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+ras"
-// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+rdm"
-// CHECK-MCPU-CARMEL-NEXT: "-target-feature" "+sha2"
-// CHECK-MCPU-CARMEL-NEXT: "-target-abi"
-
-
-// RUN: %clang -target x86_64-apple-macosx -arch arm64 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck --check-prefix=CHECK-ARCH-ARM64 %s
-// CHECK-ARCH-ARM64-LABEL: "-target-cpu" "apple-m1"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+zcm"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+zcz"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+v8.4a"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+aes"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+altnzcv"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+ccdp"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+complxnum"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+crc"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+dotprod"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+fp16fml"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+fptoint"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+fullfp16"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+jsconv"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+lse"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+neon"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+pauth"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+perfmon"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+predres"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+ras"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+rcpc"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+rdm"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+sb"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+sha2"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+sha3"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+specrestrict"
-// CHECK-ARCH-ARM64-NEXT: "-target-feature" "+ssbs"
-// CHECK-ARCH-ARM64-NEXT: "-target-abi"
-
-// RUN: %clang -target x86_64-apple-macosx -arch arm64_32 -### -c %s 2>&1 | sed -e 's/"-/~"-/g' | tr '~' '\n' | FileCheck --check-prefix=CHECK-ARCH-ARM64_32 %s
-// CHECK-ARCH-ARM64_32-LABEL: "-target-cpu" "apple-s4"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+zcm"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+zcz"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+v8.3a"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+aes"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+complxnum"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+crc"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+fp-armv8"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+fullfp16"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+jsconv"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+lse"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+neon"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+pauth"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+perfmon"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+ras"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+rcpc"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+rdm"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-feature" "+sha2"
-// CHECK-ARCH-ARM64_32-NEXT: "-target-abi"
+// RUN: %clang -target aarch64 -mcpu=apple-a7 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a9 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a10 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A10 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a11 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A11 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a12 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
+// RUN: %clang -target aarch64 -mcpu=apple-a13 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A13 %s
+// RUN: %clang -target aarch64 -mcpu=apple-s4 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
+// RUN: %clang -target aarch64 -mcpu=apple-s5 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A12 %s
+// RUN: %clang -target aarch64 -mcpu=cyclone -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a34 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A34 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a35 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A35 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a53 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A53 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a57 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A57 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a72 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A72 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-a73 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-CORTEX-A73 %s
+// RUN: %clang -target aarch64 -mcpu=cortex-r82 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-CORTEX-R82 %s
+// RUN: %clang -target aarch64 -mcpu=exynos-m3 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-M3 %s
+// RUN: %clang -target aarch64 -mcpu=exynos-m4 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-M4 %s
+// RUN: %clang -target aarch64 -mcpu=exynos-m5 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-M4 %s
+// RUN: %clang -target aarch64 -mcpu=kryo -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-KRYO %s
+// RUN: %clang -target aarch64 -mcpu=thunderx2t99 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-THUNDERX2T99 %s
+// RUN: %clang -target aarch64 -mcpu=a64fx -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A64FX %s
+// RUN: %clang -target aarch64 -mcpu=carmel -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-CARMEL %s
+// CHECK-MCPU-APPLE-A7: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A10: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+lor" "-target-feature" "+neon" "-target-feature" "+pan" "-target-feature" "+perfmon" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+vh"
+// CHECK-MCPU-APPLE-A11: "-cc1"{{.*}} "-triple" "aarch64{{.*}}"{{.*}}"-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A12: "-cc1"{{.*}} "-triple" "aarch64"{{.*}} "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
+// CHECK-MCPU-A34: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A13: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+sha3"
+// CHECK-MCPU-A35: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-A53: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-A57: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-A72: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-CORTEX-A73: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-CORTEX-R82: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8r" "-target-feature" "+ccdp" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+flagm" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+predres" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sb" "-target-feature" "+ssbs"
+// CHECK-MCPU-M3: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-M4: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
+// CHECK-MCPU-KRYO: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-THUNDERX2T99: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.1a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+rdm" "-target-feature" "+sha2
+// CHECK-MCPU-A64FX: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+sve"
+// CHECK-MCPU-CARMEL: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
+
+// RUN: %clang -target x86_64-apple-macosx -arch arm64 -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH-ARM64 %s
+// CHECK-ARCH-ARM64: "-target-cpu" "apple-m1" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+altnzcv" "-target-feature" "+ccdp" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fptoint" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+predres" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sb" "-target-feature" "+sha2" "-target-feature" "+sha3" "-target-feature" "+specrestrict" "-target-feature" "+ssbs"
+
+// RUN: %clang -target x86_64-apple-macosx -arch arm64_32 -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH-ARM64_32 %s
+// CHECK-ARCH-ARM64_32: "-target-cpu" "apple-s4" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
 
 // RUN: %clang -target aarch64 -march=armv8-a+fp+simd+crc+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MARCH-1 %s
 // RUN: %clang -target aarch64 -march=armv8-a+nofp+nosimd+nocrc+nocrypto+fp+simd+crc+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MARCH-1 %s
@@ -791,62 +536,8 @@
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=standard -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=bti -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret+bti -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI %s
-// CHECK-BTI-OFF-NOT: __ARM_FEATURE_BTI_DEFAULT
-// CHECK-BTI:         #define __ARM_FEATURE_BTI_DEFAULT 1
-
-// ================== Check Armv8.5-A random number generation extension.
-// RUN: %clang -target aarch64-none-elf -march=armv8.5-a+rng -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-RNG %s
-// RUN: %clang -target aarch64-none-elf -march=armv8.5-a -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-NO-RNG %s
-// CHECK-RNG: __ARM_FEATURE_RNG 1
-// CHECK-NO-RNG-NOT: __ARM_FEATURE_RNG 1
-
-// ================== Check BFloat16 Extensions.
-// RUN: %clang -target aarch64-none-elf -march=armv8.6-a+bf16 -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-BFLOAT %s
-// CHECK-BFLOAT: __ARM_BF16_FORMAT_ALTERNATIVE 1
-// CHECK-BFLOAT: __ARM_FEATURE_BF16 1
-// CHECK-BFLOAT: __ARM_FEATURE_BF16_VECTOR_ARITHMETIC 1
-
-// ================== Check Armv8.7-A LS64 extension.
-// RUN: %clang -target aarch64-none-elf -march=armv8.7-a+ls64 -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-LS64 %s
-// RUN: %clang -target aarch64-none-elf -march=armv8.7-a      -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-NO-LS64 %s
-// CHECK-LS64: __ARM_FEATURE_LS64 1
-// CHECK-NO-LS64-NOT: __ARM_FEATURE_LS64 1
-
-// ================== Check sve-vector-bits flag.
-// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=128  -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-SVE-VECTOR-BITS -D#VBITS=128  %s
-// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=256  -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-SVE-VECTOR-BITS -D#VBITS=256  %s
-// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=512  -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-SVE-VECTOR-BITS -D#VBITS=512  %s
-// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=1024 -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-SVE-VECTOR-BITS -D#VBITS=1024 %s
-// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=2048 -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-SVE-VECTOR-BITS -D#VBITS=2048 %s
-// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=512+ -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-NO-SVE-VECTOR-BITS %s
-// CHECK-SVE-VECTOR-BITS: __ARM_FEATURE_SVE_BITS [[#VBITS:]]
-// CHECK-NO-SVE-VECTOR-BITS-NOT: __ARM_FEATURE_SVE_BITS
-
-// ================== Check Large System Extensions (LSE)
-// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+lse -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-LSE %s
-// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a+lse -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-LSE %s
-// RUN: %clang -target aarch64-none-linux-gnu -march=armv8.1-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-LSE %s
-// RUN: %clang -target arm64-none-linux-gnu -march=armv8.1-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-LSE %s
-// CHECK-LSE: __ARM_FEATURE_ATOMICS 1
-
-// ================== Check Armv8.8-A/Armv9.3-A memcpy and memset acceleration instructions (MOPS)
-// RUN: %clang -target aarch64-none-elf -march=armv8.7-a             -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS %s
-// RUN: %clang -target aarch64-none-elf -march=armv8.7-a+mops        -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
-// RUN: %clang -target aarch64-none-elf -march=armv8.8-a             -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
-// RUN: %clang -target aarch64-none-elf -march=armv8.8-a+nomops      -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS %s
-// RUN: %clang -target aarch64-none-elf -march=armv8.8-a+nomops+mops -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
-// RUN: %clang -target aarch64-none-elf -march=armv8.8-a+mops        -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
-// RUN: %clang -target aarch64-none-elf -march=armv8.8-a+mops+nomops -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS %s
-// RUN: %clang -target aarch64-none-elf -march=armv9.2-a             -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS %s
-// RUN: %clang -target aarch64-none-elf -march=armv9.2-a+mops        -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
-// RUN: %clang -target aarch64-none-elf -march=armv9.3-a             -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
-// RUN: %clang -target aarch64-none-elf -march=armv9.3-a+nomops      -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS %s
-// RUN: %clang -target aarch64-none-elf -march=armv9.3-a+mops        -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
-// Check that -target-feature -v9.3a doesn't enable dependant features
-// RUN: %clang -target aarch64-none-elf -Xclang -target-feature -Xclang -v9.3a  -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS   %s
 // CHECK-MOPS: __ARM_FEATURE_MOPS 1
 // CHECK-NOMOPS-NOT: __ARM_FEATURE_MOPS 1
-
 // ================== Check Armv8.9-A/Armv9.4-A 128-bit System Registers (FEAT_SYSREG128)
 // RUN: %clang -target aarch64-none-elf -march=armv8.9-a      -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOSYS128 %s
 // RUN: %clang -target aarch64-none-elf -march=armv9.4-a      -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOSYS128 %s

From 719b2ac42e42807c9282052718cbc91e0cac930c Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Thu, 18 Jul 2024 15:49:07 -0500
Subject: [PATCH 082/486] [libc++] Allow testing Apple's system library as it
 is installed (#99086)

In order to test libc++ under the "Apple System Library" configuration,
we need to run the tests using DYLD_LIBRARY_PATH. This is required
because libc++ gets an install_name of /usr/lib when built as a system
library, which means that we must override the copy of libc++ used by
the whole process. This effectively reverts 2cf2f1b, which was the wrong
solution for the problem I was having.

Of course, this assumes that the just-built libc++ is sufficient to
replace the system library, which is not actually the case
out-of-the-box. Indeed, the system library contains a few symbols that
are not provided by the upstream library, leading to undefined symbols
when replacing the system library by the just-built one.

To solve this problem, we separately build shims that provide those
missing symbols and we manually link against them when we build
executables in the tests. While this is somewhat brittle, it provides a
localized and unintrusive way to allow testing the Apple system
configuration in an upstream environment, which has been a frequent
request.
---
 libcxx/cmake/caches/Apple.cmake               |  3 +
 .../test/configs/apple-libc++-shared.cfg.in   | 51 +++++++++++++++
 libcxx/utils/ci/apple-install-libcxx.sh       | 13 ++--
 libcxxabi/src/vendor/apple/shims.cpp          | 65 +++++++++++++++++++
 .../configs/apple-libc++abi-shared.cfg.in     | 49 ++++++++++++++
 5 files changed, 177 insertions(+), 4 deletions(-)
 create mode 100644 libcxx/test/configs/apple-libc++-shared.cfg.in
 create mode 100644 libcxxabi/src/vendor/apple/shims.cpp
 create mode 100644 libcxxabi/test/configs/apple-libc++abi-shared.cfg.in

diff --git a/libcxx/cmake/caches/Apple.cmake b/libcxx/cmake/caches/Apple.cmake
index 1038f1aed54e3..8768653e620ad 100644
--- a/libcxx/cmake/caches/Apple.cmake
+++ b/libcxx/cmake/caches/Apple.cmake
@@ -15,3 +15,6 @@ set(LIBCXXABI_HERMETIC_STATIC_LIBRARY ON CACHE BOOL "")
 set(LIBCXXABI_ENABLE_ASSERTIONS OFF CACHE BOOL "")
 set(LIBCXXABI_ENABLE_FORGIVING_DYNAMIC_CAST ON CACHE BOOL "")
 set(LIBCXXABI_USE_LLVM_UNWINDER OFF CACHE BOOL "") # libunwind is built separately
+
+set(LIBCXX_TEST_CONFIG "apple-libc++-shared.cfg.in" CACHE STRING "")
+set(LIBCXXABI_TEST_CONFIG "apple-libc++abi-shared.cfg.in" CACHE STRING "")
diff --git a/libcxx/test/configs/apple-libc++-shared.cfg.in b/libcxx/test/configs/apple-libc++-shared.cfg.in
new file mode 100644
index 0000000000000..5504bfd4e7f21
--- /dev/null
+++ b/libcxx/test/configs/apple-libc++-shared.cfg.in
@@ -0,0 +1,51 @@
+# Testing configuration for Apple's system libc++.
+#
+# This configuration differs from a normal LLVM shared library configuration in
+# that we must use DYLD_LIBRARY_PATH to run the tests against the just-built library,
+# since Apple's libc++ has an absolute install_name.
+#
+# We also don't use a per-target include directory layout, so we have only one
+# include directory for the libc++ headers.
+#
+# Finally, we also link against an artificial shims library that provides
+# the functionality necessary for the upstream libc++ to be usable in place
+# of a system-provided libc++. Without that, attempting to replace the system
+# libc++ with DYLD_LIBRARY_PATH would result in missing symbols and other similar
+# issues since the upstream libc++ does not contain all the symbols provided by
+# the system library.
+
+lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
+
+import os, site
+site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
+import libcxx.test.params, libcxx.test.config, libcxx.test.dsl
+ADDITIONAL_PARAMETERS = [
+    libcxx.test.dsl.Parameter(name='apple_system_shims', type=str,
+        actions=lambda path: [libcxx.test.dsl.AddSubstitution('%{apple-system-shims}', path)],
+        help="""
+        Path to a pre-built object file or static archive that contains shims necessary to
+        allow replacing the system libc++ by the just-built one.
+        """),
+]
+
+config.substitutions.append(('%{flags}',
+    '-isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else ''
+))
+config.substitutions.append(('%{compile_flags}',
+    '-nostdinc++ -I %{include-dir} -I %{libcxx-dir}/test/support'
+))
+config.substitutions.append(('%{link_flags}',
+    '-nostdlib++ -L %{lib-dir} -lc++ %{apple-system-shims}'
+))
+config.substitutions.append(('%{exec}',
+    '%{executor} --execdir %T --env DYLD_LIBRARY_PATH=%{lib-dir} -- '
+))
+
+config.stdlib = 'apple-libc++'
+
+libcxx.test.config.configure(
+    libcxx.test.params.DEFAULT_PARAMETERS + ADDITIONAL_PARAMETERS,
+    libcxx.test.features.DEFAULT_FEATURES,
+    config,
+    lit_config
+)
diff --git a/libcxx/utils/ci/apple-install-libcxx.sh b/libcxx/utils/ci/apple-install-libcxx.sh
index ddefabe77264f..e5698ce93bc12 100755
--- a/libcxx/utils/ci/apple-install-libcxx.sh
+++ b/libcxx/utils/ci/apple-install-libcxx.sh
@@ -114,8 +114,13 @@ for arch in ${architectures}; do
     #       Then LLVM would guess the LLVM_DEFAULT_TARGET_TRIPLE properly and we wouldn't have to specify it.
     target=$(xcrun clang -arch ${arch} -xc - -### 2>&1 | grep --only-matching -E '"-triple" ".+?"' | grep --only-matching -E '"[^ ]+-apple-[^ ]+?"' | tr -d '"')
 
-    step "Building libc++.dylib and libc++abi.dylib for architecture ${arch}"
     mkdir -p "${build_dir}/${arch}"
+
+    step "Building shims to make libc++ compatible with the system libc++ on Apple platforms when running the tests"
+    shims_library="${build_dir}/${arch}/apple-system-shims.a"
+    xcrun clang++ -c -std=c++2b -target ${target} "${llvm_root}/libcxxabi/src/vendor/apple/shims.cpp" -static -o "${shims_library}"
+
+    step "Building libc++.dylib and libc++abi.dylib for architecture ${arch}"
     xcrun cmake -S "${llvm_root}/runtimes" \
                 -B "${build_dir}/${arch}" \
                 -GNinja \
@@ -127,9 +132,9 @@ for arch in ${architectures}; do
                 -DCMAKE_OSX_ARCHITECTURES="${arch}" \
                 -DLIBCXXABI_LIBRARY_VERSION="${version}" \
                 -DLIBCXX_LIBRARY_VERSION="${version}" \
-                -DLIBCXX_TEST_PARAMS="target_triple=${target};stdlib=apple-libc++" \
-                -DLIBCXXABI_TEST_PARAMS="target_triple=${target};stdlib=apple-libc++" \
-                -DLIBUNWIND_TEST_PARAMS="target_triple=${target};stdlib=apple-libc++"
+                -DLIBCXX_TEST_PARAMS="target_triple=${target};apple_system_shims=${shims_library}" \
+                -DLIBCXXABI_TEST_PARAMS="target_triple=${target};apple_system_shims=${shims_library}" \
+                -DLIBUNWIND_TEST_PARAMS="target_triple=${target};apple_system_shims=${shims_library}"
 
     if [ "$headers_only" = true ]; then
         xcrun cmake --build "${build_dir}/${arch}" --target install-cxx-headers install-cxxabi-headers -- -v
diff --git a/libcxxabi/src/vendor/apple/shims.cpp b/libcxxabi/src/vendor/apple/shims.cpp
new file mode 100644
index 0000000000000..65152f7e0e8c9
--- /dev/null
+++ b/libcxxabi/src/vendor/apple/shims.cpp
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//
+// These shims implement symbols that are present in the system libc++ on Apple platforms
+// but are not implemented in upstream libc++. This allows testing libc++ under a system
+// library configuration, which requires the just-built libc++ to be ABI compatible with
+// the system library it is replacing.
+//
+
+#include <cstddef>
+#include <new>
+
+namespace std { // purposefully not versioned, like align_val_t
+enum class __type_descriptor_t : unsigned long long;
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::__type_descriptor_t) {
+  return ::operator new(__sz);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, const std::nothrow_t& __nt,
+                                                std::__type_descriptor_t) noexcept {
+  return ::operator new(__sz, __nt);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, std::__type_descriptor_t) {
+  return ::operator new[](__sz);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, const std::nothrow_t& __nt,
+                                                  std::__type_descriptor_t) noexcept {
+  return ::operator new(__sz, __nt);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::__type_descriptor_t) noexcept {
+  return ::operator delete(__p);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, const std::nothrow_t& __nt,
+                                                  std::__type_descriptor_t) noexcept {
+  return ::operator delete(__p, __nt);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::__type_descriptor_t) noexcept {
+  return ::operator delete[](__p);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, const std::nothrow_t& __nt,
+                                                    std::__type_descriptor_t) noexcept {
+  return ::operator delete[](__p, __nt);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz, std::__type_descriptor_t) noexcept {
+  return ::operator delete(__p, __sz);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz, std::__type_descriptor_t) noexcept {
+  return ::operator delete[](__p, __sz);
+}
diff --git a/libcxxabi/test/configs/apple-libc++abi-shared.cfg.in b/libcxxabi/test/configs/apple-libc++abi-shared.cfg.in
new file mode 100644
index 0000000000000..537fe8272a038
--- /dev/null
+++ b/libcxxabi/test/configs/apple-libc++abi-shared.cfg.in
@@ -0,0 +1,49 @@
+# Testing configuration for Apple's system libc++abi.
+#
+# This configuration differs from a normal LLVM shared library configuration in
+# that we must use DYLD_LIBRARY_PATH to run the tests against the just-built library,
+# since Apple's libc++abi has an absolute install_name.
+#
+# Finally, we also link against an artificial shims library that provides
+# the functionality necessary for the upstream libc++abi to be usable in place
+# of a system-provided libc++abi. Without that, attempting to replace the system
+# libc++abi with DYLD_LIBRARY_PATH would result in missing symbols and other similar
+# issues since the upstream libc++abi does not contain all the symbols provided by
+# the system library.
+
+lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
+
+import os, site
+site.addsitedir(os.path.join('@LIBCXXABI_LIBCXX_PATH@', 'utils'))
+import libcxx.test.params, libcxx.test.config, libcxx.test.dsl
+ADDITIONAL_PARAMETERS = [
+    libcxx.test.dsl.Parameter(name='apple_system_shims', type=str,
+        actions=lambda path: [libcxx.test.dsl.AddSubstitution('%{apple-system-shims}', path)],
+        help="""
+        Path to a pre-built object file or static archive that contains shims necessary to
+        allow replacing the system libc++abi by the just-built one.
+        """),
+]
+
+config.substitutions.append(('%{flags}',
+    '-isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else ''
+))
+config.substitutions.append(('%{compile_flags}',
+    '-nostdinc++ -I %{include} -I %{cxx-include} -I %{cxx-target-include} %{maybe-include-libunwind} -D_LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS ' +
+    '-I %{libcxx}/test/support -I %{libcxx}/src'
+))
+config.substitutions.append(('%{link_flags}',
+    '-nostdlib++ -L %{lib} -lc++ %{apple-system-shims}'
+))
+config.substitutions.append(('%{exec}',
+    '%{executor} --execdir %T --env DYLD_LIBRARY_PATH=%{lib} -- '
+))
+
+config.stdlib = 'apple-libc++'
+
+libcxx.test.config.configure(
+    libcxx.test.params.DEFAULT_PARAMETERS + ADDITIONAL_PARAMETERS,
+    libcxx.test.features.DEFAULT_FEATURES,
+    config,
+    lit_config
+)

From 50b5bb717ca5d2ec78587551bc90a8ddd0cc8983 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Wed, 17 Jul 2024 14:46:35 -0400
Subject: [PATCH 083/486] [libc++] Add comment about matching standard version
 in apple-install-libcxx

This was forgotten when I landed #99086
---
 libcxx/utils/ci/apple-install-libcxx.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcxx/utils/ci/apple-install-libcxx.sh b/libcxx/utils/ci/apple-install-libcxx.sh
index e5698ce93bc12..4872253a266a8 100755
--- a/libcxx/utils/ci/apple-install-libcxx.sh
+++ b/libcxx/utils/ci/apple-install-libcxx.sh
@@ -118,6 +118,7 @@ for arch in ${architectures}; do
 
     step "Building shims to make libc++ compatible with the system libc++ on Apple platforms when running the tests"
     shims_library="${build_dir}/${arch}/apple-system-shims.a"
+    # Note that this doesn't need to match the Standard version used to build the rest of the library.
     xcrun clang++ -c -std=c++2b -target ${target} "${llvm_root}/libcxxabi/src/vendor/apple/shims.cpp" -static -o "${shims_library}"
 
     step "Building libc++.dylib and libc++abi.dylib for architecture ${arch}"

From 4f786c6823956f72ef0b9eee57bf0c462842b609 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Thu, 18 Jul 2024 13:54:54 -0700
Subject: [PATCH 084/486] [SandboxIR][Tracker] Track
 Instruction::removeFromParent() (#99541)

This patch adds the necessary functionality to the Tracker and to the
Sandbox IR API functions for tracking calls to removeFromParent().
---
 llvm/include/llvm/SandboxIR/Tracker.h    | 21 +++++++++
 llvm/lib/SandboxIR/SandboxIR.cpp         |  4 ++
 llvm/lib/SandboxIR/Tracker.cpp           | 24 ++++++++++
 llvm/unittests/SandboxIR/TrackerTest.cpp | 57 ++++++++++++++++++++++++
 4 files changed, 106 insertions(+)

diff --git a/llvm/include/llvm/SandboxIR/Tracker.h b/llvm/include/llvm/SandboxIR/Tracker.h
index a91b9f178b8aa..6332b5266bb31 100644
--- a/llvm/include/llvm/SandboxIR/Tracker.h
+++ b/llvm/include/llvm/SandboxIR/Tracker.h
@@ -53,6 +53,7 @@
 namespace llvm::sandboxir {
 
 class BasicBlock;
+class Instruction;
 class Tracker;
 
 /// The base class for IR Change classes.
@@ -135,6 +136,26 @@ class EraseFromParent : public IRChangeBase {
 #endif
 };
 
+class RemoveFromParent : public IRChangeBase {
+  /// The instruction that is about to get removed.
+  Instruction *RemovedI = nullptr;
+  /// This is either the next instr, or the parent BB if at the end of the BB.
+  PointerUnion<Instruction *, BasicBlock *> NextInstrOrBB;
+
+public:
+  RemoveFromParent(Instruction *RemovedI, Tracker &Tracker);
+  void revert() final;
+  void accept() final {};
+  Instruction *getInstruction() const { return RemovedI; }
+#ifndef NDEBUG
+  void dump(raw_ostream &OS) const final {
+    dumpCommon(OS);
+    OS << "RemoveFromParent";
+  }
+  LLVM_DUMP_METHOD void dump() const final;
+#endif // NDEBUG
+};
+
 /// The tracker collects all the change objects and implements the main API for
 /// saving / reverting / accepting.
 class Tracker {
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index c6daf1a586546..249ebd7366204 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -333,6 +333,10 @@ Instruction *Instruction::getPrevNode() const {
 }
 
 void Instruction::removeFromParent() {
+  auto &Tracker = Ctx.getTracker();
+  if (Tracker.isTracking())
+    Tracker.track(std::make_unique<RemoveFromParent>(this, Tracker));
+
   // Detach all the LLVM IR instructions from their parent BB.
   for (llvm::Instruction *I : getLLVMInstrs())
     I->removeFromParent();
diff --git a/llvm/lib/SandboxIR/Tracker.cpp b/llvm/lib/SandboxIR/Tracker.cpp
index 2336a0067abbd..08ddb7b5c64c9 100644
--- a/llvm/lib/SandboxIR/Tracker.cpp
+++ b/llvm/lib/SandboxIR/Tracker.cpp
@@ -98,6 +98,30 @@ void EraseFromParent::dump() const {
   dump(dbgs());
   dbgs() << "\n";
 }
+#endif // NDEBUG
+
+RemoveFromParent::RemoveFromParent(Instruction *RemovedI, Tracker &Tracker)
+    : IRChangeBase(Tracker), RemovedI(RemovedI) {
+  if (auto *NextI = RemovedI->getNextNode())
+    NextInstrOrBB = NextI;
+  else
+    NextInstrOrBB = RemovedI->getParent();
+}
+
+void RemoveFromParent::revert() {
+  if (auto *NextI = NextInstrOrBB.dyn_cast<Instruction *>()) {
+    RemovedI->insertBefore(NextI);
+  } else {
+    auto *BB = NextInstrOrBB.get<BasicBlock *>();
+    RemovedI->insertInto(BB, BB->end());
+  }
+}
+
+#ifndef NDEBUG
+void RemoveFromParent::dump() const {
+  dump(dbgs());
+  dbgs() << "\n";
+}
 #endif
 
 void Tracker::track(std::unique_ptr<IRChangeBase> &&Change) {
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index 4e5dccf3a231f..e5c558345e187 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -198,3 +198,60 @@ define void @foo(i32 %v1) {
   EXPECT_EQ(&*It++, Ret);
   EXPECT_EQ(It, BB->end());
 }
+
+// TODO: Test multi-instruction patterns.
+TEST_F(TrackerTest, RemoveFromParent) {
+  parseIR(C, R"IR(
+define i32 @foo(i32 %arg) {
+  %add0 = add i32 %arg, %arg
+  %add1 = add i32 %add0, %arg
+  ret i32 %add1
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto *F = Ctx.createFunction(&LLVMF);
+  auto *Arg = F->getArg(0);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  sandboxir::Instruction *Add0 = &*It++;
+  sandboxir::Instruction *Add1 = &*It++;
+  sandboxir::Instruction *Ret = &*It++;
+
+  Ctx.save();
+  // Check removeFromParent().
+  Add1->removeFromParent();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+  // Removed instruction still be connected to operands and users.
+  EXPECT_EQ(Add1->getOperand(0), Add0);
+  EXPECT_EQ(Add1->getOperand(1), Arg);
+  EXPECT_EQ(Add0->getNumUses(), 1u);
+
+  // Check revert().
+  Ctx.revert();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+  EXPECT_EQ(Add1->getOperand(0), Add0);
+
+  // Same for the last instruction in the block.
+  Ctx.save();
+  Ret->removeFromParent();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(It, BB->end());
+  EXPECT_EQ(Ret->getOperand(0), Add1);
+  Ctx.revert();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+}

From ded35c0c3ad371287e80872d6bd104ce3f7d2864 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 18 Jul 2024 13:56:40 -0700
Subject: [PATCH 085/486] [vectorcombine] Pull sext/zext through
 reduce.or/and/xor (#99548)

This extends the existing foldTruncFromReductions transform to handle
sext and zext as well. This is only legal for the bitwise reductions
(and/or/xor) and not the arithmetic ones (add, mul). Use the same
costing decision to drive whether we do the transform.
---
 .../Transforms/Vectorize/VectorCombine.cpp    | 41 +++++++++++--------
 .../VectorCombine/RISCV/vecreduce-of-cast.ll  | 30 ++++++++++----
 2 files changed, 46 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 3a49f95d3f117..444598520c981 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -117,7 +117,7 @@ class VectorCombine {
   bool foldShuffleOfShuffles(Instruction &I);
   bool foldShuffleToIdentity(Instruction &I);
   bool foldShuffleFromReductions(Instruction &I);
-  bool foldTruncFromReductions(Instruction &I);
+  bool foldCastFromReductions(Instruction &I);
   bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
 
   void replaceValue(Value &Old, Value &New) {
@@ -2113,15 +2113,20 @@ bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
 
 /// Determine if its more efficient to fold:
 ///   reduce(trunc(x)) -> trunc(reduce(x)).
-bool VectorCombine::foldTruncFromReductions(Instruction &I) {
+///   reduce(sext(x))  -> sext(reduce(x)).
+///   reduce(zext(x))  -> zext(reduce(x)).
+bool VectorCombine::foldCastFromReductions(Instruction &I) {
   auto *II = dyn_cast<IntrinsicInst>(&I);
   if (!II)
     return false;
 
+  bool TruncOnly = false;
   Intrinsic::ID IID = II->getIntrinsicID();
   switch (IID) {
   case Intrinsic::vector_reduce_add:
   case Intrinsic::vector_reduce_mul:
+    TruncOnly = true;
+    break;
   case Intrinsic::vector_reduce_and:
   case Intrinsic::vector_reduce_or:
   case Intrinsic::vector_reduce_xor:
@@ -2133,35 +2138,37 @@ bool VectorCombine::foldTruncFromReductions(Instruction &I) {
   unsigned ReductionOpc = getArithmeticReductionInstruction(IID);
   Value *ReductionSrc = I.getOperand(0);
 
-  Value *TruncSrc;
-  if (!match(ReductionSrc, m_OneUse(m_Trunc(m_Value(TruncSrc)))))
+  Value *Src;
+  if (!match(ReductionSrc, m_OneUse(m_Trunc(m_Value(Src)))) &&
+      (TruncOnly || !match(ReductionSrc, m_OneUse(m_ZExtOrSExt(m_Value(Src))))))
     return false;
 
-  auto *TruncSrcTy = cast<VectorType>(TruncSrc->getType());
+  auto CastOpc =
+      (Instruction::CastOps)cast<Instruction>(ReductionSrc)->getOpcode();
+
+  auto *SrcTy = cast<VectorType>(Src->getType());
   auto *ReductionSrcTy = cast<VectorType>(ReductionSrc->getType());
   Type *ResultTy = I.getType();
 
   TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   InstructionCost OldCost = TTI.getArithmeticReductionCost(
       ReductionOpc, ReductionSrcTy, std::nullopt, CostKind);
-  if (auto *Trunc = dyn_cast<CastInst>(ReductionSrc))
-    OldCost +=
-        TTI.getCastInstrCost(Instruction::Trunc, ReductionSrcTy, TruncSrcTy,
-                             TTI::CastContextHint::None, CostKind, Trunc);
+  OldCost += TTI.getCastInstrCost(CastOpc, ReductionSrcTy, SrcTy,
+                                  TTI::CastContextHint::None, CostKind,
+                                  cast<CastInst>(ReductionSrc));
   InstructionCost NewCost =
-      TTI.getArithmeticReductionCost(ReductionOpc, TruncSrcTy, std::nullopt,
+      TTI.getArithmeticReductionCost(ReductionOpc, SrcTy, std::nullopt,
                                      CostKind) +
-      TTI.getCastInstrCost(Instruction::Trunc, ResultTy,
-                           ReductionSrcTy->getScalarType(),
+      TTI.getCastInstrCost(CastOpc, ResultTy, ReductionSrcTy->getScalarType(),
                            TTI::CastContextHint::None, CostKind);
 
   if (OldCost <= NewCost || !NewCost.isValid())
     return false;
 
-  Value *NewReduction = Builder.CreateIntrinsic(
-      TruncSrcTy->getScalarType(), II->getIntrinsicID(), {TruncSrc});
-  Value *NewTruncation = Builder.CreateTrunc(NewReduction, ResultTy);
-  replaceValue(I, *NewTruncation);
+  Value *NewReduction = Builder.CreateIntrinsic(SrcTy->getScalarType(),
+                                                II->getIntrinsicID(), {Src});
+  Value *NewCast = Builder.CreateCast(CastOpc, NewReduction, ResultTy);
+  replaceValue(I, *NewCast);
   return true;
 }
 
@@ -2559,7 +2566,7 @@ bool VectorCombine::run() {
       switch (Opcode) {
       case Instruction::Call:
         MadeChange |= foldShuffleFromReductions(I);
-        MadeChange |= foldTruncFromReductions(I);
+        MadeChange |= foldCastFromReductions(I);
         break;
       case Instruction::ICmp:
       case Instruction::FCmp:
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/vecreduce-of-cast.ll b/llvm/test/Transforms/VectorCombine/RISCV/vecreduce-of-cast.ll
index 9b1aa19f85c21..f04bcc90e5c35 100644
--- a/llvm/test/Transforms/VectorCombine/RISCV/vecreduce-of-cast.ll
+++ b/llvm/test/Transforms/VectorCombine/RISCV/vecreduce-of-cast.ll
@@ -74,8 +74,8 @@ define i16 @reduce_mul_trunc_v8i64_i16(<8 x i64> %a0)  {
 
 define i32 @reduce_or_sext_v8i8_to_v8i32(<8 x i8> %a0)  {
 ; CHECK-LABEL: @reduce_or_sext_v8i8_to_v8i32(
-; CHECK-NEXT:    [[TR:%.*]] = sext <8 x i8> [[A0:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[RED:%.*]] = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TR]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[A0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = sext i8 [[TMP1]] to i32
 ; CHECK-NEXT:    ret i32 [[RED]]
 ;
   %tr = sext <8 x i8> %a0 to <8 x i32>
@@ -85,8 +85,8 @@ define i32 @reduce_or_sext_v8i8_to_v8i32(<8 x i8> %a0)  {
 
 define i32 @reduce_or_sext_v8i16_to_v8i32(<8 x i16> %a0)  {
 ; CHECK-LABEL: @reduce_or_sext_v8i16_to_v8i32(
-; CHECK-NEXT:    [[TR:%.*]] = sext <8 x i16> [[A0:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[RED:%.*]] = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TR]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[A0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = sext i16 [[TMP1]] to i32
 ; CHECK-NEXT:    ret i32 [[RED]]
 ;
   %tr = sext <8 x i16> %a0 to <8 x i32>
@@ -96,8 +96,8 @@ define i32 @reduce_or_sext_v8i16_to_v8i32(<8 x i16> %a0)  {
 
 define i32 @reduce_or_zext_v8i8_to_v8i32(<8 x i8> %a0)  {
 ; CHECK-LABEL: @reduce_or_zext_v8i8_to_v8i32(
-; CHECK-NEXT:    [[TR:%.*]] = zext <8 x i8> [[A0:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[RED:%.*]] = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TR]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[A0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = zext i8 [[TMP1]] to i32
 ; CHECK-NEXT:    ret i32 [[RED]]
 ;
   %tr = zext <8 x i8> %a0 to <8 x i32>
@@ -107,8 +107,8 @@ define i32 @reduce_or_zext_v8i8_to_v8i32(<8 x i8> %a0)  {
 
 define i32 @reduce_or_zext_v8i16_to_v8i32(<8 x i16> %a0)  {
 ; CHECK-LABEL: @reduce_or_zext_v8i16_to_v8i32(
-; CHECK-NEXT:    [[TR:%.*]] = zext <8 x i16> [[A0:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[RED:%.*]] = tail call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TR]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[A0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = zext i16 [[TMP1]] to i32
 ; CHECK-NEXT:    ret i32 [[RED]]
 ;
   %tr = zext <8 x i16> %a0 to <8 x i32>
@@ -116,6 +116,20 @@ define i32 @reduce_or_zext_v8i16_to_v8i32(<8 x i16> %a0)  {
   ret i32 %red
 }
 
+; Negative case - narrowing the reduce (to i8) is illegal.
+; TODO: We could narrow to i16 instead.
+define i32 @reduce_add_trunc_v8i8_to_v8i32(<8 x i8> %a0)  {
+; CHECK-LABEL: @reduce_add_trunc_v8i8_to_v8i32(
+; CHECK-NEXT:    [[TR:%.*]] = zext <8 x i8> [[A0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[RED:%.*]] = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TR]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %tr = zext <8 x i8> %a0 to <8 x i32>
+  %red = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %tr)
+  ret i32 %red
+}
+
+
 declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
 declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)
 declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)

From 19c9a1c2fd2c3849439bca2ba5084dada4d1f47f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 18 Jul 2024 22:09:04 +0100
Subject: [PATCH 086/486] [LAA] Include IndirectUnsafe in ::isPossiblyBackward.

Similarly to Unknown, IndirectUnsafe should also be considered possibly
backward, as it may be a backwards dependency e.g. via loading
different base pointers.

This also brings isPossiblyBackward in line with
Dependence::isSafeForVectorization. At the moment this can't be tested,
as it is not possible to write a test with an AddRec that is based on a
loop varying value. But this may change in the future and may cause
mis-compiles in the future.
---
 llvm/lib/Analysis/LoopAccessAnalysis.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 91994f33f3046..84214c47a10e1 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1730,7 +1730,7 @@ bool MemoryDepChecker::Dependence::isBackward() const {
 }
 
 bool MemoryDepChecker::Dependence::isPossiblyBackward() const {
-  return isBackward() || Type == Unknown;
+  return isBackward() || Type == Unknown || Type == IndirectUnsafe;
 }
 
 bool MemoryDepChecker::Dependence::isForward() const {

From 2ec1a39bcb1ab893f4b6cde7fc9e24cfdff0aeb6 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 18 Jul 2024 14:11:42 -0700
Subject: [PATCH 087/486] [flang][preprocessor] Handle initial "MACRO&" with no
 space (#98684)

The prescanner checks lines that begin with a keyword macro name to see
whether they should be treated as a comment or compiler directive
instead of a source line. This fails when the potential keyword macro
name is extended with identifier characters via Fortran line
continuation. Disable line continuation during this check.
---
 flang/lib/Parser/prescan.cpp                    | 16 ++++++++++++----
 .../Preprocessing/directive-contin-with-pp.F90  | 17 ++++++++++++++---
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index ca6a823999244..aa45548408aa9 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -189,11 +189,19 @@ void Prescanner::Statement() {
       // a comment marker or directive sentinel.  If so, disable line
       // continuation, so that NextToken() won't consume anything from
       // following lines.
-      if (IsLegalIdentifierStart(*at_) && NextToken(tokens) &&
-          tokens.SizeInTokens() > 0) {
-        if (CharBlock id{tokens.TokenAt(0)}; preprocessor_.IsNameDefined(id) &&
+      if (IsLegalIdentifierStart(*at_)) {
+        // TODO: Only bother with these cases when any keyword macro has
+        // been defined with replacement text that could begin a comment
+        // or directive sentinel.
+        const char *p{at_};
+        while (IsLegalInIdentifier(*++p)) {
+        }
+        CharBlock id{at_, static_cast<std::size_t>(p - at_)};
+        if (preprocessor_.IsNameDefined(id) &&
             !preprocessor_.IsFunctionLikeDefinition(id)) {
-          if (auto replaced{preprocessor_.MacroReplacement(tokens, *this)}) {
+          TokenSequence toks;
+          toks.Put(id, GetProvenance(at_));
+          if (auto replaced{preprocessor_.MacroReplacement(toks, *this)}) {
             auto newLineClass{ClassifyLine(*replaced, GetCurrentProvenance())};
             disableSourceContinuation_ =
                 newLineClass.kind != LineClassification::Kind::Source;
diff --git a/flang/test/Preprocessing/directive-contin-with-pp.F90 b/flang/test/Preprocessing/directive-contin-with-pp.F90
index 64f1dc43f72b4..544c6619f6b53 100644
--- a/flang/test/Preprocessing/directive-contin-with-pp.F90
+++ b/flang/test/Preprocessing/directive-contin-with-pp.F90
@@ -11,7 +11,7 @@
 
 module m
  contains
-  subroutine s(x1, x2, x3, x4, x5, x6, x7)
+  subroutine s1(x1, x2, x3, x4, x5, x6, x7)
 
 !dir$ ignore_tkr x1
 
@@ -51,11 +51,18 @@ subroutine s(x1, x2, x3, x4, x5, x6, x7)
     do j3 = 1, n
     end do
   end
-end
+
+COMMENT &
+  subroutine s2
+  end subroutine
+COMMENT&
+  subroutine s3
+  end subroutine
+end module
 
 !CHECK: MODULE m
 !CHECK: CONTAINS
-!CHECK:  SUBROUTINE s (x1, x2, x3, x4, x5, x6, x7)
+!CHECK:  SUBROUTINE s1 (x1, x2, x3, x4, x5, x6, x7)
 !CHECK:   !DIR$ IGNORE_TKR x1
 !CHECK:   !DIR$ IGNORE_TKR x2
 !CHECK:   !DIR$ IGNORE_TKR x3
@@ -73,4 +80,8 @@ subroutine s(x1, x2, x3, x4, x5, x6, x7)
 !CHECK:   DO j3=1_4,n
 !CHECK:   END DO
 !CHECK:  END SUBROUTINE
+!CHECK:  SUBROUTINE s2
+!CHECK:  END SUBROUTINE
+!CHECK:  SUBROUTINE s3
+!CHECK:  END SUBROUTINE
 !CHECK: END MODULE

From f61c9a9485cc359cdd0262e66840e095dbc246c2 Mon Sep 17 00:00:00 2001
From: OverMighty <its.overmighty@gmail.com>
Date: Thu, 18 Jul 2024 23:16:48 +0200
Subject: [PATCH 088/486] [libc][CMake] Set library type of libcMPFRWrapper to
 STATIC (#99527)

Fixes linker errors due to hidden symbols when running CMake with
-DBUILD_SHARED_LIBS=ON.
---
 libc/utils/MPFRWrapper/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/utils/MPFRWrapper/CMakeLists.txt b/libc/utils/MPFRWrapper/CMakeLists.txt
index e74b02204ed6f..be0415d0956fa 100644
--- a/libc/utils/MPFRWrapper/CMakeLists.txt
+++ b/libc/utils/MPFRWrapper/CMakeLists.txt
@@ -1,5 +1,5 @@
 if(LIBC_TESTS_CAN_USE_MPFR)
-  add_library(libcMPFRWrapper
+  add_library(libcMPFRWrapper STATIC
     MPFRUtils.cpp
     MPFRUtils.h
     mpfr_inc.h

From 4afdcd98c5e28e0b353b21bf01c5cd3568878e41 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 18 Jul 2024 14:16:53 -0700
Subject: [PATCH 089/486] [AArch64,test] Remove over reliance on section
 offsets and symbol indexes

---
 llvm/test/MC/AArch64/elf-reloc-ptrauth.s | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/llvm/test/MC/AArch64/elf-reloc-ptrauth.s b/llvm/test/MC/AArch64/elf-reloc-ptrauth.s
index 3bd8f5c19932e..057b298a0a0df 100644
--- a/llvm/test/MC/AArch64/elf-reloc-ptrauth.s
+++ b/llvm/test/MC/AArch64/elf-reloc-ptrauth.s
@@ -3,16 +3,16 @@
 // RUN: llvm-mc -triple=aarch64 -filetype=obj %s | \
 // RUN:   llvm-readelf -S -r -x .test - | FileCheck %s --check-prefix=RELOC
 
-// RELOC: Relocation section '.rela.test' at offset 0x230 contains 8 entries:
+// RELOC: Relocation section '.rela.test' at offset {{.*}} contains 8 entries:
 // RELOC-NEXT:  Offset Info Type Symbol's Value Symbol's Name + Addend
-// RELOC-NEXT: 0000000000000000 0000000100000244 R_AARCH64_AUTH_ABS64 0000000000000000 .helper + 0
-// RELOC-NEXT: 0000000000000010 0000000800000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g1 + 0
-// RELOC-NEXT: 0000000000000020 0000000900000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g2 + 0
-// RELOC-NEXT: 0000000000000030 0000000a00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g3 + 0
-// RELOC-NEXT: 0000000000000040 0000000b00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g4 + 7
-// RELOC-NEXT: 0000000000000050 0000000c00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g5 - 3
-// RELOC-NEXT: 0000000000000060 0000000200000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g 6 + 0
-// RELOC-NEXT: 0000000000000070 0000000d00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g 7 + 7
+// RELOC-NEXT: 0000000000000000 {{.*}} R_AARCH64_AUTH_ABS64 0000000000000000 .helper + 0
+// RELOC-NEXT: 0000000000000010 {{.*}} R_AARCH64_AUTH_ABS64 0000000000000000 _g1 + 0
+// RELOC-NEXT: 0000000000000020 {{.*}} R_AARCH64_AUTH_ABS64 0000000000000000 _g2 + 0
+// RELOC-NEXT: 0000000000000030 {{.*}} R_AARCH64_AUTH_ABS64 0000000000000000 _g3 + 0
+// RELOC-NEXT: 0000000000000040 {{.*}} R_AARCH64_AUTH_ABS64 0000000000000000 _g4 + 7
+// RELOC-NEXT: 0000000000000050 {{.*}} R_AARCH64_AUTH_ABS64 0000000000000000 _g5 - 3
+// RELOC-NEXT: 0000000000000060 {{.*}} R_AARCH64_AUTH_ABS64 0000000000000000 _g 6 + 0
+// RELOC-NEXT: 0000000000000070 {{.*}} R_AARCH64_AUTH_ABS64 0000000000000000 _g 7 + 7
 
 // RELOC: Hex dump of section '.test':
 //                VVVVVVVV addend, not needed for rela

From 99faa038c66872f6507dfe1d5aa470ba13e58614 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 18 Jul 2024 14:41:15 -0700
Subject: [PATCH 090/486] [mlir] Fix a warning

This patch fixes:

  mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp:621:16: error:
  unused variable 'removed' [-Werror,-Wunused-variable]
---
 mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index 5cac770b03ed1..3a2042d23e534 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -620,6 +620,7 @@ void allocateTilesToLiveRanges(
         if (!activeRanges.remove(rangeToSpill)) {
           bool removed = inactiveRanges.remove(rangeToSpill);
           assert(removed && "expected a range to be removed!");
+          (void)removed;
         }
       }
       rangeToSpill->tileId = tileAllocator.allocateInMemoryTileId();

From 6c6baf019a25710cdc3c3971b09cd6332d0379c0 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Thu, 18 Jul 2024 14:46:12 -0700
Subject: [PATCH 091/486] Revert "[clang][test] Split AArch64 target feature
 checks across multiple lines. NFC (#99365)"

This reverts commit 06518cea3905556d8d1eea4088132ebb234bfdab.
This reverts commit 05275b05ca58e4d015eea1503f120e6967ef1b91.
This reverts commit c0c4ad5d9a6e05e0b1f5f98ce2e08d479b281be8.
---
 .../Preprocessor/aarch64-target-features.c    | 54 +++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index 85424c6862795..71cc36acf3f0e 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -536,8 +536,62 @@
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=standard -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=bti -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret+bti -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI %s
+// CHECK-BTI-OFF-NOT: __ARM_FEATURE_BTI_DEFAULT
+// CHECK-BTI:         #define __ARM_FEATURE_BTI_DEFAULT 1
+
+// ================== Check Armv8.5-A random number generation extension.
+// RUN: %clang -target aarch64-none-elf -march=armv8.5-a+rng -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-RNG %s
+// RUN: %clang -target aarch64-none-elf -march=armv8.5-a -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-NO-RNG %s
+// CHECK-RNG: __ARM_FEATURE_RNG 1
+// CHECK-NO-RNG-NOT: __ARM_FEATURE_RNG 1
+
+// ================== Check BFloat16 Extensions.
+// RUN: %clang -target aarch64-none-elf -march=armv8.6-a+bf16 -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-BFLOAT %s
+// CHECK-BFLOAT: __ARM_BF16_FORMAT_ALTERNATIVE 1
+// CHECK-BFLOAT: __ARM_FEATURE_BF16 1
+// CHECK-BFLOAT: __ARM_FEATURE_BF16_VECTOR_ARITHMETIC 1
+
+// ================== Check Armv8.7-A LS64 extension.
+// RUN: %clang -target aarch64-none-elf -march=armv8.7-a+ls64 -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-LS64 %s
+// RUN: %clang -target aarch64-none-elf -march=armv8.7-a      -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-NO-LS64 %s
+// CHECK-LS64: __ARM_FEATURE_LS64 1
+// CHECK-NO-LS64-NOT: __ARM_FEATURE_LS64 1
+
+// ================== Check sve-vector-bits flag.
+// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=128  -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-SVE-VECTOR-BITS -D#VBITS=128  %s
+// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=256  -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-SVE-VECTOR-BITS -D#VBITS=256  %s
+// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=512  -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-SVE-VECTOR-BITS -D#VBITS=512  %s
+// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=1024 -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-SVE-VECTOR-BITS -D#VBITS=1024 %s
+// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=2048 -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-SVE-VECTOR-BITS -D#VBITS=2048 %s
+// RUN: %clang -target aarch64-none-elf -march=armv8-a+sve -msve-vector-bits=512+ -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-NO-SVE-VECTOR-BITS %s
+// CHECK-SVE-VECTOR-BITS: __ARM_FEATURE_SVE_BITS [[#VBITS:]]
+// CHECK-NO-SVE-VECTOR-BITS-NOT: __ARM_FEATURE_SVE_BITS
+
+// ================== Check Large System Extensions (LSE)
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+lse -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-LSE %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a+lse -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-LSE %s
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8.1-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-LSE %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8.1-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-LSE %s
+// CHECK-LSE: __ARM_FEATURE_ATOMICS 1
+
+// ================== Check Armv8.8-A/Armv9.3-A memcpy and memset acceleration instructions (MOPS)
+// RUN: %clang -target aarch64-none-elf -march=armv8.7-a             -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS %s
+// RUN: %clang -target aarch64-none-elf -march=armv8.7-a+mops        -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
+// RUN: %clang -target aarch64-none-elf -march=armv8.8-a             -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
+// RUN: %clang -target aarch64-none-elf -march=armv8.8-a+nomops      -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS %s
+// RUN: %clang -target aarch64-none-elf -march=armv8.8-a+nomops+mops -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
+// RUN: %clang -target aarch64-none-elf -march=armv8.8-a+mops        -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
+// RUN: %clang -target aarch64-none-elf -march=armv8.8-a+mops+nomops -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS %s
+// RUN: %clang -target aarch64-none-elf -march=armv9.2-a             -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS %s
+// RUN: %clang -target aarch64-none-elf -march=armv9.2-a+mops        -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
+// RUN: %clang -target aarch64-none-elf -march=armv9.3-a             -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
+// RUN: %clang -target aarch64-none-elf -march=armv9.3-a+nomops      -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS %s
+// RUN: %clang -target aarch64-none-elf -march=armv9.3-a+mops        -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-MOPS   %s
+// Check that -target-feature -v9.3a doesn't enable dependant features
+// RUN: %clang -target aarch64-none-elf -Xclang -target-feature -Xclang -v9.3a  -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOMOPS   %s
 // CHECK-MOPS: __ARM_FEATURE_MOPS 1
 // CHECK-NOMOPS-NOT: __ARM_FEATURE_MOPS 1
+
 // ================== Check Armv8.9-A/Armv9.4-A 128-bit System Registers (FEAT_SYSREG128)
 // RUN: %clang -target aarch64-none-elf -march=armv8.9-a      -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOSYS128 %s
 // RUN: %clang -target aarch64-none-elf -march=armv9.4-a      -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOSYS128 %s

From 05bce3f079b677edd0efd28e3923f4776ffb8b59 Mon Sep 17 00:00:00 2001
From: Chenguang Wang <w3cing@gmail.com>
Date: Thu, 18 Jul 2024 14:55:22 -0700
Subject: [PATCH 092/486] [mlir][ArmSME] Suppress potential unused warning
 (#99573)

When building in release mode, the assert will be dropped, making
`remove` unused.
---
 mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index 3a2042d23e534..e3fc3473c6c6b 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -619,6 +619,7 @@ void allocateTilesToLiveRanges(
         // Remove the live range from the active/inactive sets.
         if (!activeRanges.remove(rangeToSpill)) {
           bool removed = inactiveRanges.remove(rangeToSpill);
+          (void)removed;
           assert(removed && "expected a range to be removed!");
           (void)removed;
         }

From a41a4b8feda789529e4e927d9d6a04b7be8e9664 Mon Sep 17 00:00:00 2001
From: Chenguang Wang <w3cing@gmail.com>
Date: Thu, 18 Jul 2024 14:59:26 -0700
Subject: [PATCH 093/486] Revert "[mlir][ArmSME] Suppress potential unused
 warning (#99573)" (#99578)

This reverts commit 05bce3f079b677edd0efd28e3923f4776ffb8b59.

The work was already done in 99faa03.
---
 mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index e3fc3473c6c6b..3a2042d23e534 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -619,7 +619,6 @@ void allocateTilesToLiveRanges(
         // Remove the live range from the active/inactive sets.
         if (!activeRanges.remove(rangeToSpill)) {
           bool removed = inactiveRanges.remove(rangeToSpill);
-          (void)removed;
           assert(removed && "expected a range to be removed!");
           (void)removed;
         }

From 8be714b6724f1f2f57442f2cdd3a0625c053d872 Mon Sep 17 00:00:00 2001
From: Ilya Leoshkevich <iii@linux.ibm.com>
Date: Fri, 19 Jul 2024 00:00:36 +0200
Subject: [PATCH 094/486] [sanitizer] Fix running
 sanitizer_set_report_path_test on Android (#99469)

sanitizer_set_report_path_test outputs the following on an Android
builder [1]:

ERROR: Can't create directory:
/data/local/tmp/Output/var/lib/buildbot/sanitizer-buildbot6/sanitizer-x86_64-linux-android/build/compiler_rt_build_android_arm/test/sanitizer_common/asan-arm-Android/Posix/Output/sanitizer_set_report_path_test.cpp.tmp
Path
/data/local/tmp/Output/var/lib/buildbot/sanitizer-buildbot6/sanitizer-x86_64-linux-android/build/compiler_rt_build_android_arm/test/sanitizer_common/asan-arm-Android/Posix/Output/sanitizer_set_report_path_test.cpp.tmp.report_path/report.24954

The order of messages is reversed.

The test can use strcmp+assert instead of CHECK for `__sanitizer_get_report_path` output.

[1]
https://lab.llvm.org/buildbot/#/builders/186/builds/703/steps/26/logs/stdio

---------

Co-authored-by: Vitaly Buka <vitalybuka@gmail.com>
---
 .../Posix/sanitizer_set_report_path_test.cpp          | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/sanitizer_set_report_path_test.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/sanitizer_set_report_path_test.cpp
index 286eafc315baf..21ffe1381bd46 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/sanitizer_set_report_path_test.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/sanitizer_set_report_path_test.cpp
@@ -14,15 +14,12 @@ int main(int argc, char **argv) {
   sprintf(buff, "%s.report_path/report", argv[0]);
   __sanitizer_set_report_path(buff);
   assert(strncmp(buff, __sanitizer_get_report_path(), strlen(buff)) == 0);
-  printf("Path %s\n", __sanitizer_get_report_path());
-  fflush(stdout);
 
   // Try setting again with an invalid/inaccessible directory.
-  sprintf(buff, "%s/report", argv[0]);
-  __sanitizer_set_report_path(buff);
-  printf("Path %s\n", __sanitizer_get_report_path());
+  char buff_bad[1000];
+  sprintf(buff_bad, "%s/report", argv[0]);
+  __sanitizer_set_report_path(buff_bad);
+  assert(strncmp(buff, __sanitizer_get_report_path(), strlen(buff)) == 0);
 }
 
-// CHECK: Path {{.*}}Posix/Output/sanitizer_set_report_path_test.cpp.tmp.report_path/report.
 // CHECK: ERROR: Can't create directory: {{.*}}Posix/Output/sanitizer_set_report_path_test.cpp.tmp
-// CHECK-NOT: Path {{.*}}Posix/Output/sanitizer_set_report_path_test.cpp.tmp

From 1f6f97e2b64a9c4c51712d32b53e07d2a20e0cc0 Mon Sep 17 00:00:00 2001
From: Bill Wendling <5993918+bwendling@users.noreply.github.com>
Date: Thu, 18 Jul 2024 22:11:56 +0000
Subject: [PATCH 095/486] [Clang] Loop over FieldDecls instead of all Decls
 (#99574)

Only FieldDecls are important when determining GEP indices. A struct
defined within another struct has the same semantics as if it were
defined outside of the struct. So there's no need to look into
RecordDecls that aren't a field.

See commit 5bcf31ebfad8 ("[Clang] Loop over FieldDecls instead of all
Decls (#89453)")

Fixes 2039.
---
 clang/lib/CodeGen/CGExpr.cpp         | 21 ++++++-----
 clang/test/CodeGen/attr-counted-by.c | 54 ++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 11 deletions(-)

diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 6a0af00b9e186..c5341e4b53651 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -1071,21 +1071,20 @@ using RecIndicesTy =
     SmallVector<std::pair<const RecordDecl *, llvm::Value *>, 8>;
 
 static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD,
-                                 const FieldDecl *FD, RecIndicesTy &Indices) {
+                                 const FieldDecl *Field,
+                                 RecIndicesTy &Indices) {
   const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD);
   int64_t FieldNo = -1;
-  for (const Decl *D : RD->decls()) {
-    if (const auto *Field = dyn_cast<FieldDecl>(D)) {
-      FieldNo = Layout.getLLVMFieldNo(Field);
-      if (FD == Field) {
-        Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo)));
-        return true;
-      }
+  for (const FieldDecl *FD : RD->fields()) {
+    FieldNo = Layout.getLLVMFieldNo(FD);
+    if (FD == Field) {
+      Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo)));
+      return true;
     }
 
-    if (const auto *Record = dyn_cast<RecordDecl>(D)) {
-      ++FieldNo;
-      if (getGEPIndicesToField(CGF, Record, FD, Indices)) {
+    QualType Ty = FD->getType();
+    if (Ty->isRecordType()) {
+      if (getGEPIndicesToField(CGF, Ty->getAsRecordDecl(), Field, Indices)) {
         if (RD->isUnion())
           FieldNo = 0;
         Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo)));
diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c
index 79922eb4159f1..32db136076d7d 100644
--- a/clang/test/CodeGen/attr-counted-by.c
+++ b/clang/test/CodeGen/attr-counted-by.c
@@ -1852,3 +1852,57 @@ struct annotated_struct_array {
 void test29(struct annotated_struct_array *ann, int idx1, int idx2) {
   ann->ann_array[idx1]->array[idx2] = __builtin_dynamic_object_size(ann->ann_array[idx1]->array, 1);
 }
+
+typedef struct {
+  char __padding[0];
+} test30_spinlock_t;
+
+struct test30_struct {
+  struct test30_decl *name_node;
+  int priv_len;
+  test30_spinlock_t pcpu_refcnt;
+  char priv[] __counted_by(priv_len);
+};
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local void @test30(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR5]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[IDX]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB39:[0-9]+]], i64 [[TMP0]]) #[[ATTR10]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test30(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.smax.i32(i32 [[DOT_COUNTED_BY_LOAD]], i32 4)
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOTINV:%.*]] = icmp slt i32 [[DOT_COUNTED_BY_LOAD]], 0
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = add i8 [[TMP1]], 12
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = select i1 [[DOTINV]], i8 0, i8 [[TMP2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[PCPU_REFCNT:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 12
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i8], ptr [[PCPU_REFCNT]], i64 0, i64 [[IDXPROM]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i8 [[CONV]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA6]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test30(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[IDX]] to i64, !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP0]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test30(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[PCPU_REFCNT:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 12
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i8], ptr [[PCPU_REFCNT]], i64 0, i64 [[IDXPROM]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store i8 -1, ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA6]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+void test30(struct test30_struct *ptr, int idx) {
+  ptr->pcpu_refcnt.__padding[idx] = __builtin_dynamic_object_size(ptr, 1);
+}

From f1a8db1a1f93a7e921e10c362a659712a4be2a33 Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj@google.com>
Date: Thu, 18 Jul 2024 15:32:32 -0700
Subject: [PATCH 096/486] [libc][newhdrgen] Remove redundant yaml prefixes
 (#99581)

Since the yaml files are already organized into folders, the name of the
file doesn't also need to be prefixed by what folder it's in.
---
 libc/include/CMakeLists.txt                   | 38 +++++++++----------
 .../yaml/arpa/{arpa_inet.yaml => inet.yaml}   |  0
 .../yaml/gpu/{gpu_rpc.yaml => rpc.yaml}       |  0
 .../yaml/sys/{sys_auxv.yaml => auxv.yaml}     |  0
 .../yaml/sys/{sys_epoll.yaml => epoll.yaml}   |  0
 .../yaml/sys/{sys_ioctl.yaml => ioctl.yaml}   |  0
 .../yaml/sys/{sys_mman.yaml => mman.yaml}     |  0
 .../yaml/sys/{sys_prctl.yaml => prctl.yaml}   |  0
 .../yaml/sys/{sys_random.yaml => random.yaml} |  0
 .../sys/{sys_resource.yaml => resource.yaml}  |  0
 .../yaml/sys/{sys_select.yaml => select.yaml} |  0
 .../sys/{sys_sendfile.yaml => sendfile.yaml}  |  0
 .../yaml/sys/{sys_socket.yaml => socket.yaml} |  0
 .../yaml/sys/{sys_stat.yaml => stat.yaml}     |  0
 .../sys/{sys_statvfs.yaml => statvfs.yaml}    |  0
 .../sys/{sys_syscall.yaml => syscall.yaml}    |  0
 .../yaml/sys/{sys_time.yaml => time.yaml}     |  0
 .../yaml/sys/{sys_types.yaml => types.yaml}   |  0
 .../sys/{sys_utsname.yaml => utsname.yaml}    |  0
 .../yaml/sys/{sys_wait.yaml => wait.yaml}     |  0
 20 files changed, 19 insertions(+), 19 deletions(-)
 rename libc/newhdrgen/yaml/arpa/{arpa_inet.yaml => inet.yaml} (100%)
 rename libc/newhdrgen/yaml/gpu/{gpu_rpc.yaml => rpc.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_auxv.yaml => auxv.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_epoll.yaml => epoll.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_ioctl.yaml => ioctl.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_mman.yaml => mman.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_prctl.yaml => prctl.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_random.yaml => random.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_resource.yaml => resource.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_select.yaml => select.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_sendfile.yaml => sendfile.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_socket.yaml => socket.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_stat.yaml => stat.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_statvfs.yaml => statvfs.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_syscall.yaml => syscall.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_time.yaml => time.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_types.yaml => types.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_utsname.yaml => utsname.yaml} (100%)
 rename libc/newhdrgen/yaml/sys/{sys_wait.yaml => wait.yaml} (100%)

diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index df01172854871..37cae19123318 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -174,7 +174,7 @@ file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/arpa)
 
 add_header_macro(
   arpa_inet
-  ../libc/newhdrgen/yaml/arpa/arpa_inet.yaml
+  ../libc/newhdrgen/yaml/arpa/inet.yaml
   arpa/inet.h.def
   arpa/inet.h
   DEPENDS
@@ -426,7 +426,7 @@ file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/sys)
 
 add_header_macro(
   sys_auxv
-  ../libc/newhdrgen/yaml/sys/sys_auxv.yaml
+  ../libc/newhdrgen/yaml/sys/auxv.yaml
   sys/auxv.h.def
   sys/auxv.h
   DEPENDS
@@ -436,7 +436,7 @@ add_header_macro(
 
 add_header_macro(
   sys_epoll
-  ../libc/newhdrgen/yaml/sys/sys_epoll.yaml
+  ../libc/newhdrgen/yaml/sys/epoll.yaml
   sys/epoll.h.def
   sys/epoll.h
   DEPENDS
@@ -449,7 +449,7 @@ add_header_macro(
 
 add_header_macro(
   sys_ioctl
-  ../libc/newhdrgen/yaml/sys/sys_ioctl.yaml
+  ../libc/newhdrgen/yaml/sys/ioctl.yaml
   sys/ioctl.h.def
   sys/ioctl.h
   DEPENDS
@@ -459,7 +459,7 @@ add_header_macro(
 
 add_header_macro(
   sys_mman
-  ../libc/newhdrgen/yaml/sys/sys_mman.yaml
+  ../libc/newhdrgen/yaml/sys/mman.yaml
   sys/mman.h.def
   sys/mman.h
   DEPENDS
@@ -472,7 +472,7 @@ add_header_macro(
 
 add_header_macro(
   sys_prctl
-  ../libc/newhdrgen/yaml/sys/sys_prctl.yaml
+  ../libc/newhdrgen/yaml/sys/prctl.yaml
   sys/prctl.h.def
   sys/prctl.h
   DEPENDS
@@ -489,7 +489,7 @@ add_header(
 
 add_header_macro(
   sys_random
-  ../libc/newhdrgen/yaml/sys/sys_random.yaml
+  ../libc/newhdrgen/yaml/sys/random.yaml
   sys/random.h.def
   sys/random.h
   DEPENDS
@@ -501,7 +501,7 @@ add_header_macro(
 
 add_header_macro(
   sys_resource
-  ../libc/newhdrgen/yaml/sys/sys_resource.yaml
+  ../libc/newhdrgen/yaml/sys/resource.yaml
   sys/resource.h.def
   sys/resource.h
   DEPENDS
@@ -513,7 +513,7 @@ add_header_macro(
 
 add_header_macro(
   sys_stat
-  ../libc/newhdrgen/yaml/sys/sys_stat.yaml
+  ../libc/newhdrgen/yaml/sys/stat.yaml
   sys/stat.h.def
   sys/stat.h
   DEPENDS
@@ -535,7 +535,7 @@ add_header_macro(
 
 add_header_macro(
   sys_select
-  ../libc/newhdrgen/yaml/sys/sys_select.yaml
+  ../libc/newhdrgen/yaml/sys/select.yaml
   sys/select.h.def
   sys/select.h
   DEPENDS
@@ -551,7 +551,7 @@ add_header_macro(
 
 add_header_macro(
   sys_sendfile
-  ../libc/newhdrgen/yaml/sys/sys_sendfile.yaml
+  ../libc/newhdrgen/yaml/sys/sendfile.yaml
   sys/sendfile.h.def
   sys/sendfile.h
   DEPENDS
@@ -563,7 +563,7 @@ add_header_macro(
 
 add_header_macro(
   sys_socket
-  ../libc/newhdrgen/yaml/sys/sys_socket.yaml
+  ../libc/newhdrgen/yaml/sys/socket.yaml
   sys/socket.h.def
   sys/socket.h
   DEPENDS
@@ -577,7 +577,7 @@ add_header_macro(
 
 add_header_macro(
   sys_statvfs
-  ../libc/newhdrgen/yaml/sys/sys_statvfs.yaml
+  ../libc/newhdrgen/yaml/sys/statvfs.yaml
   sys/statvfs.h.def
   sys/statvfs.h
   DEPENDS
@@ -587,7 +587,7 @@ add_header_macro(
 
 add_header_macro(
   sys_syscall
-  ../libc/newhdrgen/yaml/sys/sys_syscall.yaml
+  ../libc/newhdrgen/yaml/sys/syscall.yaml
   sys/syscall.h.def
   sys/syscall.h
   DEPENDS
@@ -595,7 +595,7 @@ add_header_macro(
 
 add_header_macro(
   sys_time
-  ../libc/newhdrgen/yaml/sys/sys_time.yaml
+  ../libc/newhdrgen/yaml/sys/time.yaml
   sys/time.h.def
   sys/time.h
   DEPENDS
@@ -606,7 +606,7 @@ add_header_macro(
 
 add_header_macro(
   sys_types
-  ../libc/newhdrgen/yaml/sys/sys_types.yaml
+  ../libc/newhdrgen/yaml/sys/types.yaml
   sys/types.h.def
   sys/types.h
   DEPENDS
@@ -636,7 +636,7 @@ add_header_macro(
 
 add_header_macro(
   sys_utsname
-  ../libc/newhdrgen/yaml/sys/sys_utsname.yaml
+  ../libc/newhdrgen/yaml/sys/utsname.yaml
   sys/utsname.h.def
   sys/utsname.h
   DEPENDS
@@ -646,7 +646,7 @@ add_header_macro(
 
 add_header_macro(
   sys_wait
-  ../libc/newhdrgen/yaml/sys/sys_wait.yaml
+  ../libc/newhdrgen/yaml/sys/wait.yaml
   sys/wait.h.def
   sys/wait.h
   DEPENDS
@@ -776,4 +776,4 @@ if(LLVM_LIBC_FULL_BUILD)
                             -P "${CMAKE_BINARY_DIR}/cmake_install.cmake")
   # Stripping is a no-op for headers
   add_custom_target(install-libc-headers-stripped DEPENDS install-libc-headers)
-endif()
\ No newline at end of file
+endif()
diff --git a/libc/newhdrgen/yaml/arpa/arpa_inet.yaml b/libc/newhdrgen/yaml/arpa/inet.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/arpa/arpa_inet.yaml
rename to libc/newhdrgen/yaml/arpa/inet.yaml
diff --git a/libc/newhdrgen/yaml/gpu/gpu_rpc.yaml b/libc/newhdrgen/yaml/gpu/rpc.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/gpu/gpu_rpc.yaml
rename to libc/newhdrgen/yaml/gpu/rpc.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_auxv.yaml b/libc/newhdrgen/yaml/sys/auxv.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_auxv.yaml
rename to libc/newhdrgen/yaml/sys/auxv.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_epoll.yaml b/libc/newhdrgen/yaml/sys/epoll.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_epoll.yaml
rename to libc/newhdrgen/yaml/sys/epoll.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_ioctl.yaml b/libc/newhdrgen/yaml/sys/ioctl.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_ioctl.yaml
rename to libc/newhdrgen/yaml/sys/ioctl.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_mman.yaml b/libc/newhdrgen/yaml/sys/mman.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_mman.yaml
rename to libc/newhdrgen/yaml/sys/mman.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_prctl.yaml b/libc/newhdrgen/yaml/sys/prctl.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_prctl.yaml
rename to libc/newhdrgen/yaml/sys/prctl.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_random.yaml b/libc/newhdrgen/yaml/sys/random.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_random.yaml
rename to libc/newhdrgen/yaml/sys/random.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_resource.yaml b/libc/newhdrgen/yaml/sys/resource.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_resource.yaml
rename to libc/newhdrgen/yaml/sys/resource.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_select.yaml b/libc/newhdrgen/yaml/sys/select.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_select.yaml
rename to libc/newhdrgen/yaml/sys/select.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_sendfile.yaml b/libc/newhdrgen/yaml/sys/sendfile.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_sendfile.yaml
rename to libc/newhdrgen/yaml/sys/sendfile.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_socket.yaml b/libc/newhdrgen/yaml/sys/socket.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_socket.yaml
rename to libc/newhdrgen/yaml/sys/socket.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_stat.yaml b/libc/newhdrgen/yaml/sys/stat.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_stat.yaml
rename to libc/newhdrgen/yaml/sys/stat.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_statvfs.yaml b/libc/newhdrgen/yaml/sys/statvfs.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_statvfs.yaml
rename to libc/newhdrgen/yaml/sys/statvfs.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_syscall.yaml b/libc/newhdrgen/yaml/sys/syscall.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_syscall.yaml
rename to libc/newhdrgen/yaml/sys/syscall.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_time.yaml b/libc/newhdrgen/yaml/sys/time.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_time.yaml
rename to libc/newhdrgen/yaml/sys/time.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_types.yaml b/libc/newhdrgen/yaml/sys/types.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_types.yaml
rename to libc/newhdrgen/yaml/sys/types.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_utsname.yaml b/libc/newhdrgen/yaml/sys/utsname.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_utsname.yaml
rename to libc/newhdrgen/yaml/sys/utsname.yaml
diff --git a/libc/newhdrgen/yaml/sys/sys_wait.yaml b/libc/newhdrgen/yaml/sys/wait.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sys_wait.yaml
rename to libc/newhdrgen/yaml/sys/wait.yaml

From 202785708ed0ff11770f8ae8603514d2c9a54bd9 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 18 Jul 2024 15:35:25 -0700
Subject: [PATCH 097/486] [flang][runtime] Clear leftTabLimit at in
 FinishReadingRecord (#98822)

ExternalFileUnit::FinishReadingRecord() is called at the end of a READ
statement, unless the read is non-advancing and there was no error. In
the event of an erroneous non-advancing read, however,
FinishReadingRecord() leaves ConnectionState::leftTabLimit inhabited by
a value, leaving the impression that the *next* record is undergoing a
non-advancing operation, and this cause the next operation to fail. In
the test case in the reported bug, the next operation is a BACKSPACE,
and it ends up doing nothing. The fix is to always reset leftTabLimit in
FinishReadingRecord.

Fixes https://github.com/llvm/llvm-project/issues/98783.
---
 flang/runtime/unit.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp
index a11f444d8d754..5c5bca835f3d8 100644
--- a/flang/runtime/unit.cpp
+++ b/flang/runtime/unit.cpp
@@ -268,6 +268,7 @@ void ExternalFileUnit::FinishReadingRecord(IoErrorHandler &handler) {
     recordOffsetInFrame_ = 0;
   }
   BeginRecord();
+  leftTabLimit.reset();
 }
 
 bool ExternalFileUnit::AdvanceRecord(IoErrorHandler &handler) {

From ef94732b4fa0512a3a635766b872c1574a1aade4 Mon Sep 17 00:00:00 2001
From: Connie <60797237+connieyzhu@users.noreply.github.com>
Date: Thu, 18 Jul 2024 15:54:58 -0700
Subject: [PATCH 098/486] [llvm-lit] Resolve TypeError in built-in cat -v
 implementation (#98363)

When using -v in lit's internal implementation of cat, there is a
TypeError when the file data is passed into convertToCaretAndMNotation()
as a str, because bytearray() requires an encoded string. This patch
encodes the str before passing it through bytearray().
---
 llvm/utils/lit/lit/builtin_commands/cat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/lit/lit/builtin_commands/cat.py b/llvm/utils/lit/lit/builtin_commands/cat.py
index 6fb2152ef9332..d6dbe889cc2b1 100644
--- a/llvm/utils/lit/lit/builtin_commands/cat.py
+++ b/llvm/utils/lit/lit/builtin_commands/cat.py
@@ -10,7 +10,7 @@
 def convertToCaretAndMNotation(data):
     newdata = StringIO()
     if isinstance(data, str):
-        data = bytearray(data)
+        data = bytearray(data.encode())
 
     for intval in data:
         if intval == 9 or intval == 10:

From 6c09a9bf6c6247299b53833602e6bb312dfda555 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 18 Jul 2024 15:57:32 -0700
Subject: [PATCH 099/486] [flang] Check assignment conformance for derived
 types (#99059)

Derived type assignment checking needs to account for the possibility of
derived assignment. The implementation was checking compile-time
conformance errors only on the path for assignments of intrinsic types.
Add a static array conformance check in the derived type flow once it
has been established that no defined assignment exists.

Fixes https://github.com/llvm/llvm-project/issues/98981.
---
 flang/lib/Semantics/expression.cpp           | 2 ++
 flang/test/Semantics/array-constr-values.f90 | 2 ++
 flang/test/Semantics/assign10.f90            | 6 ++++++
 3 files changed, 10 insertions(+)

diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 2d91962a24a4c..4413d77fcf255 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -4533,6 +4533,8 @@ std::optional<ProcedureRef> ArgumentAnalyzer::TryDefinedAssignment() {
         !OkLogicalIntegerAssignment(lhsType->category(), rhsType->category())) {
       SayNoMatch("ASSIGNMENT(=)", true);
     }
+  } else if (!fatalErrors_) {
+    CheckAssignmentConformance();
   }
   return std::nullopt;
 }
diff --git a/flang/test/Semantics/array-constr-values.f90 b/flang/test/Semantics/array-constr-values.f90
index b93f774e0b0d0..6742cd35fe52d 100644
--- a/flang/test/Semantics/array-constr-values.f90
+++ b/flang/test/Semantics/array-constr-values.f90
@@ -35,8 +35,10 @@ subroutine arrayconstructorvalues()
   intarray = [integer:: EMPLOYEE (19, "Jack"), 2, 3, 4, 5]
 
   ! C7112
+  !ERROR: Dimension 1 of left-hand side has extent 3, but right-hand side has extent 2
   !ERROR: Value in array constructor of type 'INTEGER(4)' could not be converted to the type of the array 'employee'
   emparray = (/ EMPLOYEE:: EMPLOYEE(19, "Ganesh"), EMPLOYEE(22, "Omkar"), 19 /)
+  !ERROR: Dimension 1 of left-hand side has extent 3, but right-hand side has extent 2
   !ERROR: Value in array constructor of type 'employeer' could not be converted to the type of the array 'employee'
   emparray = (/ EMPLOYEE:: EMPLOYEE(19, "Ganesh"), EMPLOYEE(22, "Ram"),EMPLOYEER("ShriniwasPvtLtd") /)
 
diff --git a/flang/test/Semantics/assign10.f90 b/flang/test/Semantics/assign10.f90
index 6b98097bfe62c..f0ea4b251a6d6 100644
--- a/flang/test/Semantics/assign10.f90
+++ b/flang/test/Semantics/assign10.f90
@@ -1,7 +1,11 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 ! Shape conformance checks on assignments
 program test
+  type t
+    integer n
+  end type
   real :: a0, a1a(2), a1b(3), a2a(2,3), a2b(3,2)
+  type(t) c(10)
   a0 = 0. ! ok
   !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches scalar REAL(4) and rank 1 array of REAL(4)
   a0 = [0.]
@@ -20,4 +24,6 @@ program test
   a2a(1,:) = a1a
   !ERROR: Dimension 1 of left-hand side has extent 2, but right-hand side has extent 3
   a2a = a2b
+  !ERROR: Dimension 1 of left-hand side has extent 10, but right-hand side has extent 0
+  c(1:10) = c(10:1)
 end

From 0004ca670a960d3573137679b3c67a7b3932530a Mon Sep 17 00:00:00 2001
From: RoseZhang03 <rosezhang@google.com>
Date: Thu, 18 Jul 2024 22:58:34 +0000
Subject: [PATCH 100/486] [libc] Removed __LIBC_CONST_ATTR attribute and
 updated math.yaml with the new math functions (#99571)

- deleted attribute from fabs function
- added math functions from #98972
- also updated statvfs files so that it can be generated
---
 libc/config/linux/api.td                  |  2 +-
 libc/config/linux/x86_64/headers.txt      |  3 +-
 libc/newhdrgen/yaml/math.yaml             | 64 ++++++++++++++++++++++-
 libc/newhdrgen/yaml/sys/statvfs.yaml      |  2 +
 libc/src/sys/statvfs/linux/CMakeLists.txt |  6 +--
 5 files changed, 69 insertions(+), 8 deletions(-)

diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index a10dec06e2452..320f3e92183bf 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -267,5 +267,5 @@ def SearchAPI : PublicAPI<"search.h"> {
 }
 
 def SysStatvfsAPI : PublicAPI<"sys/statvfs.h"> {
-  let Types = ["fsblkcnt_t", "fsfilcnt_t", "struct statvfs"];
+  let Types = ["struct statvfs"];
 }
diff --git a/libc/config/linux/x86_64/headers.txt b/libc/config/linux/x86_64/headers.txt
index 8a52d80e1fbfb..0294f62bc2f7a 100644
--- a/libc/config/linux/x86_64/headers.txt
+++ b/libc/config/linux/x86_64/headers.txt
@@ -45,8 +45,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.sys_select
     libc.include.sys_socket
     libc.include.sys_stat
-    # statvfs is broken, will uncomment once it's fixed.
-    # libc.include.sys_statvfs
+    libc.include.sys_statvfs
     libc.include.sys_syscall
     libc.include.sys_time
     libc.include.sys_types
diff --git a/libc/newhdrgen/yaml/math.yaml b/libc/newhdrgen/yaml/math.yaml
index 8588389bca4d2..ce562c653a6d2 100644
--- a/libc/newhdrgen/yaml/math.yaml
+++ b/libc/newhdrgen/yaml/math.yaml
@@ -64,8 +64,6 @@ functions:
     return_type: double
     arguments:
       - type: double
-    attributes:
-      - __LIBC_CONST_ATTR
   - name: fabsf
     standards: 
       - stdc
@@ -364,6 +362,20 @@ functions:
     arguments:
       - type: double
       - type: double
+  - name: fmull
+    standards:
+      - stdc
+    return_type: float
+    arguments:
+      - type: long double
+      - type: long double
+  - name: dmull
+    standards:
+      - stdc
+    return_type: double
+    arguments:
+      - type: long double
+      - type: long double
   - name: frexp
     standards: 
       - stdc
@@ -1323,6 +1335,30 @@ functions:
       - type: long double
       - type: long double
     guard: LIBC_TYPES_HAS_FLOAT16
+  - name: f16mul
+    standards:
+      - llvm_libc_ext
+    return_type: _Float16
+    arguments:
+      - type: double
+      - type: double
+    guard: LIBC_TYPES_HAS_FLOAT16
+  - name: f16mulf
+    standards:
+      - llvm_libc_ext
+    return_type: _Float16
+    arguments:
+      - type: float
+      - type: float
+    guard: LIBC_TYPES_HAS_FLOAT16
+  - name: f16mull
+    standards:
+      - llvm_libc_ext
+    return_type: _Float16
+    arguments:
+      - type: long double
+      - type: long double
+    guard: LIBC_TYPES_HAS_FLOAT16
   - name: f16sqrt
     standards: 
       - llvm_libc_ext
@@ -1756,6 +1792,14 @@ functions:
       - type: float128
       - type: float128
     guard: LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128
+  - name: f16mulf128
+    standards:
+      - stdc
+    return_type: _Float16
+    arguments:
+      - type: float128
+      - type: float128
+    guard: LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128
   - name: f16sqrtf128
     standards: 
       - llvm_libc_ext
@@ -1896,6 +1940,22 @@ functions:
       - type: float128
       - type: float128
     guard: LIBC_TYPES_HAS_FLOAT128
+  - name: fmulf128
+    standards:
+      - llvm_libc_ext
+    return_type: float
+    arguments:
+      - type: float128
+      - type: float128
+    guard: LIBC_TYPES_HAS_FLOAT128
+  - name: dmulf128
+    standards:
+      - llvm_libc_ext
+    return_type: double
+    arguments:
+      - type: float128
+      - type: float128
+    guard: LIBC_TYPES_HAS_FLOAT128
   - name: frexpf128
     standards: 
       - stdc
diff --git a/libc/newhdrgen/yaml/sys/statvfs.yaml b/libc/newhdrgen/yaml/sys/statvfs.yaml
index 3651901e9f2c7..53666770a5de6 100644
--- a/libc/newhdrgen/yaml/sys/statvfs.yaml
+++ b/libc/newhdrgen/yaml/sys/statvfs.yaml
@@ -2,6 +2,8 @@ header: sys-statvfs.h
 macros: []
 types:
   - type_name: struct_statvfs
+  - type_name: fsblkcnt_t
+  - type_name: fsfilcnt_t
 enums: []
 objects: []
 functions:
diff --git a/libc/src/sys/statvfs/linux/CMakeLists.txt b/libc/src/sys/statvfs/linux/CMakeLists.txt
index a6660c02badf7..2953717e70392 100644
--- a/libc/src/sys/statvfs/linux/CMakeLists.txt
+++ b/libc/src/sys/statvfs/linux/CMakeLists.txt
@@ -8,7 +8,7 @@ add_header_library(
     libc.src.__support.common
     libc.src.__support.CPP.optional
     libc.include.sys_syscall
-    libc.include.llvm-libc-types.struct_statvfs
+    libc.include.sys_statvfs
 )
 
 add_entrypoint_object(
@@ -19,7 +19,7 @@ add_entrypoint_object(
     ../statvfs.h
   DEPENDS
     libc.src.__support.libc_assert
-    libc.include.llvm-libc-types.struct_statvfs
+    libc.include.sys_statvfs
     .statfs_utils
 )
 
@@ -31,7 +31,7 @@ add_entrypoint_object(
     ../fstatvfs.h
   DEPENDS
     libc.src.__support.libc_assert
-    libc.include.llvm-libc-types.struct_statvfs
+    libc.include.sys_statvfs
     .statfs_utils
 )
 

From 40ed6ba016ea59106e18e610cc17df43e0192ee4 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 18 Jul 2024 15:59:24 -0700
Subject: [PATCH 101/486] [flang][NFC] Document an intentional violation of the
 standard (#99073)

The Fortran standard committees passed an "interp" request at their June
2024 meetings that is distinct from nearly every other Fortran compiler
that I tried (6) in an an ambiguous case (parent component naming when
the base type has been renamed via USE association). Document this case
in flang/docs/Extensions.md as an intentional instance of
non-conformance chosen for portability and better usability.
---
 flang/docs/Extensions.md                  |  7 +++
 flang/test/Semantics/parent-comp-name.f90 | 70 +++++++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 flang/test/Semantics/parent-comp-name.f90

diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 82f9a021c14ee..093596c9dc8eb 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -134,6 +134,13 @@ end
   implicitly simply appearing in an asynchronous data transfer statement,
   without the attribute being visible in the procedure's explicit
   interface.
+* When the name of an extended derived type's base type is the
+  result of `USE` association with renaming, the name of the extended
+  derived type's parent component is the new name by which the base
+  is known in the scope of the extended derived type, not the original.
+  This interpretation has usability advantages and is what six other
+  Fortran compilers do, but is not conforming now that J3 approved an
+  "interp" in June 2024 to the contrary.
 
 ## Extensions, deletions, and legacy features supported by default
 
diff --git a/flang/test/Semantics/parent-comp-name.f90 b/flang/test/Semantics/parent-comp-name.f90
new file mode 100644
index 0000000000000..d7878430a5849
--- /dev/null
+++ b/flang/test/Semantics/parent-comp-name.f90
@@ -0,0 +1,70 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+! Every other Fortran compiler (but one) interprets the names of parent
+! components like this when the names of their types are the product of
+! USE association with renaming.
+
+module m1
+  type originalName
+    integer m
+  end type
+end
+
+module m2
+  use m1, newName => originalName
+  type, extends(newName) :: extended
+    integer n
+  end type
+  type, extends(newName) :: extended2
+    integer originalName ! ok
+  end type
+ contains
+  subroutine s1
+    type(extended) x
+    type(extended2) x2
+    print *, x%newName%m ! ok
+    !ERROR: Component 'originalname' not found in derived type 'extended'
+    print *, x%originalName
+    print *, extended(newName=newName(m=1),n=2) ! ok
+    !ERROR: Structure constructor lacks a value for component 'm'
+    !ERROR: Keyword 'originalname=' does not name a component of derived type 'extended'
+    !ERROR: Keyword 'm=' may not appear in a reference to a procedure with an implicit interface
+    print *, extended(originalName=originalName(m=1),n=2)
+    !ERROR: Value in structure constructor of type 'REAL(4)' is incompatible with component 'newname' of type 'newname'
+    !ERROR: Keyword 'm=' may not appear in a reference to a procedure with an implicit interface
+    print *, extended(newName=originalName(m=1),n=2)
+    !ERROR: Structure constructor lacks a value for component 'm'
+    !ERROR: Keyword 'originalname=' does not name a component of derived type 'extended'
+    print *, extended(originalName=newName(m=1),n=2)
+    print *, x2%newName%m ! ok
+    print *, x2%originalName ! ok
+    print *, extended2(newName=newName(m=1),originalName=2) ! ok
+  end
+end
+
+module m3
+  use m2
+ contains
+  ! Same as above, but not in the same module as the derived
+  ! types' definitions.
+  subroutine s2
+    type(extended) x
+    type(extended2) x2
+    print *, x%newName%m ! ok
+    !ERROR: Component 'originalname' not found in derived type 'extended'
+    print *, x%originalName
+    print *, extended(newName=newName(m=1),n=2) ! ok
+    !ERROR: Structure constructor lacks a value for component 'm'
+    !ERROR: Keyword 'originalname=' does not name a component of derived type 'extended'
+    !ERROR: Keyword 'm=' may not appear in a reference to a procedure with an implicit interface
+    print *, extended(originalName=originalName(m=1),n=2)
+    !ERROR: Value in structure constructor of type 'REAL(4)' is incompatible with component 'newname' of type 'newname'
+    !ERROR: Keyword 'm=' may not appear in a reference to a procedure with an implicit interface
+    print *, extended(newName=originalName(m=1),n=2)
+    !ERROR: Structure constructor lacks a value for component 'm'
+    !ERROR: Keyword 'originalname=' does not name a component of derived type 'extended'
+    print *, extended(originalName=newName(m=1),n=2)
+    print *, x2%newName%m ! ok
+    print *, x2%originalName ! ok
+    print *, extended2(newName=newName(m=1),originalName=2) ! ok
+  end
+end

From adacb5010f5ca6e923b3cf2d8ea47cbaab96099d Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj@google.com>
Date: Thu, 18 Jul 2024 16:00:29 -0700
Subject: [PATCH 102/486] [libc] Restore DECLS_FILE_PATH property (#99583)

The DECLS_FILE_PATH property is supposed to be set on the targets for
the generated headers for the GPU build installation. It got missed when
creating the cmake rule for new headergen.
---
 libc/cmake/modules/LLVMLibCHeaderRules.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libc/cmake/modules/LLVMLibCHeaderRules.cmake b/libc/cmake/modules/LLVMLibCHeaderRules.cmake
index 91054810f5ec5..3049f4db7301f 100644
--- a/libc/cmake/modules/LLVMLibCHeaderRules.cmake
+++ b/libc/cmake/modules/LLVMLibCHeaderRules.cmake
@@ -160,6 +160,7 @@ function(add_gen_header2 target_name)
     ${fq_target_name}
     PROPERTIES
       HEADER_FILE_PATH ${out_file}
+      DECLS_FILE_PATH "${decl_out_file}"
       DEPS "${fq_deps_list}"
   )
 

From c675a9be63b67682477e5cbdc01c450f66bbc59a Mon Sep 17 00:00:00 2001
From: pcc <peter@pcc.me.uk>
Date: Thu, 18 Jul 2024 16:05:53 -0700
Subject: [PATCH 103/486] Object: Don't error out on malformed bitcode files.

An error reading a bitcode file most likely indicates that the file
was created by a compiler from the future. Normally we don't try to
implement forwards compatibility for bitcode files, but when creating
an archive we can implement best-effort forwards compatibility by
treating the file as a blob and not creating symbol index entries for
it. lld and mold ignore the archive symbol index, so provided that
you use one of these linkers, LTO will work as long as lld or the
gold plugin is newer than the compiler. We only ignore errors if the
archive format is one that is supported by a linker that is known to
ignore the index, otherwise there's no chance of this working so we
may as well error out. We print a warning on read failure so that
users of linkers that rely on the symbol index can diagnose the issue.

This is the same behavior as GNU ar when the linker plugin returns
an error when reading the input file. If the bitcode file is actually
malformed, it will be diagnosed at link time.

Reviewers: MaskRay, dwblaikie, jh7370

Reviewed By: MaskRay, dwblaikie, jh7370

Pull Request: https://github.com/llvm/llvm-project/pull/96848
---
 llvm/include/llvm/Object/ArchiveWriter.h      | 11 +++-
 llvm/lib/Object/ArchiveWriter.cpp             | 64 +++++++++++++++----
 .../test/Object/archive-malformed-object.test | 44 +++++++++++--
 3 files changed, 96 insertions(+), 23 deletions(-)

diff --git a/llvm/include/llvm/Object/ArchiveWriter.h b/llvm/include/llvm/Object/ArchiveWriter.h
index e41b3d51173d4..95b81f5f64c8f 100644
--- a/llvm/include/llvm/Object/ArchiveWriter.h
+++ b/llvm/include/llvm/Object/ArchiveWriter.h
@@ -48,25 +48,30 @@ enum class SymtabWritingMode {
   BigArchive64  // Only write the 64-bit symbol table.
 };
 
+void warnToStderr(Error Err);
+
 // Write an archive directly to an output stream.
 Error writeArchiveToStream(raw_ostream &Out,
                            ArrayRef<NewArchiveMember> NewMembers,
                            SymtabWritingMode WriteSymtab,
                            object::Archive::Kind Kind, bool Deterministic,
-                           bool Thin, std::optional<bool> IsEC = std::nullopt);
+                           bool Thin, std::optional<bool> IsEC = std::nullopt,
+                           function_ref<void(Error)> Warn = warnToStderr);
 
 Error writeArchive(StringRef ArcName, ArrayRef<NewArchiveMember> NewMembers,
                    SymtabWritingMode WriteSymtab, object::Archive::Kind Kind,
                    bool Deterministic, bool Thin,
                    std::unique_ptr<MemoryBuffer> OldArchiveBuf = nullptr,
-                   std::optional<bool> IsEC = std::nullopt);
+                   std::optional<bool> IsEC = std::nullopt,
+                   function_ref<void(Error)> Warn = warnToStderr);
 
 // writeArchiveToBuffer is similar to writeArchive but returns the Archive in a
 // buffer instead of writing it out to a file.
 Expected<std::unique_ptr<MemoryBuffer>>
 writeArchiveToBuffer(ArrayRef<NewArchiveMember> NewMembers,
                      SymtabWritingMode WriteSymtab, object::Archive::Kind Kind,
-                     bool Deterministic, bool Thin);
+                     bool Deterministic, bool Thin,
+                     function_ref<void(Error)> Warn = warnToStderr);
 }
 
 #endif
diff --git a/llvm/lib/Object/ArchiveWriter.cpp b/llvm/lib/Object/ArchiveWriter.cpp
index 34f12cf0111cf..114045561366d 100644
--- a/llvm/lib/Object/ArchiveWriter.cpp
+++ b/llvm/lib/Object/ArchiveWriter.cpp
@@ -482,7 +482,8 @@ static uint64_t computeHeadersSize(object::Archive::Kind Kind,
 }
 
 static Expected<std::unique_ptr<SymbolicFile>>
-getSymbolicFile(MemoryBufferRef Buf, LLVMContext &Context) {
+getSymbolicFile(MemoryBufferRef Buf, LLVMContext &Context,
+                object::Archive::Kind Kind, function_ref<void(Error)> Warn) {
   const file_magic Type = identify_magic(Buf.getBuffer());
   // Don't attempt to read non-symbolic file types.
   if (!object::SymbolicFile::isSymbolicFile(Type, &Context))
@@ -490,8 +491,36 @@ getSymbolicFile(MemoryBufferRef Buf, LLVMContext &Context) {
   if (Type == file_magic::bitcode) {
     auto ObjOrErr = object::SymbolicFile::createSymbolicFile(
         Buf, file_magic::bitcode, &Context);
-    if (!ObjOrErr)
-      return ObjOrErr.takeError();
+    // An error reading a bitcode file most likely indicates that the file
+    // was created by a compiler from the future. Normally we don't try to
+    // implement forwards compatibility for bitcode files, but when creating an
+    // archive we can implement best-effort forwards compatibility by treating
+    // the file as a blob and not creating symbol index entries for it. lld and
+    // mold ignore the archive symbol index, so provided that you use one of
+    // these linkers, LTO will work as long as lld or the gold plugin is newer
+    // than the compiler. We only ignore errors if the archive format is one
+    // that is supported by a linker that is known to ignore the index,
+    // otherwise there's no chance of this working so we may as well error out.
+    // We print a warning on read failure so that users of linkers that rely on
+    // the symbol index can diagnose the issue.
+    //
+    // This is the same behavior as GNU ar when the linker plugin returns an
+    // error when reading the input file. If the bitcode file is actually
+    // malformed, it will be diagnosed at link time.
+    if (!ObjOrErr) {
+      switch (Kind) {
+      case object::Archive::K_BSD:
+      case object::Archive::K_GNU:
+      case object::Archive::K_GNU64:
+        Warn(ObjOrErr.takeError());
+        return nullptr;
+      case object::Archive::K_AIXBIG:
+      case object::Archive::K_COFF:
+      case object::Archive::K_DARWIN:
+      case object::Archive::K_DARWIN64:
+        return ObjOrErr.takeError();
+      }
+    }
     return std::move(*ObjOrErr);
   } else {
     auto ObjOrErr = object::SymbolicFile::createSymbolicFile(Buf);
@@ -751,7 +780,7 @@ computeMemberData(raw_ostream &StringTable, raw_ostream &SymNames,
                   object::Archive::Kind Kind, bool Thin, bool Deterministic,
                   SymtabWritingMode NeedSymbols, SymMap *SymMap,
                   LLVMContext &Context, ArrayRef<NewArchiveMember> NewMembers,
-                  std::optional<bool> IsEC) {
+                  std::optional<bool> IsEC, function_ref<void(Error)> Warn) {
   static char PaddingData[8] = {'\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'};
   uint64_t MemHeadPadSize = 0;
   uint64_t Pos =
@@ -819,8 +848,10 @@ computeMemberData(raw_ostream &StringTable, raw_ostream &SymNames,
 
   if (NeedSymbols != SymtabWritingMode::NoSymtab || isAIXBigArchive(Kind)) {
     for (const NewArchiveMember &M : NewMembers) {
-      Expected<std::unique_ptr<SymbolicFile>> SymFileOrErr =
-          getSymbolicFile(M.Buf->getMemBufferRef(), Context);
+      Expected<std::unique_ptr<SymbolicFile>> SymFileOrErr = getSymbolicFile(
+          M.Buf->getMemBufferRef(), Context, Kind, [&](Error Err) {
+            Warn(createFileError(M.MemberName, std::move(Err)));
+          });
       if (!SymFileOrErr)
         return createFileError(M.MemberName, SymFileOrErr.takeError());
       SymFiles.push_back(std::move(*SymFileOrErr));
@@ -1001,7 +1032,8 @@ Error writeArchiveToStream(raw_ostream &Out,
                            ArrayRef<NewArchiveMember> NewMembers,
                            SymtabWritingMode WriteSymtab,
                            object::Archive::Kind Kind, bool Deterministic,
-                           bool Thin, std::optional<bool> IsEC) {
+                           bool Thin, std::optional<bool> IsEC,
+                           function_ref<void(Error)> Warn) {
   assert((!Thin || !isBSDLike(Kind)) && "Only the gnu format has a thin mode");
 
   SmallString<0> SymNamesBuf;
@@ -1023,7 +1055,7 @@ Error writeArchiveToStream(raw_ostream &Out,
 
   Expected<std::vector<MemberData>> DataOrErr = computeMemberData(
       StringTable, SymNames, Kind, Thin, Deterministic, WriteSymtab,
-      isCOFFArchive(Kind) ? &SymMap : nullptr, Context, NewMembers, IsEC);
+      isCOFFArchive(Kind) ? &SymMap : nullptr, Context, NewMembers, IsEC, Warn);
   if (Error E = DataOrErr.takeError())
     return E;
   std::vector<MemberData> &Data = *DataOrErr;
@@ -1266,11 +1298,15 @@ Error writeArchiveToStream(raw_ostream &Out,
   return Error::success();
 }
 
+void warnToStderr(Error Err) {
+  llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "warning: ");
+}
+
 Error writeArchive(StringRef ArcName, ArrayRef<NewArchiveMember> NewMembers,
                    SymtabWritingMode WriteSymtab, object::Archive::Kind Kind,
                    bool Deterministic, bool Thin,
                    std::unique_ptr<MemoryBuffer> OldArchiveBuf,
-                   std::optional<bool> IsEC) {
+                   std::optional<bool> IsEC, function_ref<void(Error)> Warn) {
   Expected<sys::fs::TempFile> Temp =
       sys::fs::TempFile::create(ArcName + ".temp-archive-%%%%%%%.a");
   if (!Temp)
@@ -1278,7 +1314,7 @@ Error writeArchive(StringRef ArcName, ArrayRef<NewArchiveMember> NewMembers,
   raw_fd_ostream Out(Temp->FD, false);
 
   if (Error E = writeArchiveToStream(Out, NewMembers, WriteSymtab, Kind,
-                                     Deterministic, Thin, IsEC)) {
+                                     Deterministic, Thin, IsEC, Warn)) {
     if (Error DiscardError = Temp->discard())
       return joinErrors(std::move(E), std::move(DiscardError));
     return E;
@@ -1302,12 +1338,14 @@ Error writeArchive(StringRef ArcName, ArrayRef<NewArchiveMember> NewMembers,
 Expected<std::unique_ptr<MemoryBuffer>>
 writeArchiveToBuffer(ArrayRef<NewArchiveMember> NewMembers,
                      SymtabWritingMode WriteSymtab, object::Archive::Kind Kind,
-                     bool Deterministic, bool Thin) {
+                     bool Deterministic, bool Thin,
+                     function_ref<void(Error)> Warn) {
   SmallVector<char, 0> ArchiveBufferVector;
   raw_svector_ostream ArchiveStream(ArchiveBufferVector);
 
-  if (Error E = writeArchiveToStream(ArchiveStream, NewMembers, WriteSymtab,
-                                     Kind, Deterministic, Thin, std::nullopt))
+  if (Error E =
+          writeArchiveToStream(ArchiveStream, NewMembers, WriteSymtab, Kind,
+                               Deterministic, Thin, std::nullopt, Warn))
     return std::move(E);
 
   return std::make_unique<SmallVectorMemoryBuffer>(
diff --git a/llvm/test/Object/archive-malformed-object.test b/llvm/test/Object/archive-malformed-object.test
index a92762975bda6..021df7ed873d1 100644
--- a/llvm/test/Object/archive-malformed-object.test
+++ b/llvm/test/Object/archive-malformed-object.test
@@ -1,27 +1,51 @@
 ## Show that the archive library emits error messages when adding malformed
-## objects.
+## object files and skips symbol tables for "malformed" bitcode files, which
+## are assumed to be bitcode files generated by compilers from the future.
 
 # RUN: rm -rf %t.dir
 # RUN: split-file %s %t.dir
 # RUN: cd %t.dir
 
-## Malformed bitcode object is the first file member of archive if the symbol table is required.
+## Create a malformed bitcode object.
 # RUN: llvm-as input.ll -o input.bc
 # RUN: cp input.bc good.bc
 # RUN: %python -c "with open('input.bc', 'a') as f: f.truncate(10)"
-# RUN: not llvm-ar rc bad.a input.bc 2>&1 | FileCheck %s --check-prefix=ERR1
 
-## Malformed bitcode object is the last file member of archive if the symbol table is required.
+## Malformed bitcode objects either warn or error depending on the archive format
+## (see switch in getSymbolicFile). If the archive was created with a warning,
+## we want to check that the archive map is empty. llvm-nm will fail when it
+## tries to read the malformed bitcode file, but it's supposed to print the
+## archive map first, which in this case it won't because there won't be one.
 # RUN: rm -rf bad.a
-# RUN: not llvm-ar rc bad.a good.bc input.bc 2>&1 | FileCheck %s --check-prefix=ERR1
+# RUN: llvm-ar --format=bsd rc bad.a input.bc 2>&1 | FileCheck %s --check-prefix=WARN1
+# RUN: not llvm-nm --print-armap bad.a | count 0
+# RUN: rm -rf bad.a
+# RUN: llvm-ar --format=gnu rc bad.a input.bc 2>&1 | FileCheck %s --check-prefix=WARN1
+# RUN: not llvm-nm --print-armap bad.a | count 0
+# RUN: rm -rf bad.a
+# RUN: not llvm-ar --format=bigarchive rc bad.a input.bc 2>&1 | FileCheck %s --check-prefix=ERR1
+# RUN: rm -rf bad.a
+# RUN: not llvm-ar --format=coff rc bad.a input.bc 2>&1 | FileCheck %s --check-prefix=ERR1
+# RUN: rm -rf bad.a
+# RUN: not llvm-ar --format=darwin rc bad.a input.bc 2>&1 | FileCheck %s --check-prefix=ERR1
+
+## Malformed bitcode object is the last file member of archive and
+## the symbol table is required. In this case we check that the
+## symbol table contains entries for the good object only.
+# RUN: rm -rf bad.a
+# RUN: llvm-ar rc bad.a good.bc input.bc 2>&1 | FileCheck %s --check-prefix=WARN1
+# RUN: not llvm-nm --print-armap bad.a | FileCheck %s --check-prefix=ARMAP
 
 ## Malformed bitcode object if the symbol table is not required for big archive.
+## For big archives we print an error instead of a warning because the AIX linker
+## presumably requires the index.
 # RUN: rm -rf bad.a
 # RUN: not llvm-ar --format=bigarchive rcS bad.a input.bc 2>&1 | FileCheck %s --check-prefix=ERR1
 # RUN: rm -rf bad.a
 # RUN: not llvm-ar --format=bigarchive rcS bad.a good.bc input.bc 2>&1 | FileCheck %s --check-prefix=ERR1
 
 # ERR1: error: bad.a: 'input.bc': Invalid bitcode signature
+# WARN1: warning: 'input.bc': Invalid bitcode signature
 
 ## Non-bitcode malformed file.
 # RUN: yaml2obj input.yaml -o input.o
@@ -29,17 +53,23 @@
 
 # ERR2: error: bad.a: 'input.o': section header table goes past the end of the file: e_shoff = 0x9999
 
-## Don't emit an error if the symbol table is not required for formats other than the big archive format.
-# RUN: llvm-ar --format=gnu rcS good.a input.o input.bc
+## Don't emit an error or warning if the symbol table is not required for formats other than the big archive format.
+# RUN: llvm-ar --format=gnu rcS good.a input.o input.bc 2>&1 | count 0
 # RUN: llvm-ar t good.a | FileCheck %s --check-prefix=CONTENTS
 
 # CONTENTS:      input.o
 # CONTENTS-NEXT: input.bc
 
+# ARMAP: Archive map
+# ARMAP-NEXT: foo in good.bc
+# ARMAP-EMPTY:
+
 #--- input.ll
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux"
 
+@foo = global i32 1
+
 #--- input.yaml
 --- !ELF
 FileHeader:

From 433e09cf257526a24afc98326bd60fbf09923359 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Thu, 18 Jul 2024 16:08:20 -0700
Subject: [PATCH 104/486] [bazel] Add filegroups for MLIR bindings sources
 (#98396)

This can be useful if downstream projects configure their pybind
differently, similar to how local_config_python isn't defined here.
---
 .../llvm-project-overlay/mlir/BUILD.bazel     | 44 +++++++++++--------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index fe67286422f15..9ec5b793a1723 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -931,11 +931,14 @@ exports_files(
 # Some out-of-tree projects alias @python_runtime//:headers to
 # @local_config_python//:python_headers.
 
-MLIR_BINDINGS_PYTHON_HEADERS = [
-    "lib/Bindings/Python/*.h",
-    "include/mlir-c/Bindings/Python/*.h",
-    "include/mlir/Bindings/Python/*.h",
-]
+filegroup(
+    name = "MLIRBindingsPythonHeaderFiles",
+    srcs = glob([
+        "lib/Bindings/Python/*.h",
+        "include/mlir-c/Bindings/Python/*.h",
+        "include/mlir/Bindings/Python/*.h",
+    ]),
+)
 
 cc_library(
     name = "MLIRBindingsPythonHeaders",
@@ -947,7 +950,7 @@ cc_library(
         "manual",  # External dependency
         "nobuildkite",  # TODO(gcmn): Add support for this target
     ],
-    textual_hdrs = glob(MLIR_BINDINGS_PYTHON_HEADERS),
+    textual_hdrs = [":MLIRBindingsPythonHeaderFiles"],
     deps = [
         ":CAPIIRHeaders",
         ":CAPITransformsHeaders",
@@ -966,7 +969,7 @@ cc_library(
         "manual",  # External dependency
         "nobuildkite",  # TODO(gcmn): Add support for this target
     ],
-    textual_hdrs = glob(MLIR_BINDINGS_PYTHON_HEADERS),
+    textual_hdrs = [":MLIRBindingsPythonHeaderFiles"],
     deps = [
         ":CAPIIR",
         ":CAPITransforms",
@@ -986,20 +989,23 @@ PYBIND11_FEATURES = [
     "-use_header_modules",
 ]
 
-MLIR_PYTHON_BINDINGS_SOURCES = [
-    "lib/Bindings/Python/IRAffine.cpp",
-    "lib/Bindings/Python/IRAttributes.cpp",
-    "lib/Bindings/Python/IRCore.cpp",
-    "lib/Bindings/Python/IRInterfaces.cpp",
-    "lib/Bindings/Python/IRModule.cpp",
-    "lib/Bindings/Python/IRTypes.cpp",
-    "lib/Bindings/Python/Pass.cpp",
-    "lib/Bindings/Python/Rewrite.cpp",
-]
+filegroup(
+    name = "MLIRBindingsPythonSourceFiles",
+    srcs = [
+        "lib/Bindings/Python/IRAffine.cpp",
+        "lib/Bindings/Python/IRAttributes.cpp",
+        "lib/Bindings/Python/IRCore.cpp",
+        "lib/Bindings/Python/IRInterfaces.cpp",
+        "lib/Bindings/Python/IRModule.cpp",
+        "lib/Bindings/Python/IRTypes.cpp",
+        "lib/Bindings/Python/Pass.cpp",
+        "lib/Bindings/Python/Rewrite.cpp",
+    ],
+)
 
 cc_library(
     name = "MLIRBindingsPythonCore",
-    srcs = MLIR_PYTHON_BINDINGS_SOURCES,
+    srcs = [":MLIRBindingsPythonSourceFiles"],
     copts = PYBIND11_COPTS,
     features = PYBIND11_FEATURES,
     tags = [
@@ -1022,7 +1028,7 @@ cc_library(
 
 cc_library(
     name = "MLIRBindingsPythonCoreNoCAPI",
-    srcs = MLIR_PYTHON_BINDINGS_SOURCES,
+    srcs = [":MLIRBindingsPythonSourceFiles"],
     copts = PYBIND11_COPTS,
     features = PYBIND11_FEATURES,
     tags = [

From e73d51d3c8ea61fa34658f22147e65f95411eca2 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 18 Jul 2024 16:14:19 -0700
Subject: [PATCH 105/486] [flang] ASSOCIATE/SELECT TYPE entities aren't
 pointer/allocatable (#99364)

Fix what seems to be a regression in semantics in definability checking:
the construct entities of ASSOCIATE and SELECT TYPE constructs are never
pointers or allocatables, even when their selectors are so. SELECT RANK
construct entities, however, can be pointers or allocatables.
---
 flang/lib/Semantics/definable.cpp    |  5 +-
 flang/test/Semantics/associate03.f90 | 79 ++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Semantics/associate03.f90

diff --git a/flang/lib/Semantics/definable.cpp b/flang/lib/Semantics/definable.cpp
index 96af46abd6180..d594b1eca567f 100644
--- a/flang/lib/Semantics/definable.cpp
+++ b/flang/lib/Semantics/definable.cpp
@@ -178,7 +178,10 @@ static std::optional<parser::Message> WhyNotDefinableBase(parser::CharBlock at,
 static std::optional<parser::Message> WhyNotDefinableLast(parser::CharBlock at,
     const Scope &scope, DefinabilityFlags flags, const Symbol &original) {
   const Symbol &ultimate{original.GetUltimate()};
-  if (const auto *association{ultimate.detailsIf<AssocEntityDetails>()}) {
+  if (const auto *association{ultimate.detailsIf<AssocEntityDetails>()};
+      association &&
+      (association->rank().has_value() ||
+          !flags.test(DefinabilityFlag::PointerDefinition))) {
     if (auto dataRef{
             evaluate::ExtractDataRef(*association->expr(), true, true)}) {
       return WhyNotDefinableLast(at, scope, flags, dataRef->GetLastSymbol());
diff --git a/flang/test/Semantics/associate03.f90 b/flang/test/Semantics/associate03.f90
new file mode 100644
index 0000000000000..f57dc17839aab
--- /dev/null
+++ b/flang/test/Semantics/associate03.f90
@@ -0,0 +1,79 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
+! A construct entity does not have the POINTER or ALLOCATABLE attribute,
+! except in SELECT RANK.
+
+subroutine test(up,ua,rp,ra)
+  class(*), pointer :: up
+  class(*), allocatable :: ua
+  real, pointer :: rp(..)
+  real, allocatable :: ra(..)
+  real, target :: x
+  real, pointer :: p
+  real, allocatable :: a
+  associate (s => p)
+    !ERROR: The left-hand side of a pointer assignment is not definable
+    !BECAUSE: 's' is not a pointer
+    s => x
+    !ERROR: Entity in ALLOCATE statement must have the ALLOCATABLE or POINTER attribute
+    allocate(s)
+    !ERROR: Name in DEALLOCATE statement must have the ALLOCATABLE or POINTER attribute
+    deallocate(s)
+    !ERROR: 's' may not appear in NULLIFY
+    !BECAUSE: 's' is not a pointer
+    nullify(s)
+  end associate
+  select type(s => up)
+  type is (real)
+    !ERROR: The left-hand side of a pointer assignment is not definable
+    !BECAUSE: 's' is not a pointer
+    s => x
+    !ERROR: Entity in ALLOCATE statement must have the ALLOCATABLE or POINTER attribute
+    allocate(s)
+    !ERROR: Name in DEALLOCATE statement must have the ALLOCATABLE or POINTER attribute
+    deallocate(s)
+    !ERROR: 's' may not appear in NULLIFY
+    !BECAUSE: 's' is not a pointer
+    nullify(s)
+  end select
+  select rank(s => rp)
+  rank(0)
+    s => x ! ok
+    allocate(s) ! ok
+    deallocate(s) ! ok
+    nullify(s) ! ok
+  !ERROR: RANK (*) cannot be used when selector is POINTER or ALLOCATABLE
+  rank(*)
+  rank default
+    !ERROR: The left-hand side of a pointer assignment must not be an assumed-rank dummy argument
+    !ERROR: pointer 's' associated with object 'x' with incompatible type or shape
+    s => x
+    !ERROR: An assumed-rank dummy argument may not appear in an ALLOCATE statement
+    allocate(s)
+    deallocate(s) ! ok
+    nullify(s) ! ok
+  end select
+  associate (s => a)
+    !ERROR: Entity in ALLOCATE statement must have the ALLOCATABLE or POINTER attribute
+    allocate(s)
+    !ERROR: Name in DEALLOCATE statement must have the ALLOCATABLE or POINTER attribute
+    deallocate(s)
+  end associate
+  select type(s => ua)
+  type is (real)
+    !ERROR: Entity in ALLOCATE statement must have the ALLOCATABLE or POINTER attribute
+    allocate(s)
+    !ERROR: Name in DEALLOCATE statement must have the ALLOCATABLE or POINTER attribute
+    deallocate(s)
+  end select
+  select rank(s => ra)
+  rank(0)
+    allocate(s) ! ok
+    deallocate(s) ! ok
+  !ERROR: RANK (*) cannot be used when selector is POINTER or ALLOCATABLE
+  rank(*)
+  rank default
+    !ERROR: An assumed-rank dummy argument may not appear in an ALLOCATE statement
+    allocate(s)
+    deallocate(s) ! ok
+  end select
+end

From 043aca3990e3d47a25777f83569260aa4d822ae5 Mon Sep 17 00:00:00 2001
From: RoseZhang03 <rosezhang@google.com>
Date: Thu, 18 Jul 2024 23:19:09 +0000
Subject: [PATCH 106/486] [libc] newhdrgen: removed Include class (#99584)

Include class is not used (includes are made through Macro class, Type
class, and imports from .h.def files).
---
 .../class_implementation/classes/include.py     | 17 -----------------
 libc/newhdrgen/gpu_headers.py                   |  1 -
 libc/newhdrgen/header.py                        |  7 -------
 libc/newhdrgen/yaml_to_classes.py               |  4 ----
 4 files changed, 29 deletions(-)
 delete mode 100644 libc/newhdrgen/class_implementation/classes/include.py

diff --git a/libc/newhdrgen/class_implementation/classes/include.py b/libc/newhdrgen/class_implementation/classes/include.py
deleted file mode 100644
index 2657f2e7c931c..0000000000000
--- a/libc/newhdrgen/class_implementation/classes/include.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env python
-#
-# ====-- Include class for libc function headers --------------*- python -*--==#
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# ==-------------------------------------------------------------------------==#
-
-
-class Include:
-    def __init__(self, name):
-        self.name = name
-
-    def __str__(self):
-        return f'#include "{self.name}"'
diff --git a/libc/newhdrgen/gpu_headers.py b/libc/newhdrgen/gpu_headers.py
index cc13096cd47c1..d123e4af2de65 100644
--- a/libc/newhdrgen/gpu_headers.py
+++ b/libc/newhdrgen/gpu_headers.py
@@ -17,7 +17,6 @@ def __init__(self, name):
         self.enumerations = []
         self.objects = []
         self.functions = []
-        self.includes = []
 
     def add_macro(self, macro):
         self.macros.append(macro)
diff --git a/libc/newhdrgen/header.py b/libc/newhdrgen/header.py
index 141e3c9b2736b..7ec2dee596f8c 100644
--- a/libc/newhdrgen/header.py
+++ b/libc/newhdrgen/header.py
@@ -17,7 +17,6 @@ def __init__(self, name):
         self.enumerations = []
         self.objects = []
         self.functions = []
-        self.includes = []
 
     def add_macro(self, macro):
         self.macros.append(macro)
@@ -34,15 +33,9 @@ def add_object(self, object):
     def add_function(self, function):
         self.functions.append(function)
 
-    def add_include(self, include):
-        self.includes.append(include)
-
     def __str__(self):
         content = [""]
 
-        for include in self.includes:
-            content.append(str(include))
-
         for macro in self.macros:
             content.append(f"{macro}\n")
 
diff --git a/libc/newhdrgen/yaml_to_classes.py b/libc/newhdrgen/yaml_to_classes.py
index 205bb35fe691a..78588055cc960 100644
--- a/libc/newhdrgen/yaml_to_classes.py
+++ b/libc/newhdrgen/yaml_to_classes.py
@@ -16,7 +16,6 @@
 from class_implementation.classes.macro import Macro
 from class_implementation.classes.type import Type
 from class_implementation.classes.function import Function
-from class_implementation.classes.include import Include
 from class_implementation.classes.enumeration import Enumeration
 from class_implementation.classes.object import Object
 
@@ -103,9 +102,6 @@ def yaml_to_classes(yaml_data, header_class, entry_points=None):
             Object(object_data["object_name"], object_data["object_type"])
         )
 
-    for include_data in yaml_data.get("includes", []):
-        header.add_include(Include(include_data))
-
     return header
 
 

From 0684db30a1e3226a0bfe4c5d839d1355a66ea4ba Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 18 Jul 2024 16:32:24 -0700
Subject: [PATCH 107/486] [flang] A nested STRUCTURE must declare entities
 (#99379)

When a DEC legacy STRUCTURE definition appears within another, its
STRUCTURE statement must also declare some components of the enclosing
structure.

Fixes https://github.com/llvm/llvm-project/issues/99288.
---
 flang/lib/Parser/Fortran-parsers.cpp | 16 ++++++++++++----
 flang/test/Semantics/struct03.f90    |  7 +++++++
 2 files changed, 19 insertions(+), 4 deletions(-)
 create mode 100644 flang/test/Semantics/struct03.f90

diff --git a/flang/lib/Parser/Fortran-parsers.cpp b/flang/lib/Parser/Fortran-parsers.cpp
index 746d04ad649d1..0bdc4c4e033c7 100644
--- a/flang/lib/Parser/Fortran-parsers.cpp
+++ b/flang/lib/Parser/Fortran-parsers.cpp
@@ -1304,10 +1304,16 @@ TYPE_PARSER(extension<LanguageFeature::CUDA>(construct<CUDAAttributesStmt>(
     defaulted(
         maybe("::"_tok) >> nonemptyList("expected names"_err_en_US, name)))))
 
-// Subtle: the name includes the surrounding slashes, which avoids
+// Subtle: A structure's name includes the surrounding slashes, which avoids
 // clashes with other uses of the name in the same scope.
-TYPE_PARSER(construct<StructureStmt>(
-    "STRUCTURE" >> maybe(sourced("/" >> name / "/")), optionalList(entityDecl)))
+constexpr auto structureName{maybe(sourced("/" >> name / "/"))};
+
+// Note that Parser<StructureStmt>{} has a mandatory list of entity-decls
+// and is used only by NestedStructureStmt{}.Parse() in user-state.cpp.
+TYPE_PARSER(construct<StructureStmt>("STRUCTURE" >> structureName,
+    localRecovery(
+        "entity declarations are required on a nested structure"_err_en_US,
+        nonemptyList(entityDecl), ok)))
 
 constexpr auto nestedStructureDef{
     CONTEXT_PARSER("nested STRUCTURE definition"_en_US,
@@ -1323,7 +1329,9 @@ TYPE_PARSER(construct<StructureField>(statement(StructureComponents{})) ||
 TYPE_CONTEXT_PARSER("STRUCTURE definition"_en_US,
     extension<LanguageFeature::DECStructures>(
         "nonstandard usage: STRUCTURE"_port_en_US,
-        construct<StructureDef>(statement(Parser<StructureStmt>{}),
+        construct<StructureDef>(
+            statement(construct<StructureStmt>(
+                "STRUCTURE" >> structureName, optionalList(entityDecl))),
             many(Parser<StructureField>{}),
             statement(construct<StructureDef::EndStructureStmt>(
                 "END STRUCTURE"_tok)))))
diff --git a/flang/test/Semantics/struct03.f90 b/flang/test/Semantics/struct03.f90
new file mode 100644
index 0000000000000..a334cca8945fc
--- /dev/null
+++ b/flang/test/Semantics/struct03.f90
@@ -0,0 +1,7 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+  structure /s/
+    !ERROR: entity declarations are required on a nested structure
+    structure /nested/
+    end structure
+  end structure
+end

From bda1893a62b2081780fe1c000d6447b73d42aa2c Mon Sep 17 00:00:00 2001
From: Dmitriy Chestnykh <dm.chestnykh@gmail.com>
Date: Fri, 19 Jul 2024 02:51:45 +0300
Subject: [PATCH 108/486] [compiler-rt] Add `DumpAllRegisters` impl (#99049)

- Add implementation for x86_64 and linux
- Add test

The output is like

==XXYYZZ==Register values:
rax = 0x...  rbx = 0x...  rcx = 0x...  rdx = 0x...
rdi = 0x...  rsi = 0x...  rbp = 0x...  rsp = 0x...
 r8 = 0x...   r9 = 0x...  r10 = 0x...  r11 = 0x...
r12 = 0x...  r13 = 0x...  r14 = 0x...  r15 = 0x...
---
 .../lib/sanitizer_common/sanitizer_linux.cpp  | 116 +++++++++++++++++-
 .../TestCases/Linux/dump_registers_i386.cpp   |  17 +++
 .../TestCases/Linux/dump_registers_x86_64.cpp |  19 +++
 3 files changed, 151 insertions(+), 1 deletion(-)
 create mode 100644 compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_i386.cpp
 create mode 100644 compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_x86_64.cpp

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index 794e3e7b2fb6c..aa74a4c581dd0 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -2118,8 +2118,122 @@ bool SignalContext::IsTrueFaultingAddress() const {
   return si->si_signo == SIGSEGV && si->si_code != 128;
 }
 
+UNUSED
+static const char *RegNumToRegName(int reg) {
+  switch (reg) {
+#  if defined(__x86_64__)
+    case REG_RAX:
+      return "rax";
+    case REG_RBX:
+      return "rbx";
+    case REG_RCX:
+      return "rcx";
+    case REG_RDX:
+      return "rdx";
+    case REG_RDI:
+      return "rdi";
+    case REG_RSI:
+      return "rsi";
+    case REG_RBP:
+      return "rbp";
+    case REG_RSP:
+      return "rsp";
+    case REG_R8:
+      return "r8";
+    case REG_R9:
+      return "r9";
+    case REG_R10:
+      return "r10";
+    case REG_R11:
+      return "r11";
+    case REG_R12:
+      return "r12";
+    case REG_R13:
+      return "r13";
+    case REG_R14:
+      return "r14";
+    case REG_R15:
+      return "r15";
+#  elif defined(__i386__)
+    case REG_EAX:
+      return "eax";
+    case REG_EBX:
+      return "ebx";
+    case REG_ECX:
+      return "ecx";
+    case REG_EDX:
+      return "edx";
+    case REG_EDI:
+      return "edi";
+    case REG_ESI:
+      return "esi";
+    case REG_EBP:
+      return "ebp";
+    case REG_ESP:
+      return "esp";
+#  endif
+    default:
+      return NULL;
+  }
+  return NULL;
+}
+
+UNUSED
+static void DumpSingleReg(ucontext_t *ctx, int RegNum) {
+  const char *RegName = RegNumToRegName(RegNum);
+#  if defined(__x86_64__)
+  Printf("%s%s = 0x%016llx  ", internal_strlen(RegName) == 2 ? " " : "",
+         RegName, ctx->uc_mcontext.gregs[RegNum]);
+#  elif defined(__i386__)
+  Printf("%s = 0x%08x  ", RegName, ctx->uc_mcontext.gregs[RegNum]);
+#  endif
+}
+
 void SignalContext::DumpAllRegisters(void *context) {
-  // FIXME: Implement this.
+#  if SANITIZER_LINUX
+  ucontext_t *ucontext = (ucontext_t *)context;
+#    if defined(__x86_64__)
+  Report("Register values:\n");
+  DumpSingleReg(ucontext, REG_RAX);
+  DumpSingleReg(ucontext, REG_RBX);
+  DumpSingleReg(ucontext, REG_RCX);
+  DumpSingleReg(ucontext, REG_RDX);
+  Printf("\n");
+  DumpSingleReg(ucontext, REG_RDI);
+  DumpSingleReg(ucontext, REG_RSI);
+  DumpSingleReg(ucontext, REG_RBP);
+  DumpSingleReg(ucontext, REG_RSP);
+  Printf("\n");
+  DumpSingleReg(ucontext, REG_R8);
+  DumpSingleReg(ucontext, REG_R9);
+  DumpSingleReg(ucontext, REG_R10);
+  DumpSingleReg(ucontext, REG_R11);
+  Printf("\n");
+  DumpSingleReg(ucontext, REG_R12);
+  DumpSingleReg(ucontext, REG_R13);
+  DumpSingleReg(ucontext, REG_R14);
+  DumpSingleReg(ucontext, REG_R15);
+  Printf("\n");
+#    elif defined(__i386__)
+  // Duplication of this report print is caused by partial support
+  // of register values dumping. In case of unsupported yet architecture let's
+  // avoid printing 'Register values:' without actual values in the following
+  // output.
+  Report("Register values:\n");
+  DumpSingleReg(ucontext, REG_EAX);
+  DumpSingleReg(ucontext, REG_EBX);
+  DumpSingleReg(ucontext, REG_ECX);
+  DumpSingleReg(ucontext, REG_EDX);
+  Printf("\n");
+  DumpSingleReg(ucontext, REG_EDI);
+  DumpSingleReg(ucontext, REG_ESI);
+  DumpSingleReg(ucontext, REG_EBP);
+  DumpSingleReg(ucontext, REG_ESP);
+  Printf("\n");
+#    endif
+  (void)ucontext;
+#  endif
+  // FIXME: Implement this for other OSes and architectures.
 }
 
 static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_i386.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_i386.cpp
new file mode 100644
index 0000000000000..0a6c3e361732c
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_i386.cpp
@@ -0,0 +1,17 @@
+// Check that sanitizer prints registers dump_registers on dump_registers=1
+// RUN: %clangxx  %s -o %t
+// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-NODUMP
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-DUMP
+//
+// REQUIRES: i386-target-arch
+
+#include <signal.h>
+
+int main() {
+  raise(SIGSEGV);
+  // CHECK-DUMP: Register values
+  // CHECK-DUMP-NEXT: eax = {{0x[0-9a-f]+}}  ebx = {{0x[0-9a-f]+}}  ecx = {{0x[0-9a-f]+}}  edx = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: edi = {{0x[0-9a-f]+}}  esi = {{0x[0-9a-f]+}}  ebp = {{0x[0-9a-f]+}}  esp = {{0x[0-9a-f]+}}
+  // CHECK-NODUMP-NOT: Register values
+  return 0;
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_x86_64.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_x86_64.cpp
new file mode 100644
index 0000000000000..1805343c23063
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_x86_64.cpp
@@ -0,0 +1,19 @@
+// Check that sanitizer prints registers dump_registers on dump_registers=1
+// RUN: %clangxx  %s -o %t
+// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-NODUMP
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-DUMP
+//
+// REQUIRES: x86_64-target-arch
+
+#include <signal.h>
+
+int main() {
+  raise(SIGSEGV);
+  // CHECK-DUMP: Register values
+  // CHECK-DUMP-NEXT: rax = {{0x[0-9a-f]+}}  rbx = {{0x[0-9a-f]+}}  rcx = {{0x[0-9a-f]+}}  rdx = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: rdi = {{0x[0-9a-f]+}}  rsi = {{0x[0-9a-f]+}}  rbp = {{0x[0-9a-f]+}}  rsp = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT:  r8 = {{0x[0-9a-f]+}}   r9 = {{0x[0-9a-f]+}}  r10 = {{0x[0-9a-f]+}}  r11 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: r12 = {{0x[0-9a-f]+}}   r13 = {{0x[0-9a-f]+}} r14 = {{0x[0-9a-f]+}}  r15 = {{0x[0-9a-f]+}}
+  // CHECK-NODUMP-NOT: Register values
+  return 0;
+}

From 996d31c7ba841fdc3bd375f3fed4d8324618425b Mon Sep 17 00:00:00 2001
From: Sam James <sam@gentoo.org>
Date: Fri, 19 Jul 2024 00:52:54 +0100
Subject: [PATCH 109/486] [msan] Fix goo.gl link in comment for Valgrind paper

goo.gl is going away: https://developers.googleblog.com/en/google-url-shortener-links-will-no-longer-be-available/

Fix goo.gl link from:
- http://goo.gl/QKbem
+ https://static.usenix.org/event/usenix05/tech/general/full_papers/seward/seward_html/usenix2005.html
and reflow the comment a bit to make it look a bit better after the URL change,
although it's not perfect now.

Committed as obvious.

Bug: https://github.com/llvm/llvm-project/issues/99586
---
 .../lib/Transforms/Instrumentation/MemorySanitizer.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index c7d41f6298372..9bda288242299 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -11,11 +11,11 @@
 /// reads.
 ///
 /// The algorithm of the tool is similar to Memcheck
-/// (http://goo.gl/QKbem). We associate a few shadow bits with every
-/// byte of the application memory, poison the shadow of the malloc-ed
-/// or alloca-ed memory, load the shadow bits on every memory read,
-/// propagate the shadow bits through some of the arithmetic
-/// instruction (including MOV), store the shadow bits on every memory
+/// (https://static.usenix.org/event/usenix05/tech/general/full_papers/seward/seward_html/usenix2005.html)
+/// We associate a few shadow bits with every byte of the application memory,
+/// poison the shadow of the malloc-ed or alloca-ed memory, load the shadow,
+/// bits on every memory read, propagate the shadow bits through some of the
+/// arithmetic instruction (including MOV), store the shadow bits on every memory
 /// write, report a bug on some other instructions (e.g. JMP) if the
 /// associated shadow is poisoned.
 ///

From 4120570dc408a6ccc7133b4bdbaf5cf6c4af9db7 Mon Sep 17 00:00:00 2001
From: Jacob Lalonde <jalalonde@fb.com>
Date: Thu, 18 Jul 2024 17:10:15 -0700
Subject: [PATCH 110/486] [LLDB][SaveCore] Add SBSaveCoreOptions Object, and
 SBProcess::SaveCore() overload (#98403)

This PR adds `SBSaveCoreOptions`, which is a container class for options
when LLDB is taking coredumps. For this first iteration this container
just keeps parity with the extant API of `file, style, plugin`. In the
future this options object can be extended to allow users to take a
subset of their core dumps.
---
 lldb/bindings/headers.swig                    |  1 +
 .../interface/SBSaveCoreOptionsDocstrings.i   |  0
 lldb/bindings/interfaces.swig                 |  2 +
 lldb/include/lldb/API/LLDB.h                  |  1 +
 lldb/include/lldb/API/SBDefines.h             |  1 +
 lldb/include/lldb/API/SBError.h               |  1 +
 lldb/include/lldb/API/SBFileSpec.h            |  1 +
 lldb/include/lldb/API/SBProcess.h             |  6 ++
 lldb/include/lldb/API/SBSaveCoreOptions.h     | 69 +++++++++++++++
 lldb/include/lldb/Core/PluginManager.h        |  6 +-
 lldb/include/lldb/Symbol/SaveCoreOptions.h    | 44 ++++++++++
 lldb/include/lldb/lldb-private-interfaces.h   |  4 +-
 lldb/source/API/CMakeLists.txt                |  1 +
 lldb/source/API/SBProcess.cpp                 | 23 +++--
 lldb/source/API/SBSaveCoreOptions.cpp         | 85 +++++++++++++++++++
 lldb/source/Commands/CommandObjectProcess.cpp | 25 +++---
 lldb/source/Core/PluginManager.cpp            | 46 +++++++---
 .../ObjectFile/Mach-O/ObjectFileMachO.cpp     | 13 +--
 .../ObjectFile/Mach-O/ObjectFileMachO.h       |  3 +-
 .../Minidump/ObjectFileMinidump.cpp           | 16 ++--
 .../ObjectFile/Minidump/ObjectFileMinidump.h  |  3 +-
 .../ObjectFile/PECOFF/ObjectFilePECOFF.cpp    |  9 +-
 .../ObjectFile/PECOFF/ObjectFilePECOFF.h      |  3 +-
 .../ObjectFile/PECOFF/WindowsMiniDump.cpp     |  3 +-
 .../ObjectFile/PECOFF/WindowsMiniDump.h       |  2 +-
 lldb/source/Symbol/CMakeLists.txt             |  1 +
 lldb/source/Symbol/SaveCoreOptions.cpp        | 53 ++++++++++++
 .../process_save_core/TestProcessSaveCore.py  |  3 +-
 .../TestProcessSaveCoreMinidump.py            | 21 ++++-
 .../TestSBSaveCoreOptions.py                  | 28 ++++++
 30 files changed, 412 insertions(+), 62 deletions(-)
 create mode 100644 lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i
 create mode 100644 lldb/include/lldb/API/SBSaveCoreOptions.h
 create mode 100644 lldb/include/lldb/Symbol/SaveCoreOptions.h
 create mode 100644 lldb/source/API/SBSaveCoreOptions.cpp
 create mode 100644 lldb/source/Symbol/SaveCoreOptions.cpp
 create mode 100644 lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py

diff --git a/lldb/bindings/headers.swig b/lldb/bindings/headers.swig
index c91504604b6ac..c0dde905f986b 100644
--- a/lldb/bindings/headers.swig
+++ b/lldb/bindings/headers.swig
@@ -21,6 +21,7 @@
 #include "lldb/API/SBCommandReturnObject.h"
 #include "lldb/API/SBCommunication.h"
 #include "lldb/API/SBCompileUnit.h"
+#include "lldb/API/SBSaveCoreOptions.h"
 #include "lldb/API/SBData.h"
 #include "lldb/API/SBDebugger.h"
 #include "lldb/API/SBDeclaration.h"
diff --git a/lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i b/lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/lldb/bindings/interfaces.swig b/lldb/bindings/interfaces.swig
index 0953f4c72a910..8a6fed95f0b72 100644
--- a/lldb/bindings/interfaces.swig
+++ b/lldb/bindings/interfaces.swig
@@ -25,6 +25,7 @@
 %include "./interface/SBCommandReturnObjectDocstrings.i"
 %include "./interface/SBCommunicationDocstrings.i"
 %include "./interface/SBCompileUnitDocstrings.i"
+%include "./interface/SBSaveCoreOptionsDocstrings.i"
 %include "./interface/SBDataDocstrings.i"
 %include "./interface/SBDebuggerDocstrings.i"
 %include "./interface/SBDeclarationDocstrings.i"
@@ -101,6 +102,7 @@
 %include "lldb/API/SBCommandReturnObject.h"
 %include "lldb/API/SBCommunication.h"
 %include "lldb/API/SBCompileUnit.h"
+%include "lldb/API/SBSaveCoreOptions.h"
 %include "lldb/API/SBData.h"
 %include "lldb/API/SBDebugger.h"
 %include "lldb/API/SBDeclaration.h"
diff --git a/lldb/include/lldb/API/LLDB.h b/lldb/include/lldb/API/LLDB.h
index d8cc9f5067fe9..40368e036e0e4 100644
--- a/lldb/include/lldb/API/LLDB.h
+++ b/lldb/include/lldb/API/LLDB.h
@@ -57,6 +57,7 @@
 #include "lldb/API/SBQueue.h"
 #include "lldb/API/SBQueueItem.h"
 #include "lldb/API/SBReproducer.h"
+#include "lldb/API/SBSaveCoreOptions.h"
 #include "lldb/API/SBSection.h"
 #include "lldb/API/SBSourceManager.h"
 #include "lldb/API/SBStatisticsOptions.h"
diff --git a/lldb/include/lldb/API/SBDefines.h b/lldb/include/lldb/API/SBDefines.h
index 87c0a1c3661ca..f42e2be558d2e 100644
--- a/lldb/include/lldb/API/SBDefines.h
+++ b/lldb/include/lldb/API/SBDefines.h
@@ -61,6 +61,7 @@ class LLDB_API SBCommandPluginInterface;
 class LLDB_API SBCommandReturnObject;
 class LLDB_API SBCommunication;
 class LLDB_API SBCompileUnit;
+class LLDB_API SBSaveCoreOptions;
 class LLDB_API SBData;
 class LLDB_API SBDebugger;
 class LLDB_API SBDeclaration;
diff --git a/lldb/include/lldb/API/SBError.h b/lldb/include/lldb/API/SBError.h
index 1a720a479d9a6..17f2c6c3027af 100644
--- a/lldb/include/lldb/API/SBError.h
+++ b/lldb/include/lldb/API/SBError.h
@@ -77,6 +77,7 @@ class LLDB_API SBError {
   friend class SBBreakpointName;
   friend class SBCommandReturnObject;
   friend class SBCommunication;
+  friend class SBSaveCoreOptions;
   friend class SBData;
   friend class SBDebugger;
   friend class SBFile;
diff --git a/lldb/include/lldb/API/SBFileSpec.h b/lldb/include/lldb/API/SBFileSpec.h
index beefa19ad7f36..36641843aabeb 100644
--- a/lldb/include/lldb/API/SBFileSpec.h
+++ b/lldb/include/lldb/API/SBFileSpec.h
@@ -78,6 +78,7 @@ class LLDB_API SBFileSpec {
   friend class SBTarget;
   friend class SBThread;
   friend class SBTrace;
+  friend class SBSaveCoreOptions;
 
   SBFileSpec(const lldb_private::FileSpec &fspec);
 
diff --git a/lldb/include/lldb/API/SBProcess.h b/lldb/include/lldb/API/SBProcess.h
index a6ab7ae759918..778be79583990 100644
--- a/lldb/include/lldb/API/SBProcess.h
+++ b/lldb/include/lldb/API/SBProcess.h
@@ -378,6 +378,12 @@ class LLDB_API SBProcess {
   /// \param[in] file_name - The name of the file to save the core file to.
   lldb::SBError SaveCore(const char *file_name);
 
+  /// Save the state of the process with the desired settings
+  /// as defined in the options object.
+  ///
+  /// \param[in] options - The options to use when saving the core file.
+  lldb::SBError SaveCore(SBSaveCoreOptions &options);
+
   /// Query the address load_addr and store the details of the memory
   /// region that contains it in the supplied SBMemoryRegionInfo object.
   /// To iterate over all memory regions use GetMemoryRegionList.
diff --git a/lldb/include/lldb/API/SBSaveCoreOptions.h b/lldb/include/lldb/API/SBSaveCoreOptions.h
new file mode 100644
index 0000000000000..b8659bf128a78
--- /dev/null
+++ b/lldb/include/lldb/API/SBSaveCoreOptions.h
@@ -0,0 +1,69 @@
+//===-- SBSaveCoreOptions.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_API_SBSAVECOREOPTIONS_H
+#define LLDB_API_SBSAVECOREOPTIONS_H
+
+#include "lldb/API/SBDefines.h"
+#include "lldb/Symbol/SaveCoreOptions.h"
+
+namespace lldb {
+
+class LLDB_API SBSaveCoreOptions {
+public:
+  SBSaveCoreOptions();
+  SBSaveCoreOptions(const lldb::SBSaveCoreOptions &rhs);
+  ~SBSaveCoreOptions() = default;
+
+  const SBSaveCoreOptions &operator=(const lldb::SBSaveCoreOptions &rhs);
+
+  /// Set the plugin name. Supplying null or empty string will reset
+  /// the option.
+  ///
+  /// \param plugin Name of the object file plugin.
+  SBError SetPluginName(const char *plugin);
+
+  /// Get the Core dump plugin name, if set.
+  ///
+  /// \return The name of the plugin, or null if not set.
+  const char *GetPluginName() const;
+
+  /// Set the Core dump style.
+  ///
+  /// \param style The style of the core dump.
+  void SetStyle(lldb::SaveCoreStyle style);
+
+  /// Get the Core dump style, if set.
+  ///
+  /// \return The core dump style, or undefined if not set.
+  lldb::SaveCoreStyle GetStyle() const;
+
+  /// Set the output file path
+  ///
+  /// \param output_file a
+  /// \class SBFileSpec object that describes the output file.
+  void SetOutputFile(SBFileSpec output_file);
+
+  /// Get the output file spec
+  ///
+  /// \return The output file spec.
+  SBFileSpec GetOutputFile() const;
+
+  /// Reset all options.
+  void Clear();
+
+protected:
+  friend class SBProcess;
+  lldb_private::SaveCoreOptions &ref() const;
+
+private:
+  std::unique_ptr<lldb_private::SaveCoreOptions> m_opaque_up;
+}; // SBSaveCoreOptions
+} // namespace lldb
+
+#endif // LLDB_API_SBSAVECOREOPTIONS_H
diff --git a/lldb/include/lldb/Core/PluginManager.h b/lldb/include/lldb/Core/PluginManager.h
index f2296e2920238..38a291d9f0afd 100644
--- a/lldb/include/lldb/Core/PluginManager.h
+++ b/lldb/include/lldb/Core/PluginManager.h
@@ -178,6 +178,8 @@ class PluginManager {
 
   static bool UnregisterPlugin(ObjectFileCreateInstance create_callback);
 
+  static bool IsRegisteredObjectFilePluginName(llvm::StringRef name);
+
   static ObjectFileCreateInstance
   GetObjectFileCreateCallbackAtIndex(uint32_t idx);
 
@@ -191,9 +193,7 @@ class PluginManager {
   GetObjectFileCreateMemoryCallbackForPluginName(llvm::StringRef name);
 
   static Status SaveCore(const lldb::ProcessSP &process_sp,
-                         const FileSpec &outfile,
-                         lldb::SaveCoreStyle &core_style,
-                         llvm::StringRef plugin_name);
+                         const lldb_private::SaveCoreOptions &core_options);
 
   // ObjectContainer
   static bool RegisterPlugin(
diff --git a/lldb/include/lldb/Symbol/SaveCoreOptions.h b/lldb/include/lldb/Symbol/SaveCoreOptions.h
new file mode 100644
index 0000000000000..583bc1f483d04
--- /dev/null
+++ b/lldb/include/lldb/Symbol/SaveCoreOptions.h
@@ -0,0 +1,44 @@
+//===-- SaveCoreOptions.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_OBJECTFILE_SaveCoreOPTIONS_H
+#define LLDB_SOURCE_PLUGINS_OBJECTFILE_SaveCoreOPTIONS_H
+
+#include "lldb/Utility/FileSpec.h"
+#include "lldb/lldb-forward.h"
+#include "lldb/lldb-types.h"
+
+#include <optional>
+#include <string>
+
+namespace lldb_private {
+
+class SaveCoreOptions {
+public:
+  SaveCoreOptions(){};
+  ~SaveCoreOptions() = default;
+
+  lldb_private::Status SetPluginName(const char *name);
+  std::optional<std::string> GetPluginName() const;
+
+  void SetStyle(lldb::SaveCoreStyle style);
+  lldb::SaveCoreStyle GetStyle() const;
+
+  void SetOutputFile(lldb_private::FileSpec file);
+  const std::optional<lldb_private::FileSpec> GetOutputFile() const;
+
+  void Clear();
+
+private:
+  std::optional<std::string> m_plugin_name;
+  std::optional<lldb_private::FileSpec> m_file;
+  std::optional<lldb::SaveCoreStyle> m_style;
+};
+} // namespace lldb_private
+
+#endif // LLDB_SOURCE_PLUGINS_OBJECTFILE_SaveCoreOPTIONS_H
diff --git a/lldb/include/lldb/lldb-private-interfaces.h b/lldb/include/lldb/lldb-private-interfaces.h
index cdd9b51d9329c..10eaf1e6a5add 100644
--- a/lldb/include/lldb/lldb-private-interfaces.h
+++ b/lldb/include/lldb/lldb-private-interfaces.h
@@ -9,6 +9,7 @@
 #ifndef LLDB_LLDB_PRIVATE_INTERFACES_H
 #define LLDB_LLDB_PRIVATE_INTERFACES_H
 
+#include "lldb/Symbol/SaveCoreOptions.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-private-enumerations.h"
@@ -55,8 +56,7 @@ typedef ObjectFile *(*ObjectFileCreateMemoryInstance)(
     const lldb::ModuleSP &module_sp, lldb::WritableDataBufferSP data_sp,
     const lldb::ProcessSP &process_sp, lldb::addr_t offset);
 typedef bool (*ObjectFileSaveCore)(const lldb::ProcessSP &process_sp,
-                                   const FileSpec &outfile,
-                                   lldb::SaveCoreStyle &core_style,
+                                   const lldb_private::SaveCoreOptions &options,
                                    Status &error);
 typedef EmulateInstruction *(*EmulateInstructionCreateInstance)(
     const ArchSpec &arch, InstructionType inst_type);
diff --git a/lldb/source/API/CMakeLists.txt b/lldb/source/API/CMakeLists.txt
index 6397101609315..a32bc58507d8e 100644
--- a/lldb/source/API/CMakeLists.txt
+++ b/lldb/source/API/CMakeLists.txt
@@ -56,6 +56,7 @@ add_lldb_library(liblldb SHARED ${option_framework}
   SBCommandReturnObject.cpp
   SBCommunication.cpp
   SBCompileUnit.cpp
+  SBSaveCoreOptions.cpp
   SBData.cpp
   SBDebugger.cpp
   SBDeclaration.cpp
diff --git a/lldb/source/API/SBProcess.cpp b/lldb/source/API/SBProcess.cpp
index efb3c8def2796..b88f897ff5280 100644
--- a/lldb/source/API/SBProcess.cpp
+++ b/lldb/source/API/SBProcess.cpp
@@ -40,6 +40,7 @@
 #include "lldb/API/SBFileSpec.h"
 #include "lldb/API/SBMemoryRegionInfo.h"
 #include "lldb/API/SBMemoryRegionInfoList.h"
+#include "lldb/API/SBSaveCoreOptions.h"
 #include "lldb/API/SBScriptObject.h"
 #include "lldb/API/SBStream.h"
 #include "lldb/API/SBStringList.h"
@@ -1216,13 +1217,28 @@ bool SBProcess::IsInstrumentationRuntimePresent(
 
 lldb::SBError SBProcess::SaveCore(const char *file_name) {
   LLDB_INSTRUMENT_VA(this, file_name);
-  return SaveCore(file_name, "", SaveCoreStyle::eSaveCoreFull);
+  SBSaveCoreOptions options;
+  options.SetOutputFile(SBFileSpec(file_name));
+  options.SetStyle(SaveCoreStyle::eSaveCoreFull);
+  return SaveCore(options);
 }
 
 lldb::SBError SBProcess::SaveCore(const char *file_name,
                                   const char *flavor,
                                   SaveCoreStyle core_style) {
   LLDB_INSTRUMENT_VA(this, file_name, flavor, core_style);
+  SBSaveCoreOptions options;
+  options.SetOutputFile(SBFileSpec(file_name));
+  options.SetStyle(core_style);
+  SBError error = options.SetPluginName(flavor);
+  if (error.Fail())
+    return error;
+  return SaveCore(options);
+}
+
+lldb::SBError SBProcess::SaveCore(SBSaveCoreOptions &options) {
+
+  LLDB_INSTRUMENT_VA(this, options);
 
   lldb::SBError error;
   ProcessSP process_sp(GetSP());
@@ -1239,10 +1255,7 @@ lldb::SBError SBProcess::SaveCore(const char *file_name,
     return error;
   }
 
-  FileSpec core_file(file_name);
-  FileSystem::Instance().Resolve(core_file);
-  error.ref() = PluginManager::SaveCore(process_sp, core_file, core_style,
-                                        flavor);
+  error.ref() = PluginManager::SaveCore(process_sp, options.ref());
 
   return error;
 }
diff --git a/lldb/source/API/SBSaveCoreOptions.cpp b/lldb/source/API/SBSaveCoreOptions.cpp
new file mode 100644
index 0000000000000..6c3f74596203d
--- /dev/null
+++ b/lldb/source/API/SBSaveCoreOptions.cpp
@@ -0,0 +1,85 @@
+//===-- SBSaveCoreOptions.cpp -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/API/SBSaveCoreOptions.h"
+#include "lldb/API/SBError.h"
+#include "lldb/API/SBFileSpec.h"
+#include "lldb/Host/FileSystem.h"
+#include "lldb/Symbol/SaveCoreOptions.h"
+#include "lldb/Utility/Instrumentation.h"
+
+#include "Utils.h"
+
+using namespace lldb;
+
+SBSaveCoreOptions::SBSaveCoreOptions() {
+  LLDB_INSTRUMENT_VA(this)
+
+  m_opaque_up = std::make_unique<lldb_private::SaveCoreOptions>();
+}
+
+SBSaveCoreOptions::SBSaveCoreOptions(const SBSaveCoreOptions &rhs) {
+  LLDB_INSTRUMENT_VA(this, rhs);
+
+  m_opaque_up = clone(rhs.m_opaque_up);
+}
+
+const SBSaveCoreOptions &
+SBSaveCoreOptions::operator=(const SBSaveCoreOptions &rhs) {
+  LLDB_INSTRUMENT_VA(this, rhs);
+
+  if (this != &rhs)
+    m_opaque_up = clone(rhs.m_opaque_up);
+  return *this;
+}
+
+SBError SBSaveCoreOptions::SetPluginName(const char *name) {
+  LLDB_INSTRUMENT_VA(this, name);
+  lldb_private::Status error = m_opaque_up->SetPluginName(name);
+  return SBError(error);
+}
+
+void SBSaveCoreOptions::SetStyle(lldb::SaveCoreStyle style) {
+  LLDB_INSTRUMENT_VA(this, style);
+  m_opaque_up->SetStyle(style);
+}
+
+void SBSaveCoreOptions::SetOutputFile(lldb::SBFileSpec file_spec) {
+  LLDB_INSTRUMENT_VA(this, file_spec);
+  m_opaque_up->SetOutputFile(file_spec.ref());
+}
+
+const char *SBSaveCoreOptions::GetPluginName() const {
+  LLDB_INSTRUMENT_VA(this);
+  const auto name = m_opaque_up->GetPluginName();
+  if (!name)
+    return nullptr;
+  return lldb_private::ConstString(name.value()).GetCString();
+}
+
+SBFileSpec SBSaveCoreOptions::GetOutputFile() const {
+  LLDB_INSTRUMENT_VA(this);
+  const auto file_spec = m_opaque_up->GetOutputFile();
+  if (file_spec)
+    return SBFileSpec(file_spec.value());
+  return SBFileSpec();
+}
+
+lldb::SaveCoreStyle SBSaveCoreOptions::GetStyle() const {
+  LLDB_INSTRUMENT_VA(this);
+  return m_opaque_up->GetStyle();
+}
+
+void SBSaveCoreOptions::Clear() {
+  LLDB_INSTRUMENT_VA(this);
+  m_opaque_up->Clear();
+}
+
+lldb_private::SaveCoreOptions &SBSaveCoreOptions::ref() const {
+  return *m_opaque_up.get();
+}
diff --git a/lldb/source/Commands/CommandObjectProcess.cpp b/lldb/source/Commands/CommandObjectProcess.cpp
index 8685d5761557b..50695af556939 100644
--- a/lldb/source/Commands/CommandObjectProcess.cpp
+++ b/lldb/source/Commands/CommandObjectProcess.cpp
@@ -1273,13 +1273,13 @@ class CommandObjectProcessSaveCore : public CommandObjectParsed {
 
       switch (short_option) {
       case 'p':
-        m_requested_plugin_name = option_arg.str();
+        error = m_core_dump_options.SetPluginName(option_arg.data());
         break;
       case 's':
-        m_requested_save_core_style =
+        m_core_dump_options.SetStyle(
             (lldb::SaveCoreStyle)OptionArgParser::ToOptionEnum(
                 option_arg, GetDefinitions()[option_idx].enum_values,
-                eSaveCoreUnspecified, error);
+                eSaveCoreUnspecified, error));
         break;
       default:
         llvm_unreachable("Unimplemented option");
@@ -1289,13 +1289,11 @@ class CommandObjectProcessSaveCore : public CommandObjectParsed {
     }
 
     void OptionParsingStarting(ExecutionContext *execution_context) override {
-      m_requested_save_core_style = eSaveCoreUnspecified;
-      m_requested_plugin_name.clear();
+      m_core_dump_options.Clear();
     }
 
     // Instance variables to hold the values for command options.
-    SaveCoreStyle m_requested_save_core_style = eSaveCoreUnspecified;
-    std::string m_requested_plugin_name;
+    SaveCoreOptions m_core_dump_options;
   };
 
 protected:
@@ -1305,13 +1303,14 @@ class CommandObjectProcessSaveCore : public CommandObjectParsed {
       if (command.GetArgumentCount() == 1) {
         FileSpec output_file(command.GetArgumentAtIndex(0));
         FileSystem::Instance().Resolve(output_file);
-        SaveCoreStyle corefile_style = m_options.m_requested_save_core_style;
-        Status error =
-            PluginManager::SaveCore(process_sp, output_file, corefile_style,
-                                    m_options.m_requested_plugin_name);
+        auto &core_dump_options = m_options.m_core_dump_options;
+        core_dump_options.SetOutputFile(output_file);
+        Status error = PluginManager::SaveCore(process_sp, core_dump_options);
         if (error.Success()) {
-          if (corefile_style == SaveCoreStyle::eSaveCoreDirtyOnly ||
-              corefile_style == SaveCoreStyle::eSaveCoreStackOnly) {
+          if (core_dump_options.GetStyle() ==
+                  SaveCoreStyle::eSaveCoreDirtyOnly ||
+              core_dump_options.GetStyle() ==
+                  SaveCoreStyle::eSaveCoreStackOnly) {
             result.AppendMessageWithFormat(
                 "\nModified-memory or stack-memory only corefile "
                 "created.  This corefile may \n"
diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp
index b428370d7f333..759ef3a8afe02 100644
--- a/lldb/source/Core/PluginManager.cpp
+++ b/lldb/source/Core/PluginManager.cpp
@@ -12,6 +12,7 @@
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Interpreter/OptionValueProperties.h"
+#include "lldb/Symbol/SaveCoreOptions.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/Status.h"
@@ -639,6 +640,18 @@ static ObjectFileInstances &GetObjectFileInstances() {
   return g_instances;
 }
 
+bool PluginManager::IsRegisteredObjectFilePluginName(llvm::StringRef name) {
+  if (name.empty())
+    return false;
+
+  const auto &instances = GetObjectFileInstances().GetInstances();
+  for (auto &instance : instances) {
+    if (instance.name == name)
+      return true;
+  }
+  return false;
+}
+
 bool PluginManager::RegisterPlugin(
     llvm::StringRef name, llvm::StringRef description,
     ObjectFileCreateInstance create_callback,
@@ -689,12 +702,22 @@ PluginManager::GetObjectFileCreateMemoryCallbackForPluginName(
 }
 
 Status PluginManager::SaveCore(const lldb::ProcessSP &process_sp,
-                               const FileSpec &outfile,
-                               lldb::SaveCoreStyle &core_style,
-                               llvm::StringRef plugin_name) {
-  if (plugin_name.empty()) {
+                               const lldb_private::SaveCoreOptions &options) {
+  Status error;
+  if (!options.GetOutputFile()) {
+    error.SetErrorString("No output file specified");
+    return error;
+  }
+
+  if (!process_sp) {
+    error.SetErrorString("Invalid process");
+    return error;
+  }
+
+  if (!options.GetPluginName().has_value()) {
     // Try saving core directly from the process plugin first.
-    llvm::Expected<bool> ret = process_sp->SaveCore(outfile.GetPath());
+    llvm::Expected<bool> ret =
+        process_sp->SaveCore(options.GetOutputFile()->GetPath());
     if (!ret)
       return Status(ret.takeError());
     if (ret.get())
@@ -702,17 +725,20 @@ Status PluginManager::SaveCore(const lldb::ProcessSP &process_sp,
   }
 
   // Fall back to object plugins.
-  Status error;
+  const auto &plugin_name = options.GetPluginName().value_or("");
   auto &instances = GetObjectFileInstances().GetInstances();
   for (auto &instance : instances) {
     if (plugin_name.empty() || instance.name == plugin_name) {
-      if (instance.save_core &&
-          instance.save_core(process_sp, outfile, core_style, error))
+      if (instance.save_core && instance.save_core(process_sp, options, error))
         return error;
     }
   }
-  error.SetErrorString(
-      "no ObjectFile plugins were able to save a core for this process");
+
+  // Check to see if any of the object file plugins tried and failed to save.
+  // If none ran, set the error message.
+  if (error.Success())
+    error.SetErrorString(
+        "no ObjectFile plugins were able to save a core for this process");
   return error;
 }
 
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index 0dcb1bed23548..2c7005449f9d7 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -6519,14 +6519,15 @@ struct page_object {
 };
 
 bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
-                               const FileSpec &outfile,
-                               lldb::SaveCoreStyle &core_style, Status &error) {
-  if (!process_sp)
-    return false;
-
-  // Default on macOS is to create a dirty-memory-only corefile.
+                               const lldb_private::SaveCoreOptions &options,
+                               Status &error) {
+  auto core_style = options.GetStyle();
   if (core_style == SaveCoreStyle::eSaveCoreUnspecified)
     core_style = SaveCoreStyle::eSaveCoreDirtyOnly;
+  // The FileSpec and Process are already checked in PluginManager::SaveCore.
+  assert(options.GetOutputFile().has_value());
+  assert(process_sp);
+  const FileSpec outfile = options.GetOutputFile().value();
 
   Target &target = process_sp->GetTarget();
   const ArchSpec target_arch = target.GetArchitecture();
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
index 55bc688126eb3..e7af90e28bc4b 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
@@ -62,8 +62,7 @@ class ObjectFileMachO : public lldb_private::ObjectFile {
                                         lldb_private::ModuleSpecList &specs);
 
   static bool SaveCore(const lldb::ProcessSP &process_sp,
-                       const lldb_private::FileSpec &outfile,
-                       lldb::SaveCoreStyle &core_style,
+                       const lldb_private::SaveCoreOptions &options,
                        lldb_private::Status &error);
 
   static bool MagicBytesMatch(lldb::DataBufferSP data_sp, lldb::addr_t offset,
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp
index 7c875aa3223ed..faa144bfb5f6a 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp
+++ b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp
@@ -56,18 +56,20 @@ size_t ObjectFileMinidump::GetModuleSpecifications(
 }
 
 bool ObjectFileMinidump::SaveCore(const lldb::ProcessSP &process_sp,
-                                  const lldb_private::FileSpec &outfile,
-                                  lldb::SaveCoreStyle &core_style,
+                                  const lldb_private::SaveCoreOptions &options,
                                   lldb_private::Status &error) {
-  // Set default core style if it isn't set.
+  // Output file and process_sp are both checked in PluginManager::SaveCore.
+  assert(options.GetOutputFile().has_value());
+  assert(process_sp);
+
+  // Minidump defaults to stacks only.
+  SaveCoreStyle core_style = options.GetStyle();
   if (core_style == SaveCoreStyle::eSaveCoreUnspecified)
     core_style = SaveCoreStyle::eSaveCoreStackOnly;
 
-  if (!process_sp)
-    return false;
-
   llvm::Expected<lldb::FileUP> maybe_core_file = FileSystem::Instance().Open(
-      outfile, File::eOpenOptionWriteOnly | File::eOpenOptionCanCreate);
+      options.GetOutputFile().value(),
+      File::eOpenOptionWriteOnly | File::eOpenOptionCanCreate);
   if (!maybe_core_file) {
     error = maybe_core_file.takeError();
     return false;
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h
index b5c40445fe742..0cd31a0e482d5 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h
+++ b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h
@@ -55,8 +55,7 @@ class ObjectFileMinidump : public lldb_private::PluginInterface {
 
   // Saves dump in Minidump file format
   static bool SaveCore(const lldb::ProcessSP &process_sp,
-                       const lldb_private::FileSpec &outfile,
-                       lldb::SaveCoreStyle &core_style,
+                       const lldb_private::SaveCoreOptions &options,
                        lldb_private::Status &error);
 
 private:
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
index be0020cad5bee..bda691ade8af0 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
@@ -355,11 +355,12 @@ size_t ObjectFilePECOFF::GetModuleSpecifications(
 }
 
 bool ObjectFilePECOFF::SaveCore(const lldb::ProcessSP &process_sp,
-                                const lldb_private::FileSpec &outfile,
-                                lldb::SaveCoreStyle &core_style,
+                                const lldb_private::SaveCoreOptions &options,
                                 lldb_private::Status &error) {
-  core_style = eSaveCoreFull;
-  return SaveMiniDump(process_sp, outfile, error);
+  // Outfile and process_sp are validated by PluginManager::SaveCore
+  assert(options.GetOutputFile().has_value());
+  assert(process_sp);
+  return SaveMiniDump(process_sp, options, error);
 }
 
 bool ObjectFilePECOFF::MagicBytesMatch(DataBufferSP data_sp) {
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
index c59701fa65ec3..2eb2b3b774538 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
@@ -82,8 +82,7 @@ class ObjectFilePECOFF : public lldb_private::ObjectFile {
                                         lldb_private::ModuleSpecList &specs);
 
   static bool SaveCore(const lldb::ProcessSP &process_sp,
-                       const lldb_private::FileSpec &outfile,
-                       lldb::SaveCoreStyle &core_style,
+                       const lldb_private::SaveCoreOptions &options,
                        lldb_private::Status &error);
 
   static bool MagicBytesMatch(lldb::DataBufferSP data_sp);
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp
index e5cb36910dd25..61cd74da22223 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp
@@ -21,11 +21,12 @@
 namespace lldb_private {
 
 bool SaveMiniDump(const lldb::ProcessSP &process_sp,
-                  const lldb_private::FileSpec &outfile,
+                  const SaveCoreOptions &core_options,
                   lldb_private::Status &error) {
   if (!process_sp)
     return false;
 #ifdef _WIN32
+  const auto &outfile = core_options.GetOutputFile().value();
   HANDLE process_handle = ::OpenProcess(
       PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, process_sp->GetID());
   const std::string file_name = outfile.GetPath();
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h
index 04b93e221362a..03c0ece306143 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h
@@ -14,7 +14,7 @@
 namespace lldb_private {
 
 bool SaveMiniDump(const lldb::ProcessSP &process_sp,
-                  const lldb_private::FileSpec &outfile,
+                  const SaveCoreOptions &core_options,
                   lldb_private::Status &error);
 
 } // namespace lldb_private
diff --git a/lldb/source/Symbol/CMakeLists.txt b/lldb/source/Symbol/CMakeLists.txt
index ad3488dcdfcec..e49477df06c94 100644
--- a/lldb/source/Symbol/CMakeLists.txt
+++ b/lldb/source/Symbol/CMakeLists.txt
@@ -6,6 +6,7 @@ add_lldb_library(lldbSymbol NO_PLUGIN_DEPENDENCIES
   CompilerDecl.cpp
   CompilerDeclContext.cpp
   CompilerType.cpp
+  SaveCoreOptions.cpp
   DWARFCallFrameInfo.cpp
   DebugMacros.cpp
   DeclVendor.cpp
diff --git a/lldb/source/Symbol/SaveCoreOptions.cpp b/lldb/source/Symbol/SaveCoreOptions.cpp
new file mode 100644
index 0000000000000..0f6fdac1ce22e
--- /dev/null
+++ b/lldb/source/Symbol/SaveCoreOptions.cpp
@@ -0,0 +1,53 @@
+//===-- SaveCoreOptions.cpp -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Symbol/SaveCoreOptions.h"
+#include "lldb/Core/PluginManager.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+Status SaveCoreOptions::SetPluginName(const char *name) {
+  Status error;
+  if (!name || !name[0]) {
+    m_plugin_name = std::nullopt;
+    return error;
+  }
+
+  if (!PluginManager::IsRegisteredObjectFilePluginName(name)) {
+    error.SetErrorStringWithFormat(
+        "plugin name '%s' is not a valid ObjectFile plugin name", name);
+    return error;
+  }
+
+  m_plugin_name = name;
+  return error;
+}
+
+void SaveCoreOptions::SetStyle(lldb::SaveCoreStyle style) { m_style = style; }
+
+void SaveCoreOptions::SetOutputFile(FileSpec file) { m_file = file; }
+
+std::optional<std::string> SaveCoreOptions::GetPluginName() const {
+  return m_plugin_name;
+}
+
+lldb::SaveCoreStyle SaveCoreOptions::GetStyle() const {
+  return m_style.value_or(lldb::eSaveCoreUnspecified);
+}
+
+const std::optional<lldb_private::FileSpec>
+SaveCoreOptions::GetOutputFile() const {
+  return m_file;
+}
+
+void SaveCoreOptions::Clear() {
+  m_file = std::nullopt;
+  m_plugin_name = std::nullopt;
+  m_style = std::nullopt;
+}
diff --git a/lldb/test/API/functionalities/process_save_core/TestProcessSaveCore.py b/lldb/test/API/functionalities/process_save_core/TestProcessSaveCore.py
index c3f0e7597758a..07d06bdc116ec 100644
--- a/lldb/test/API/functionalities/process_save_core/TestProcessSaveCore.py
+++ b/lldb/test/API/functionalities/process_save_core/TestProcessSaveCore.py
@@ -2,7 +2,6 @@
 Test saving a core file (or mini dump).
 """
 
-
 import os
 import lldb
 from lldbsuite.test.decorators import *
@@ -21,6 +20,8 @@ def test_cannot_save_core_unless_process_stopped(self):
         target = self.dbg.CreateTarget(exe)
         process = target.LaunchSimple(None, None, self.get_process_working_directory())
         self.assertNotEqual(process.GetState(), lldb.eStateStopped)
+        options = SBSaveCoreOptions()
+        options.SetOutputFile(SBFileSpec(core))
         error = process.SaveCore(core)
         self.assertTrue(error.Fail())
 
diff --git a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
index 1e171e726fb6b..96511d790271f 100644
--- a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
+++ b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
@@ -132,8 +132,13 @@ def test_save_linux_mini_dump(self):
                 stacks_to_sp_map,
             )
 
+            options = lldb.SBSaveCoreOptions()
+            core_sb_stack_spec = lldb.SBFileSpec(core_sb_stack)
+            options.SetOutputFile(core_sb_stack_spec)
+            options.SetPluginName("minidump")
+            options.SetStyle(lldb.eSaveCoreStackOnly)
             # validate saving via SBProcess
-            error = process.SaveCore(core_sb_stack, "minidump", lldb.eSaveCoreStackOnly)
+            error = process.SaveCore(options)
             self.assertTrue(error.Success())
             self.assertTrue(os.path.isfile(core_sb_stack))
             self.verify_core_file(
@@ -144,7 +149,12 @@ def test_save_linux_mini_dump(self):
                 stacks_to_sp_map,
             )
 
-            error = process.SaveCore(core_sb_dirty, "minidump", lldb.eSaveCoreDirtyOnly)
+            options = lldb.SBSaveCoreOptions()
+            core_sb_dirty_spec = lldb.SBFileSpec(core_sb_dirty)
+            options.SetOutputFile(core_sb_dirty_spec)
+            options.SetPluginName("minidump")
+            options.SetStyle(lldb.eSaveCoreDirtyOnly)
+            error = process.SaveCore(options)
             self.assertTrue(error.Success())
             self.assertTrue(os.path.isfile(core_sb_dirty))
             self.verify_core_file(
@@ -157,7 +167,12 @@ def test_save_linux_mini_dump(self):
 
             # Minidump can now save full core files, but they will be huge and
             # they might cause this test to timeout.
-            error = process.SaveCore(core_sb_full, "minidump", lldb.eSaveCoreFull)
+            options = lldb.SBSaveCoreOptions()
+            core_sb_full_spec = lldb.SBFileSpec(core_sb_full)
+            options.SetOutputFile(core_sb_full_spec)
+            options.SetPluginName("minidump")
+            options.SetStyle(lldb.eSaveCoreFull)
+            error = process.SaveCore(options)
             self.assertTrue(error.Success())
             self.assertTrue(os.path.isfile(core_sb_full))
             self.verify_core_file(
diff --git a/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py b/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py
new file mode 100644
index 0000000000000..c509e81d8951a
--- /dev/null
+++ b/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py
@@ -0,0 +1,28 @@
+"""Test the SBSaveCoreOptions APIs."""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+
+
+class SBSaveCoreOptionsAPICase(TestBase):
+    def test_plugin_name_assignment(self):
+        """Test assignment ensuring valid plugin names only."""
+        options = lldb.SBSaveCoreOptions()
+        error = options.SetPluginName(None)
+        self.assertTrue(error.Success())
+        self.assertEqual(options.GetPluginName(), None)
+        error = options.SetPluginName("Not a real plugin")
+        self.assertTrue(error.Fail())
+        self.assertEqual(options.GetPluginName(), None)
+        error = options.SetPluginName("minidump")
+        self.assertTrue(error.Success())
+        self.assertEqual(options.GetPluginName(), "minidump")
+        error = options.SetPluginName("")
+        self.assertTrue(error.Success())
+        self.assertEqual(options.GetPluginName(), None)
+
+    def test_default_corestyle_behavior(self):
+        """Test that the default core style is unspecified."""
+        options = lldb.SBSaveCoreOptions()
+        self.assertEqual(options.GetStyle(), lldb.eSaveCoreUnspecified)

From 9e4c236650ac8220c4121ceb335de7df6ec4cb36 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 19 Jul 2024 00:10:19 +0000
Subject: [PATCH 111/486] [gn build] Port 4120570dc408

---
 llvm/utils/gn/secondary/lldb/source/API/BUILD.gn    | 1 +
 llvm/utils/gn/secondary/lldb/source/Symbol/BUILD.gn | 1 +
 2 files changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/lldb/source/API/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/API/BUILD.gn
index f0bf6a8f3dbaf..148f2cbf72044 100644
--- a/llvm/utils/gn/secondary/lldb/source/API/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/API/BUILD.gn
@@ -86,6 +86,7 @@ target(liblldb_type, "liblldb") {
     "SBQueue.cpp",
     "SBQueueItem.cpp",
     "SBReproducer.cpp",
+    "SBSaveCoreOptions.cpp",
     "SBScriptObject.cpp",
     "SBSection.cpp",
     "SBSourceManager.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/source/Symbol/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Symbol/BUILD.gn
index db769f2b4372e..3b1df82a6b2d2 100644
--- a/llvm/utils/gn/secondary/lldb/source/Symbol/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Symbol/BUILD.gn
@@ -29,6 +29,7 @@ static_library("Symbol") {
     "ObjectContainer.cpp",
     "ObjectFile.cpp",
     "PostfixExpression.cpp",
+    "SaveCoreOptions.cpp",
     "Symbol.cpp",
     "SymbolContext.cpp",
     "SymbolFile.cpp",

From 71ab0dc6f02b7fd07de272dfcd67ccdb2399ab4f Mon Sep 17 00:00:00 2001
From: Sam James <sam@gentoo.org>
Date: Fri, 19 Jul 2024 01:11:30 +0100
Subject: [PATCH 112/486] [docs] Fix goo.gl link in comment for 'Straight-line
 scalar optimizations' paper

goo.gl is going away: https://developers.googleblog.com/en/google-url-shortener-links-will-no-longer-be-available/

Fix goo.gl link from:
- https://goo.gl/4Rb9As
+ https://docs.google.com/document/d/1momWzKFf4D6h8H3YlfgKQ3qeZy5ayvMRh6yR-Xn2hUE

Committed as obvious.

Bug: https://github.com/llvm/llvm-project/issues/99586
---
 llvm/docs/CompileCudaWithLLVM.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/docs/CompileCudaWithLLVM.rst b/llvm/docs/CompileCudaWithLLVM.rst
index 0371d7a3bdfcb..1b1a27255cc40 100644
--- a/llvm/docs/CompileCudaWithLLVM.rst
+++ b/llvm/docs/CompileCudaWithLLVM.rst
@@ -514,7 +514,7 @@ Modern CPUs and GPUs are architecturally quite different, so code that's fast
 on a CPU isn't necessarily fast on a GPU.  We've made a number of changes to
 LLVM to make it generate good GPU code.  Among these changes are:
 
-* `Straight-line scalar optimizations <https://goo.gl/4Rb9As>`_ -- These
+* `Straight-line scalar optimizations <https://docs.google.com/document/d/1momWzKFf4D6h8H3YlfgKQ3qeZy5ayvMRh6yR-Xn2hUE>`_ -- These
   reduce redundancy within straight-line code.
 
 * `Aggressive speculative execution

From 59441f29323c37c7d0b21bcdd3db96506d1b24d2 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 18 Jul 2024 17:21:06 -0700
Subject: [PATCH 113/486] [sanitizer] Use strict-whitespace in tests

---
 .../TestCases/Linux/dump_registers_i386.cpp                 | 4 ++--
 .../TestCases/Linux/dump_registers_x86_64.cpp               | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_i386.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_i386.cpp
index 0a6c3e361732c..74aea4d8b360a 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_i386.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_i386.cpp
@@ -1,7 +1,7 @@
 // Check that sanitizer prints registers dump_registers on dump_registers=1
 // RUN: %clangxx  %s -o %t
-// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-NODUMP
-// RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-DUMP
+// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-NODUMP --strict-whitespace
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-DUMP --strict-whitespace
 //
 // REQUIRES: i386-target-arch
 
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_x86_64.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_x86_64.cpp
index 1805343c23063..3d11ef0e098f1 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_x86_64.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/dump_registers_x86_64.cpp
@@ -1,7 +1,7 @@
 // Check that sanitizer prints registers dump_registers on dump_registers=1
 // RUN: %clangxx  %s -o %t
-// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-NODUMP
-// RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-DUMP
+// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-NODUMP --strict-whitespace
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-DUMP --strict-whitespace
 //
 // REQUIRES: x86_64-target-arch
 
@@ -13,7 +13,7 @@ int main() {
   // CHECK-DUMP-NEXT: rax = {{0x[0-9a-f]+}}  rbx = {{0x[0-9a-f]+}}  rcx = {{0x[0-9a-f]+}}  rdx = {{0x[0-9a-f]+}}
   // CHECK-DUMP-NEXT: rdi = {{0x[0-9a-f]+}}  rsi = {{0x[0-9a-f]+}}  rbp = {{0x[0-9a-f]+}}  rsp = {{0x[0-9a-f]+}}
   // CHECK-DUMP-NEXT:  r8 = {{0x[0-9a-f]+}}   r9 = {{0x[0-9a-f]+}}  r10 = {{0x[0-9a-f]+}}  r11 = {{0x[0-9a-f]+}}
-  // CHECK-DUMP-NEXT: r12 = {{0x[0-9a-f]+}}   r13 = {{0x[0-9a-f]+}} r14 = {{0x[0-9a-f]+}}  r15 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: r12 = {{0x[0-9a-f]+}}  r13 = {{0x[0-9a-f]+}}  r14 = {{0x[0-9a-f]+}}  r15 = {{0x[0-9a-f]+}}
   // CHECK-NODUMP-NOT: Register values
   return 0;
 }

From 914a00a9c94bf1a0b857abdc5a06857914cc46c7 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Thu, 18 Jul 2024 17:23:02 -0700
Subject: [PATCH 114/486] [bazel] Add support for pybind (#98398)

Previously these targets were disabled, but with a relatively new
rules_python we can build these pointing at a hermetic python, which
allows us to build these safely. Users can still access the files
directly if they need to customize how these are built.
---
 utils/bazel/WORKSPACE                         | 26 +++++++
 .../llvm-project-overlay/mlir/BUILD.bazel     | 73 ++-----------------
 utils/bazel/third_party_build/pybind.BUILD    | 15 ++++
 3 files changed, 47 insertions(+), 67 deletions(-)
 create mode 100644 utils/bazel/third_party_build/pybind.BUILD

diff --git a/utils/bazel/WORKSPACE b/utils/bazel/WORKSPACE
index 298b64fd56291..3822bcff61f47 100644
--- a/utils/bazel/WORKSPACE
+++ b/utils/bazel/WORKSPACE
@@ -27,6 +27,14 @@ load("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure")
 
 llvm_configure(name = "llvm-project")
 
+maybe(
+    http_archive,
+    name = "rules_python",
+    sha256 = "778aaeab3e6cfd56d681c89f5c10d7ad6bf8d2f1a72de9de55b23081b2d31618",
+    strip_prefix = "rules_python-0.34.0",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.34.0/rules_python-0.34.0.tar.gz",
+)
+
 maybe(
     http_archive,
     name = "llvm_zlib",
@@ -129,3 +137,21 @@ maybe(
         "https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz",
     ],
 )
+
+maybe(
+    http_archive,
+    name = "pybind11",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:pybind.BUILD",
+    sha256 = "201966a61dc826f1b1879a24a3317a1ec9214a918c8eb035be2f30c3e9cfbdcb",
+    strip_prefix = "pybind11-2.10.3",
+    url = "https://github.com/pybind/pybind11/archive/v2.10.3.zip",
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
+
+py_repositories()
+
+python_register_toolchains(
+    name = "python_3_12",
+    python_version = "3.12",
+)
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 9ec5b793a1723..0d0224e6d6802 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -918,19 +918,6 @@ exports_files(
     glob(["lib/Bindings/Python/**/*.cpp"]),
 )
 
-# In the targets related to Python bindings, the projects @pybind11 and
-# @local_config_python are defined by @pybind11_bazel. The latter contains
-# python headers, and can be configured in an out-of-tree bazel project via
-#
-#   load("@pybind11_bazel//:python_configure.bzl", "python_configure")
-#   python_configure(name = "local_config_python")
-#
-# For more up-to-date instructions, see
-# https://github.com/pybind/pybind11_bazel
-#
-# Some out-of-tree projects alias @python_runtime//:headers to
-# @local_config_python//:python_headers.
-
 filegroup(
     name = "MLIRBindingsPythonHeaderFiles",
     srcs = glob([
@@ -946,16 +933,12 @@ cc_library(
         "include",
         "lib/Bindings/Python",
     ],
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     textual_hdrs = [":MLIRBindingsPythonHeaderFiles"],
     deps = [
         ":CAPIIRHeaders",
         ":CAPITransformsHeaders",
-        "@local_config_python//:python_headers",
         "@pybind11",
+        "@rules_python//python/cc:current_py_cc_headers",
     ],
 )
 
@@ -965,16 +948,12 @@ cc_library(
         "include",
         "lib/Bindings/Python",
     ],
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     textual_hdrs = [":MLIRBindingsPythonHeaderFiles"],
     deps = [
         ":CAPIIR",
         ":CAPITransforms",
-        "@local_config_python//:python_headers",
         "@pybind11",
+        "@rules_python//python/cc:current_py_cc_headers",
     ],
 )
 
@@ -1008,10 +987,6 @@ cc_library(
     srcs = [":MLIRBindingsPythonSourceFiles"],
     copts = PYBIND11_COPTS,
     features = PYBIND11_FEATURES,
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     deps = [
         ":CAPIAsync",
         ":CAPIDebug",
@@ -1021,8 +996,8 @@ cc_library(
         ":Support",
         ":config",
         "//llvm:Support",
-        "@local_config_python//:python_headers",
         "@pybind11",
+        "@rules_python//python/cc:current_py_cc_headers",
     ],
 )
 
@@ -1031,10 +1006,6 @@ cc_library(
     srcs = [":MLIRBindingsPythonSourceFiles"],
     copts = PYBIND11_COPTS,
     features = PYBIND11_FEATURES,
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     deps = [
         ":CAPIAsyncHeaders",
         ":CAPIDebugHeaders",
@@ -1043,8 +1014,8 @@ cc_library(
         ":Support",
         ":config",
         "//llvm:Support",
-        "@local_config_python//:python_headers",
         "@pybind11",
+        "@rules_python//python/cc:current_py_cc_headers",
     ],
 )
 
@@ -1052,10 +1023,6 @@ cc_library(
 # MLIRBindingsPythonCoreNoCAPI.
 cc_library(
     name = "MLIRBindingsPythonCAPIObjects",
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     deps = [
         ":CAPIAsyncObjects",
         ":CAPIDebugObjects",
@@ -1074,10 +1041,6 @@ cc_binary(
     features = PYBIND11_FEATURES,
     linkshared = 1,
     linkstatic = 0,
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     deps = [
         ":MLIRBindingsPythonCore",
         ":MLIRBindingsPythonHeadersAndDeps",
@@ -1091,10 +1054,6 @@ cc_binary(
     features = PYBIND11_FEATURES,
     linkshared = 1,
     linkstatic = 0,
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     deps = [
         ":CAPIIR",
         ":CAPILinalg",
@@ -1109,10 +1068,6 @@ cc_binary(
     features = PYBIND11_FEATURES,
     linkshared = 1,
     linkstatic = 0,
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",
-    ],
     deps = [
         ":CAPIIR",
         ":CAPILLVM",
@@ -1128,10 +1083,6 @@ cc_binary(
     features = PYBIND11_FEATURES,
     linkshared = 1,
     linkstatic = 0,
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     deps = [
         ":CAPIIR",
         ":CAPIQuant",
@@ -1147,10 +1098,6 @@ cc_binary(
     features = PYBIND11_FEATURES,
     linkshared = 1,
     linkstatic = 0,
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     deps = [
         ":CAPIIR",
         ":CAPISparseTensor",
@@ -1167,15 +1114,11 @@ cc_binary(
     features = PYBIND11_FEATURES,
     linkshared = 1,
     linkstatic = 0,
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     deps = [
         ":CAPIExecutionEngine",
         ":MLIRBindingsPythonHeadersAndDeps",
-        "@local_config_python//:python_headers",
         "@pybind11",
+        "@rules_python//python/cc:current_py_cc_headers",
     ],
 )
 
@@ -1187,15 +1130,11 @@ cc_binary(
     features = PYBIND11_FEATURES,
     linkshared = 1,
     linkstatic = 0,
-    tags = [
-        "manual",  # External dependency
-        "nobuildkite",  # TODO(gcmn): Add support for this target
-    ],
     deps = [
         ":CAPILinalg",
         ":MLIRBindingsPythonHeadersAndDeps",
-        "@local_config_python//:python_headers",
         "@pybind11",
+        "@rules_python//python/cc:current_py_cc_headers",
     ],
 )
 
diff --git a/utils/bazel/third_party_build/pybind.BUILD b/utils/bazel/third_party_build/pybind.BUILD
new file mode 100644
index 0000000000000..d9fb431dbabb5
--- /dev/null
+++ b/utils/bazel/third_party_build/pybind.BUILD
@@ -0,0 +1,15 @@
+cc_library(
+    name = "pybind11",
+    hdrs = glob(
+        include = ["include/pybind11/**/*.h"],
+        exclude = [
+            # Deprecated file that just emits a warning
+            "include/pybind11/common.h",
+        ],
+    ),
+    includes = ["include"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@rules_python//python/cc:current_py_cc_headers",
+    ],
+)

From f304b883730785a204a5197dd407305b5b1dfaaa Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Thu, 18 Jul 2024 17:25:37 -0700
Subject: [PATCH 115/486] [bazel] Port #98403 (#99592)

---
 utils/bazel/llvm-project-overlay/lldb/BUILD.bazel | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index cc573864e29b1..6d5a1b8415ab6 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -707,7 +707,9 @@ cc_library(
 
 cc_library(
     name = "Headers",
-    hdrs = glob(["include/lldb/lldb-*.h"]),
+    hdrs = glob(["include/lldb/lldb-*.h"]) + [
+        "include/lldb/Symbol/SaveCoreOptions.h",
+    ],
     strip_include_prefix = "include",
 )
 

From c0725804e6bc0f1b418db096099897e7fb029a0e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 18 Jul 2024 18:06:19 -0700
Subject: [PATCH 116/486] [ADT] Add unit tests for set_subtract (#99561)

This patch adds a couple of unit tests:

- SetSubtractSmallPtrSet exercises the code path involving remove_if,
  added in d772cdd6279de1e578dfdfca7432327a1806c659.  Note that
  SmallPtrSet supports remove_if.

- SetSubtractSmallVector exercises the code path involving
  S1.erase(*SI) and ensures that set_subtract continues to accept S2
  being a vector, which does not have contains.
---
 llvm/unittests/ADT/SetOperationsTest.cpp | 35 ++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/llvm/unittests/ADT/SetOperationsTest.cpp b/llvm/unittests/ADT/SetOperationsTest.cpp
index b3d931cbfd479..da84c8afbd2ba 100644
--- a/llvm/unittests/ADT/SetOperationsTest.cpp
+++ b/llvm/unittests/ADT/SetOperationsTest.cpp
@@ -8,6 +8,8 @@
 
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
@@ -16,6 +18,7 @@
 using namespace llvm;
 
 using testing::IsEmpty;
+using testing::UnorderedElementsAre;
 
 namespace {
 
@@ -199,6 +202,38 @@ TEST(SetOperationsTest, SetSubtract) {
   EXPECT_EQ(ExpectedSet2, Set2);
 }
 
+TEST(SetOperationsTest, SetSubtractSmallPtrSet) {
+  int A[4];
+
+  // Set1.size() < Set2.size()
+  SmallPtrSet<int *, 4> Set1 = {&A[0], &A[1]};
+  SmallPtrSet<int *, 4> Set2 = {&A[1], &A[2], &A[3]};
+  set_subtract(Set1, Set2);
+  EXPECT_THAT(Set1, UnorderedElementsAre(&A[0]));
+
+  // Set1.size() > Set2.size()
+  Set1 = {&A[0], &A[1], &A[2]};
+  Set2 = {&A[0], &A[2]};
+  set_subtract(Set1, Set2);
+  EXPECT_THAT(Set1, UnorderedElementsAre(&A[1]));
+}
+
+TEST(SetOperationsTest, SetSubtractSmallVector) {
+  int A[4];
+
+  // Set1.size() < Set2.size()
+  SmallPtrSet<int *, 4> Set1 = {&A[0], &A[1]};
+  SmallVector<int *> Set2 = {&A[1], &A[2], &A[3]};
+  set_subtract(Set1, Set2);
+  EXPECT_THAT(Set1, UnorderedElementsAre(&A[0]));
+
+  // Set1.size() > Set2.size()
+  Set1 = {&A[0], &A[1], &A[2]};
+  Set2 = {&A[0], &A[2]};
+  set_subtract(Set1, Set2);
+  EXPECT_THAT(Set1, UnorderedElementsAre(&A[1]));
+}
+
 TEST(SetOperationsTest, SetSubtractRemovedRemaining) {
   std::set<int> Removed, Remaining;
 

From bf4347b3da31625ce1ae2dd4ffb5c557072e03c0 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 18 Jul 2024 18:14:54 -0700
Subject: [PATCH 117/486] [sanitizer_common] Use %p to print addresses (#98578)

Pointers print more leading zeroes for better alignment.
---
 .../sanitizer_allocator_primary64.h           |  4 ++--
 .../sanitizer_coverage_libcdep_new.cpp        |  3 ++-
 .../sanitizer_common/sanitizer_libignore.cpp  |  4 ++--
 .../lib/sanitizer_common/sanitizer_mac.cpp    |  4 ++--
 .../lib/sanitizer_common/sanitizer_posix.cpp  |  4 ++--
 .../sanitizer_posix_libcdep.cpp               |  7 +++---
 .../sanitizer_stacktrace_printer.cpp          |  2 +-
 .../sanitizer_stoptheworld_linux_libcdep.cpp  |  4 ++--
 .../sanitizer_stoptheworld_netbsd_libcdep.cpp |  4 ++--
 .../sanitizer_tls_get_addr.cpp                | 14 +++++------
 .../lib/sanitizer_common/sanitizer_win.cpp    |  8 +++----
 .../tests/sanitizer_stackdepot_test.cpp       |  8 +++----
 .../sanitizer_stacktrace_printer_test.cpp     | 23 ++++++++++++-------
 .../tests/sanitizer_stacktrace_test.cpp       |  2 +-
 14 files changed, 50 insertions(+), 41 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
index 16cdc4ce53b35..0b0bdb07041e5 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -316,13 +316,13 @@ class SizeClassAllocator64 {
     Printf(
         "%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd "
         "num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd "
-        "last released: %6lldK region: 0x%zx\n",
+        "last released: %6lldK region: %p\n",
         region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id),
         region->mapped_user >> 10, region->stats.n_allocated,
         region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks,
         rss >> 10, region->rtoi.num_releases,
         region->rtoi.last_released_bytes >> 10,
-        SpaceBeg() + kRegionSize * class_id);
+        (void *)(SpaceBeg() + kRegionSize * class_id));
   }
 
   void PrintStats() {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp
index ce4326967180d..506659a58c45e 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp
@@ -75,7 +75,8 @@ static void SanitizerDumpCoverage(const uptr* unsorted_pcs, uptr len) {
     if (!pc) continue;
 
     if (!GetModuleAndOffsetForPc(pc, nullptr, 0, &pcs[i])) {
-      Printf("ERROR: unknown pc 0x%zx (may happen if dlclose is used)\n", pc);
+      Printf("ERROR: unknown pc %p (may happen if dlclose is used)\n",
+             (void*)pc);
       continue;
     }
     uptr module_base = pc - pcs[i];
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp
index b7fc9444cc66b..f0e1e3d69def5 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp
@@ -105,8 +105,8 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
           continue;
         if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1))
           continue;
-        VReport(1, "Adding instrumented range 0x%zx-0x%zx from library '%s'\n",
-                range.beg, range.end, mod.full_name());
+        VReport(1, "Adding instrumented range %p-%p from library '%s'\n",
+                (void *)range.beg, (void *)range.end, mod.full_name());
         const uptr idx =
             atomic_load(&instrumented_ranges_count_, memory_order_relaxed);
         CHECK_LT(idx, ARRAY_SIZE(instrumented_code_ranges_));
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index cbdf3e95925bf..8ebe37d649415 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -1372,8 +1372,8 @@ void DumpProcessMap() {
   for (uptr i = 0; i < modules.size(); ++i) {
     char uuid_str[128];
     FormatUUID(uuid_str, sizeof(uuid_str), modules[i].uuid());
-    Printf("0x%zx-0x%zx %s (%s) %s\n", modules[i].base_address(),
-           modules[i].max_address(), modules[i].full_name(),
+    Printf("%p-%p %s (%s) %s\n", (void *)modules[i].base_address(),
+           (void *)modules[i].max_address(), modules[i].full_name(),
            ModuleArchToString(modules[i].arch()), uuid_str);
   }
   Printf("End of module map.\n");
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp
index 969327a7a0fbe..7d7d575431994 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp
@@ -130,8 +130,8 @@ static void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem,
     if (tolerate_enomem && reserrno == ENOMEM)
       return nullptr;
     char mem_type[40];
-    internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
-                      fixed_addr);
+    internal_snprintf(mem_type, sizeof(mem_type), "memory at address %p",
+                      (void *)fixed_addr);
     ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
   }
   IncreaseTotalMmap(size);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
index ece2d7d63dd61..9ffb36f812c45 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
@@ -327,9 +327,10 @@ static bool MmapFixed(uptr fixed_addr, uptr size, int additional_flags,
                 MAP_PRIVATE | MAP_FIXED | additional_flags | MAP_ANON, name);
   int reserrno;
   if (internal_iserror(p, &reserrno)) {
-    Report("ERROR: %s failed to "
-           "allocate 0x%zx (%zd) bytes at address %zx (errno: %d)\n",
-           SanitizerToolName, size, size, fixed_addr, reserrno);
+    Report(
+        "ERROR: %s failed to "
+        "allocate 0x%zx (%zd) bytes at address %p (errno: %d)\n",
+        SanitizerToolName, size, size, (void *)fixed_addr, reserrno);
     return false;
   }
   IncreaseTotalMmap(size);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
index b23796fccf6e3..dddae441d5de9 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
@@ -192,7 +192,7 @@ void FormattedStackTracePrinter::RenderFrame(InternalScopedString *buffer,
       buffer->AppendF("%u", frame_no);
       break;
     case 'p':
-      buffer->AppendF("0x%zx", address);
+      buffer->AppendF("%p", (void *)address);
       break;
     case 'm':
       buffer->AppendF("%s", StripPathPrefix(info->module, strip_path_prefix));
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
index 25c4af7085608..526a71c398260 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
@@ -257,8 +257,8 @@ static void TracerThreadDieCallback() {
 static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
                                       void *uctx) {
   SignalContext ctx(siginfo, uctx);
-  Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
-         ctx.addr, ctx.pc, ctx.sp);
+  Printf("Tracer caught signal %d: addr=%p pc=%p sp=%p\n", signum,
+         (void *)ctx.addr, (void *)ctx.pc, (void *)ctx.sp);
   ThreadSuspender *inst = thread_suspender_instance;
   if (inst) {
     if (signum == SIGABRT)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp
index 701db72619a3d..58a0cfdbf9d48 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp
@@ -158,8 +158,8 @@ static void TracerThreadDieCallback() {
 static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
                                       void *uctx) {
   SignalContext ctx(siginfo, uctx);
-  Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
-         ctx.addr, ctx.pc, ctx.sp);
+  Printf("Tracer caught signal %d: addr=%p pc=%p sp=%p\n", signum,
+         (void *)ctx.addr, (void *)ctx.pc, (void *)ctx.sp);
   ThreadSuspender *inst = thread_suspender_instance;
   if (inst) {
     if (signum == SIGABRT)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
index 252979f1c2baa..7fc9edde34c3d 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
@@ -121,25 +121,25 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
   uptr tls_size = 0;
   uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
   VReport(2,
-          "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: 0x%zx; sp: %p "
+          "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: %p; sp: %p "
           "num_live_dtls %zd\n",
-          (void *)arg, arg->dso_id, arg->offset, res, tls_beg, (void *)&tls_beg,
+          (void *)arg, arg->dso_id, arg->offset, res, (void *)tls_beg, &tls_beg,
           atomic_load(&number_of_live_dtls, memory_order_relaxed));
   if (dtls.last_memalign_ptr == tls_beg) {
     tls_size = dtls.last_memalign_size;
-    VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={0x%zx,0x%zx}\n",
-            tls_beg, tls_size);
+    VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={%p,0x%zx}\n",
+            (void *)tls_beg, tls_size);
   } else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
     // This is the static TLS block which was initialized / unpoisoned at thread
     // creation.
-    VReport(2, "__tls_get_addr: static tls: 0x%zx\n", tls_beg);
+    VReport(2, "__tls_get_addr: static tls: %p\n", (void *)tls_beg);
     tls_size = 0;
   } else if (const void *start =
                  __sanitizer_get_allocated_begin((void *)tls_beg)) {
     tls_beg = (uptr)start;
     tls_size = __sanitizer_get_allocated_size(start);
-    VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={0x%zx,0x%zx}\n",
-            tls_beg, tls_size);
+    VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={%p,0x%zx}\n",
+            (void *)tls_beg, tls_size);
   } else {
     VReport(2, "__tls_get_addr: Can't guess glibc version\n");
     // This may happen inside the DTOR of main thread, so just ignore it.
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
index 0b198890fc798..995f00eddc38a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
@@ -276,8 +276,8 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name) {
       MEM_COMMIT, PAGE_READWRITE);
   if (p == 0) {
     char mem_type[30];
-    internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
-                      fixed_addr);
+    internal_snprintf(mem_type, sizeof(mem_type), "memory at address %p",
+                      (void *)fixed_addr);
     ReportMmapFailureAndDie(size, mem_type, "allocate", GetLastError());
   }
   return p;
@@ -308,8 +308,8 @@ void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, const char *name) {
       MEM_COMMIT, PAGE_READWRITE);
   if (p == 0) {
     char mem_type[30];
-    internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
-                      fixed_addr);
+    internal_snprintf(mem_type, sizeof(mem_type), "memory at address %p",
+                      (void *)fixed_addr);
     return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate");
   }
   return p;
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stackdepot_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stackdepot_test.cpp
index e810122a824f6..02833888747ac 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stackdepot_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stackdepot_test.cpp
@@ -98,13 +98,13 @@ TEST_F(StackDepotTest, Print) {
       return s;
     return std::regex_replace(s, std::regex("\\.\\*"), ".*\\n.*");
   };
-  EXPECT_EXIT(
-      (StackDepotPrintAll(), exit(0)), ::testing::ExitedWithCode(0),
-      fix_regex("Stack for id .*#0 0x1.*#1 0x2.*#2 0x3.*#3 0x4.*#4 0x7.*"));
   EXPECT_EXIT(
       (StackDepotPrintAll(), exit(0)), ::testing::ExitedWithCode(0),
       fix_regex(
-          "Stack for id .*#0 0x1.*#1 0x2.*#2 0x3.*#3 0x4.*#4 0x8.*#5 0x9.*"));
+          "Stack for id .*#0 0x0*1.*#1 0x0*2.*#2 0x0*3.*#3 0x0*4.*#4 0x0*7.*"));
+  EXPECT_EXIT((StackDepotPrintAll(), exit(0)), ::testing::ExitedWithCode(0),
+              fix_regex("Stack for id .*#0 0x0*1.*#1 0x0*2.*#2 0x0*3.*#3 "
+                        "0x0*4.*#4 0x0*8.*#5 0x0*9.*"));
 }
 
 TEST_F(StackDepotTest, PrintNoLock) {
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cpp
index 9602ba31621e6..76a434be275e5 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cpp
@@ -11,9 +11,12 @@
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common/sanitizer_stacktrace_printer.h"
 
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include "interception/interception.h"
 
+using testing::MatchesRegex;
+
 namespace __sanitizer {
 
 class TestFormattedStackTracePrinter final : public FormattedStackTracePrinter {
@@ -96,10 +99,12 @@ TEST(FormattedStackTracePrinter, RenderFrame) {
                       "Function:%f FunctionOffset:%q Source:%s Line:%l "
                       "Column:%c",
                       frame_no, info.address, &info, false, "/path/to/");
-  EXPECT_STREQ("% Frame:42 PC:0x400000 Module:my/module ModuleOffset:0x200 "
-               "Function:foo FunctionOffset:0x100 Source:my/source Line:10 "
-               "Column:5",
-               str.data());
+  EXPECT_THAT(
+      str.data(),
+      MatchesRegex(
+          "% Frame:42 PC:0x0*400000 Module:my/module ModuleOffset:0x200 "
+          "Function:foo FunctionOffset:0x100 Source:my/source Line:10 "
+          "Column:5"));
 
   str.clear();
   // Check that RenderFrame() strips interceptor prefixes.
@@ -109,10 +114,12 @@ TEST(FormattedStackTracePrinter, RenderFrame) {
                       "Function:%f FunctionOffset:%q Source:%s Line:%l "
                       "Column:%c",
                       frame_no, info.address, &info, false, "/path/to/");
-  EXPECT_STREQ("% Frame:42 PC:0x400000 Module:my/module ModuleOffset:0x200 "
-               "Function:bar FunctionOffset:0x100 Source:my/source Line:10 "
-               "Column:5",
-               str.data());
+  EXPECT_THAT(
+      str.data(),
+      MatchesRegex(
+          "% Frame:42 PC:0x0*400000 Module:my/module ModuleOffset:0x200 "
+          "Function:bar FunctionOffset:0x100 Source:my/source Line:10 "
+          "Column:5"));
   info.Clear();
   str.clear();
 
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cpp
index 769a9b9a9fa0a..11ca1fd7f0517 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cpp
@@ -284,7 +284,7 @@ TEST(GetCurrentPc, Basic) {
           StackTrace::GetCurrentPc(),
       };
       for (uptr i = 0; i < ARRAY_SIZE(pcs); i++)
-        Printf("pc%zu: 0x%zx\n", i, pcs[i]);
+        Printf("pc%zu: %p\n", i, (void *)(pcs[i]));
       for (uptr i = 1; i < ARRAY_SIZE(pcs); i++) {
         EXPECT_GT(pcs[i], pcs[0]);
         EXPECT_LT(pcs[i], pcs[0] + 1000);

From 467f96951c44fc7851e296ad76228503a49d7675 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 18 Jul 2024 18:22:56 -0700
Subject: [PATCH 118/486] [compiler-rt] Fix a warning

This patch fixes:

  compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp:126:72:
  error: format specifies type 'void *' but the argument has type
  'uptr *' (aka 'unsigned long *') [-Werror,-Wformat-pedantic]
---
 compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
index 7fc9edde34c3d..ee293bbd68875 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
@@ -123,7 +123,8 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
   VReport(2,
           "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: %p; sp: %p "
           "num_live_dtls %zd\n",
-          (void *)arg, arg->dso_id, arg->offset, res, (void *)tls_beg, &tls_beg,
+          (void *)arg, arg->dso_id, arg->offset, res, (void *)tls_beg,
+          (void *)&tls_beg,
           atomic_load(&number_of_live_dtls, memory_order_relaxed));
   if (dtls.last_memalign_ptr == tls_beg) {
     tls_size = dtls.last_memalign_size;

From d4b28fb7516c5a5ecded4154c207288da1949a6b Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 18 Jul 2024 18:33:24 -0700
Subject: [PATCH 119/486] [compiler-rt] Cleanup use of
 COMPILER_RT_INCLUDE_TESTS (#98246)

1. Move checks into parent test/CMakeLists.txt
2. COMPILER_RT_INCLUDE_TESTS disable both lit and
   gtests. Before it was very inconsistent between
   sanitizers.
---
 compiler-rt/cmake/base-config-ix.cmake        |  4 +-
 compiler-rt/test/CMakeLists.txt               |  2 +-
 compiler-rt/test/asan/CMakeLists.txt          | 56 +++++++++----------
 compiler-rt/test/ctx_profile/CMakeLists.txt   | 20 +++----
 compiler-rt/test/fuzzer/CMakeLists.txt        | 18 +++---
 compiler-rt/test/gwp_asan/CMakeLists.txt      |  4 +-
 compiler-rt/test/interception/CMakeLists.txt  |  2 +-
 compiler-rt/test/memprof/CMakeLists.txt       | 20 +++----
 compiler-rt/test/msan/CMakeLists.txt          |  4 +-
 compiler-rt/test/nsan/CMakeLists.txt          |  4 +-
 compiler-rt/test/rtsan/CMakeLists.txt         | 22 ++++----
 .../test/sanitizer_common/CMakeLists.txt      | 16 +++---
 .../test/scudo/standalone/CMakeLists.txt      | 34 ++++++-----
 compiler-rt/test/tsan/CMakeLists.txt          | 22 ++++----
 14 files changed, 103 insertions(+), 125 deletions(-)

diff --git a/compiler-rt/cmake/base-config-ix.cmake b/compiler-rt/cmake/base-config-ix.cmake
index 80bbca1cbfceb..5a97992756a9c 100644
--- a/compiler-rt/cmake/base-config-ix.cmake
+++ b/compiler-rt/cmake/base-config-ix.cmake
@@ -43,7 +43,7 @@ if (LLVM_TREE_AVAILABLE)
   get_clang_resource_dir(COMPILER_RT_OUTPUT_DIR PREFIX ${LLVM_LIBRARY_OUTPUT_INTDIR}/..)
   set(COMPILER_RT_EXEC_OUTPUT_DIR ${LLVM_RUNTIME_OUTPUT_INTDIR})
   get_clang_resource_dir(COMPILER_RT_INSTALL_PATH)
-  option(COMPILER_RT_INCLUDE_TESTS "Generate and build compiler-rt unit tests."
+  option(COMPILER_RT_INCLUDE_TESTS "Generate and build compiler-rt tests."
          ${LLVM_INCLUDE_TESTS})
   option(COMPILER_RT_ENABLE_WERROR "Fail and stop if warning is triggered"
          ${LLVM_ENABLE_WERROR})
@@ -70,7 +70,7 @@ else()
     "Path where built compiler-rt executables should be stored.")
   set(COMPILER_RT_INSTALL_PATH "" CACHE PATH
     "Prefix for directories where built compiler-rt artifacts should be installed.")
-  option(COMPILER_RT_INCLUDE_TESTS "Generate and build compiler-rt unit tests." OFF)
+  option(COMPILER_RT_INCLUDE_TESTS "Generate and build compiler-rt tests." OFF)
   option(COMPILER_RT_ENABLE_WERROR "Fail and stop if warning is triggered" OFF)
   # Use a host compiler to compile/link tests.
   set(COMPILER_RT_TEST_COMPILER ${CMAKE_C_COMPILER} CACHE PATH "Compiler to use for testing")
diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt
index 83e443944c01c..84a98f3674749 100644
--- a/compiler-rt/test/CMakeLists.txt
+++ b/compiler-rt/test/CMakeLists.txt
@@ -48,7 +48,7 @@ umbrella_lit_testsuite_begin(check-compiler-rt)
 
 function(compiler_rt_test_runtime runtime)
   string(TOUPPER ${runtime} runtime_uppercase)
-  if(COMPILER_RT_HAS_${runtime_uppercase})
+  if(COMPILER_RT_HAS_${runtime_uppercase} AND COMPILER_RT_INCLUDE_TESTS)
     if (${runtime} STREQUAL cfi AND NOT COMPILER_RT_HAS_UBSAN)
       # CFI tests require diagnostic mode, which is implemented in UBSan.
     elseif (${runtime} STREQUAL scudo_standalone)
diff --git a/compiler-rt/test/asan/CMakeLists.txt b/compiler-rt/test/asan/CMakeLists.txt
index 2d683e61d6954..fb9e81bbe8082 100644
--- a/compiler-rt/test/asan/CMakeLists.txt
+++ b/compiler-rt/test/asan/CMakeLists.txt
@@ -130,39 +130,37 @@ if(APPLE)
 endif()
 
 # Add unit tests.
-if(COMPILER_RT_INCLUDE_TESTS)
-  foreach(arch ${ASAN_TEST_ARCH})
-    string(TOUPPER ${arch} ARCH_UPPER_CASE)
-    set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
-    set(CONFIG_NAME_DYNAMIC ${ARCH_UPPER_CASE}${OS_NAME}DynamicConfig)
+foreach(arch ${ASAN_TEST_ARCH})
+  string(TOUPPER ${arch} ARCH_UPPER_CASE)
+  set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
+  set(CONFIG_NAME_DYNAMIC ${ARCH_UPPER_CASE}${OS_NAME}DynamicConfig)
 
-    if(NOT MINGW)
-      # MinGW environments don't provide a statically linked CRT, so only the
-      # dynamic asan test configuration can be expected to work.
-      set(ASAN_TEST_DYNAMIC False)
-      configure_lit_site_cfg(
-        ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
-        ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME}/lit.site.cfg.py)
+  if(NOT MINGW)
+    # MinGW environments don't provide a statically linked CRT, so only the
+    # dynamic asan test configuration can be expected to work.
+    set(ASAN_TEST_DYNAMIC False)
+    configure_lit_site_cfg(
+      ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
+      ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME}/lit.site.cfg.py)
+  endif()
+  if(COMPILER_RT_ASAN_HAS_STATIC_RUNTIME)
+    set(ASAN_TEST_DYNAMIC True)
+    configure_lit_site_cfg(
+      ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
+      ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME_DYNAMIC}/lit.site.cfg.py)
+  endif()
+  # FIXME: support unit test in the android test runner
+  if (NOT ANDROID)
+    if (NOT MINGW)
+      list(APPEND ASAN_TEST_DEPS AsanUnitTests)
+      list(APPEND ASAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME})
     endif()
     if(COMPILER_RT_ASAN_HAS_STATIC_RUNTIME)
-      set(ASAN_TEST_DYNAMIC True)
-      configure_lit_site_cfg(
-        ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
-        ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME_DYNAMIC}/lit.site.cfg.py)
+      list(APPEND ASAN_DYNAMIC_TEST_DEPS AsanDynamicUnitTests)
+      list(APPEND ASAN_DYNAMIC_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME_DYNAMIC})
     endif()
-    # FIXME: support unit test in the android test runner
-    if (NOT ANDROID)
-      if (NOT MINGW)
-        list(APPEND ASAN_TEST_DEPS AsanUnitTests)
-        list(APPEND ASAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME})
-      endif()
-      if(COMPILER_RT_ASAN_HAS_STATIC_RUNTIME)
-        list(APPEND ASAN_DYNAMIC_TEST_DEPS AsanDynamicUnitTests)
-        list(APPEND ASAN_DYNAMIC_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME_DYNAMIC})
-      endif()
-    endif()
-  endforeach()
-endif()
+  endif()
+endforeach()
 
 if (SHADOW_MAPPING_UNRELIABLE)
   set(exclude_from_check_all.g "EXCLUDE_FROM_CHECK_ALL")
diff --git a/compiler-rt/test/ctx_profile/CMakeLists.txt b/compiler-rt/test/ctx_profile/CMakeLists.txt
index 371f1a2dcbb05..fc3b3f3ec431f 100644
--- a/compiler-rt/test/ctx_profile/CMakeLists.txt
+++ b/compiler-rt/test/ctx_profile/CMakeLists.txt
@@ -25,17 +25,15 @@ foreach(arch ${CTX_PROFILE_SUPPORTED_ARCH})
 endforeach()
 
 # Add unit tests.
-if(COMPILER_RT_INCLUDE_TESTS)
-  foreach(arch ${CTX_PROFILE_SUPPORTED_ARCH})
-    string(TOUPPER ${arch} ARCH_UPPER_CASE)
-    set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
-    configure_lit_site_cfg(
-      ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
-      ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME}/lit.site.cfg.py)
-    list(APPEND CTX_PROFILE_TEST_DEPS CtxProfileUnitTests)
-    list(APPEND CTX_PROFILE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME})
-  endforeach()
-endif()
+foreach(arch ${CTX_PROFILE_SUPPORTED_ARCH})
+  string(TOUPPER ${arch} ARCH_UPPER_CASE)
+  set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
+  configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
+    ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME}/lit.site.cfg.py)
+  list(APPEND CTX_PROFILE_TEST_DEPS CtxProfileUnitTests)
+  list(APPEND CTX_PROFILE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME})
+endforeach()
 
 add_lit_testsuite(check-ctx_profile "Running the Contextual Profiler tests"
   ${CTX_PROFILE_TESTSUITES}
diff --git a/compiler-rt/test/fuzzer/CMakeLists.txt b/compiler-rt/test/fuzzer/CMakeLists.txt
index 8d29a4bea5f34..b19f52e591fc0 100644
--- a/compiler-rt/test/fuzzer/CMakeLists.txt
+++ b/compiler-rt/test/fuzzer/CMakeLists.txt
@@ -22,20 +22,16 @@ if (APPLE)
   darwin_filter_host_archs(FUZZER_SUPPORTED_ARCH FUZZER_TEST_ARCH)
 endif()
 
-if(COMPILER_RT_INCLUDE_TESTS)
-  list(APPEND LIBFUZZER_TEST_DEPS FuzzerUnitTests)
-  list(APPEND LIBFUZZER_TEST_DEPS FuzzedDataProviderUnitTests)
-endif()
+list(APPEND LIBFUZZER_TEST_DEPS FuzzerUnitTests)
+list(APPEND LIBFUZZER_TEST_DEPS FuzzedDataProviderUnitTests)
 
 set(LIBFUZZER_TESTSUITES)
 
-if(COMPILER_RT_INCLUDE_TESTS)
-  # libFuzzer unit tests.
-  configure_lit_site_cfg(
-    ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg.py)
-  list(APPEND LIBFUZZER_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/unit)
-endif()
+# libFuzzer unit tests.
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg.py)
+list(APPEND LIBFUZZER_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/unit)
 
 macro(test_fuzzer stdlib)
   cmake_parse_arguments(TEST "" "" "DEPS" ${ARGN})
diff --git a/compiler-rt/test/gwp_asan/CMakeLists.txt b/compiler-rt/test/gwp_asan/CMakeLists.txt
index f9580e2592b2f..6ffa36e7ebbda 100644
--- a/compiler-rt/test/gwp_asan/CMakeLists.txt
+++ b/compiler-rt/test/gwp_asan/CMakeLists.txt
@@ -14,7 +14,7 @@ set(GWP_ASAN_TEST_DEPS
 # exported libc++ from the Android NDK is x86-64, even though it's part of the
 # ARM[64] toolchain... See similar measures for ASan and sanitizer-common that
 # disable unit tests for Android.
-if (COMPILER_RT_INCLUDE_TESTS AND NOT ANDROID)
+if (NOT ANDROID)
   list(APPEND GWP_ASAN_TEST_DEPS GwpAsanUnitTests)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.py.in
@@ -22,7 +22,7 @@ if (COMPILER_RT_INCLUDE_TESTS AND NOT ANDROID)
   list(APPEND GWP_ASAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/unit)
 endif()
 
-if (COMPILER_RT_INCLUDE_TESTS AND COMPILER_RT_HAS_SCUDO_STANDALONE)
+if (COMPILER_RT_HAS_SCUDO_STANDALONE)
   foreach(arch ${GWP_ASAN_SUPPORTED_ARCH})
     set(GWP_ASAN_TEST_TARGET_ARCH ${arch})
     string(TOLOWER "-${arch}" GWP_ASAN_TEST_CONFIG_SUFFIX)
diff --git a/compiler-rt/test/interception/CMakeLists.txt b/compiler-rt/test/interception/CMakeLists.txt
index df69453c4fb4e..8ed1a003c1b3c 100644
--- a/compiler-rt/test/interception/CMakeLists.txt
+++ b/compiler-rt/test/interception/CMakeLists.txt
@@ -3,7 +3,7 @@ set(INTERCEPTION_TESTSUITES)
 
 # Unit tests. There are currently no unit tests capable to running on Apple or
 # Android targets.
-if(COMPILER_RT_INCLUDE_TESTS AND NOT ANDROID AND NOT APPLE)
+if(NOT ANDROID AND NOT APPLE)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
     ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py)
diff --git a/compiler-rt/test/memprof/CMakeLists.txt b/compiler-rt/test/memprof/CMakeLists.txt
index 001245215201e..4c50ae6b83719 100644
--- a/compiler-rt/test/memprof/CMakeLists.txt
+++ b/compiler-rt/test/memprof/CMakeLists.txt
@@ -44,17 +44,15 @@ foreach(arch ${MEMPROF_TEST_ARCH})
 endforeach()
 
 # Add unit tests.
-if(COMPILER_RT_INCLUDE_TESTS)
-  foreach(arch ${MEMPROF_TEST_ARCH})
-    string(TOUPPER ${arch} ARCH_UPPER_CASE)
-    set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
-    configure_lit_site_cfg(
-      ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
-      ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME}/lit.site.cfg.py)
-    list(APPEND MEMPROF_TEST_DEPS MemProfUnitTests)
-    list(APPEND MEMPROF_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME})
-  endforeach()
-endif()
+foreach(arch ${MEMPROF_TEST_ARCH})
+  string(TOUPPER ${arch} ARCH_UPPER_CASE)
+  set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
+  configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
+    ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME}/lit.site.cfg.py)
+  list(APPEND MEMPROF_TEST_DEPS MemProfUnitTests)
+  list(APPEND MEMPROF_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME})
+endforeach()
 
 add_lit_testsuite(check-memprof "Running the MemProfiler tests"
   ${MEMPROF_TESTSUITES}
diff --git a/compiler-rt/test/msan/CMakeLists.txt b/compiler-rt/test/msan/CMakeLists.txt
index ff19c11971b2d..df5719761e607 100644
--- a/compiler-rt/test/msan/CMakeLists.txt
+++ b/compiler-rt/test/msan/CMakeLists.txt
@@ -41,9 +41,7 @@ foreach(arch ${MSAN_TEST_ARCH})
   endif()
 endforeach()
 
-if(COMPILER_RT_INCLUDE_TESTS AND
-   COMPILER_RT_LIBCXX_PATH AND
-   COMPILER_RT_LIBCXXABI_PATH)
+if(COMPILER_RT_LIBCXX_PATH AND COMPILER_RT_LIBCXXABI_PATH)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
     ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py)
diff --git a/compiler-rt/test/nsan/CMakeLists.txt b/compiler-rt/test/nsan/CMakeLists.txt
index fb73587574fba..d702e122a85ef 100644
--- a/compiler-rt/test/nsan/CMakeLists.txt
+++ b/compiler-rt/test/nsan/CMakeLists.txt
@@ -3,9 +3,7 @@ set(NSAN_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(NSAN_TESTSUITES)
 set(NSAN_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS} nsan)
 
-if(COMPILER_RT_INCLUDE_TESTS AND
-   COMPILER_RT_LIBCXX_PATH AND
-   COMPILER_RT_LIBCXXABI_PATH)
+if(COMPILER_RT_LIBCXX_PATH AND COMPILER_RT_LIBCXXABI_PATH)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
     ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py)
diff --git a/compiler-rt/test/rtsan/CMakeLists.txt b/compiler-rt/test/rtsan/CMakeLists.txt
index 2a59a57f27372..e1f9eb39408dc 100644
--- a/compiler-rt/test/rtsan/CMakeLists.txt
+++ b/compiler-rt/test/rtsan/CMakeLists.txt
@@ -34,20 +34,18 @@ foreach(arch ${RTSAN_TEST_ARCH})
   list(APPEND RTSAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
 endforeach()
 
-if(COMPILER_RT_INCLUDE_TESTS)
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py)
+if(COMPILER_RT_RTSAN_HAS_STATIC_RUNTIME)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py)
-  if(COMPILER_RT_RTSAN_HAS_STATIC_RUNTIME)
-    configure_lit_site_cfg(
-      ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
-      ${CMAKE_CURRENT_BINARY_DIR}/Unit/dynamic/lit.site.cfg.py)
-  endif()
-  list(APPEND RTSAN_TEST_DEPS RtsanUnitTests)
-  list(APPEND RTSAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit)
-  if(COMPILER_RT_RTSAN_HAS_STATIC_RUNTIME)
-    list(APPEND RTSAN_DYNAMIC_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/dynamic)
-  endif()
+    ${CMAKE_CURRENT_BINARY_DIR}/Unit/dynamic/lit.site.cfg.py)
+endif()
+list(APPEND RTSAN_TEST_DEPS RtsanUnitTests)
+list(APPEND RTSAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit)
+if(COMPILER_RT_RTSAN_HAS_STATIC_RUNTIME)
+  list(APPEND RTSAN_DYNAMIC_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/dynamic)
 endif()
 
 add_lit_testsuite(check-rtsan "Running the Rtsan tests"
diff --git a/compiler-rt/test/sanitizer_common/CMakeLists.txt b/compiler-rt/test/sanitizer_common/CMakeLists.txt
index edecc04d48c75..fa06b82acebd9 100644
--- a/compiler-rt/test/sanitizer_common/CMakeLists.txt
+++ b/compiler-rt/test/sanitizer_common/CMakeLists.txt
@@ -99,15 +99,13 @@ foreach(tool ${SUPPORTED_TOOLS})
 endforeach()
 
 # Unit tests.
-if(COMPILER_RT_INCLUDE_TESTS)
-  configure_lit_site_cfg(
-    ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py)
-  # FIXME: support unit test in the android test runner
-  if (NOT ANDROID)
-    list(APPEND SANITIZER_COMMON_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit)
-    list(APPEND SANITIZER_COMMON_TEST_DEPS SanitizerUnitTests)
-  endif()
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py)
+# FIXME: support unit test in the android test runner
+if (NOT ANDROID)
+  list(APPEND SANITIZER_COMMON_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit)
+  list(APPEND SANITIZER_COMMON_TEST_DEPS SanitizerUnitTests)
 endif()
 
 if(SANITIZER_COMMON_TESTSUITES)
diff --git a/compiler-rt/test/scudo/standalone/CMakeLists.txt b/compiler-rt/test/scudo/standalone/CMakeLists.txt
index 0034cea360f37..3e6c8ab234b7e 100644
--- a/compiler-rt/test/scudo/standalone/CMakeLists.txt
+++ b/compiler-rt/test/scudo/standalone/CMakeLists.txt
@@ -1,21 +1,19 @@
-if(COMPILER_RT_INCLUDE_TESTS)
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg.py)
+list(APPEND SCUDO_STANDALONE_TEST_DEPS ScudoUnitTests)
+list(APPEND SCUDO_STANDALONE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/unit)
+if (COMPILER_RT_HAS_GWP_ASAN)
   configure_lit_site_cfg(
-    ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg.py)
-  list(APPEND SCUDO_STANDALONE_TEST_DEPS ScudoUnitTests)
-  list(APPEND SCUDO_STANDALONE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/unit)
-  if (COMPILER_RT_HAS_GWP_ASAN)
-    configure_lit_site_cfg(
-      ${CMAKE_CURRENT_SOURCE_DIR}/unit/gwp_asan/lit.site.cfg.py.in
-      ${CMAKE_CURRENT_BINARY_DIR}/unit/gwp_asan/lit.site.cfg.py)
-    list(APPEND SCUDO_STANDALONE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/unit/gwp_asan)
-  endif()
+    ${CMAKE_CURRENT_SOURCE_DIR}/unit/gwp_asan/lit.site.cfg.py.in
+    ${CMAKE_CURRENT_BINARY_DIR}/unit/gwp_asan/lit.site.cfg.py)
+  list(APPEND SCUDO_STANDALONE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/unit/gwp_asan)
+endif()
 
-  add_lit_testsuite(check-scudo_standalone
-    "Running Scudo Standalone tests"
-    ${SCUDO_STANDALONE_TESTSUITES}
-    DEPENDS ${SCUDO_STANDALONE_TEST_DEPS})
+add_lit_testsuite(check-scudo_standalone
+  "Running Scudo Standalone tests"
+  ${SCUDO_STANDALONE_TESTSUITES}
+  DEPENDS ${SCUDO_STANDALONE_TEST_DEPS})
 
-  set_target_properties(check-scudo_standalone
-    PROPERTIES FOLDER "Compiler-RT Tests")
-endif()
+set_target_properties(check-scudo_standalone
+  PROPERTIES FOLDER "Compiler-RT Tests")
diff --git a/compiler-rt/test/tsan/CMakeLists.txt b/compiler-rt/test/tsan/CMakeLists.txt
index 31cda53efd47f..163355d68ebc2 100644
--- a/compiler-rt/test/tsan/CMakeLists.txt
+++ b/compiler-rt/test/tsan/CMakeLists.txt
@@ -111,20 +111,18 @@ if(APPLE)
   endforeach()
 endif()
 
-if(COMPILER_RT_INCLUDE_TESTS)
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py)
+if(COMPILER_RT_TSAN_HAS_STATIC_RUNTIME)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py)
-  if(COMPILER_RT_TSAN_HAS_STATIC_RUNTIME)
-    configure_lit_site_cfg(
-      ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
-      ${CMAKE_CURRENT_BINARY_DIR}/Unit/dynamic/lit.site.cfg.py)
-  endif()
-  list(APPEND TSAN_TEST_DEPS TsanUnitTests)
-  list(APPEND TSAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit)
-  if(COMPILER_RT_TSAN_HAS_STATIC_RUNTIME)
-    list(APPEND TSAN_DYNAMIC_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/dynamic)
-  endif()
+    ${CMAKE_CURRENT_BINARY_DIR}/Unit/dynamic/lit.site.cfg.py)
+endif()
+list(APPEND TSAN_TEST_DEPS TsanUnitTests)
+list(APPEND TSAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit)
+if(COMPILER_RT_TSAN_HAS_STATIC_RUNTIME)
+  list(APPEND TSAN_DYNAMIC_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/dynamic)
 endif()
 
 add_lit_testsuite(check-tsan "Running ThreadSanitizer tests"

From 98ebdd0ca9a72c072a1dbe1af2fa7f1adcbb09ae Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 18 Jul 2024 18:38:42 -0700
Subject: [PATCH 120/486] [NFC][sanitizer] Fix `unused variable 'RegName'`
 warning

---
 compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index aa74a4c581dd0..47f9f0c4590fb 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -2186,6 +2186,8 @@ static void DumpSingleReg(ucontext_t *ctx, int RegNum) {
          RegName, ctx->uc_mcontext.gregs[RegNum]);
 #  elif defined(__i386__)
   Printf("%s = 0x%08x  ", RegName, ctx->uc_mcontext.gregs[RegNum]);
+#  else
+  (void)RegName;
 #  endif
 }
 

From cbbd15323469a70bf7d5f1d6e1177bb7f531ea2d Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Thu, 18 Jul 2024 19:03:55 -0700
Subject: [PATCH 121/486] [SandboxIR][Tracker] Track Instruction::moveBefore()
 (#99568)

This implements tracking of moving instrs with `moveBefore()`.
---
 llvm/include/llvm/SandboxIR/Tracker.h    |  20 ++++
 llvm/lib/SandboxIR/SandboxIR.cpp         |   5 +
 llvm/lib/SandboxIR/Tracker.cpp           |  24 +++++
 llvm/unittests/SandboxIR/TrackerTest.cpp | 115 +++++++++++++++++++++++
 4 files changed, 164 insertions(+)

diff --git a/llvm/include/llvm/SandboxIR/Tracker.h b/llvm/include/llvm/SandboxIR/Tracker.h
index 6332b5266bb31..b88eb3d2a5280 100644
--- a/llvm/include/llvm/SandboxIR/Tracker.h
+++ b/llvm/include/llvm/SandboxIR/Tracker.h
@@ -156,6 +156,26 @@ class RemoveFromParent : public IRChangeBase {
 #endif // NDEBUG
 };
 
+class MoveInstr : public IRChangeBase {
+  /// The instruction that moved.
+  Instruction *MovedI;
+  /// This is either the next instruction in the block, or the parent BB if at
+  /// the end of the BB.
+  PointerUnion<Instruction *, BasicBlock *> NextInstrOrBB;
+
+public:
+  MoveInstr(sandboxir::Instruction *I, Tracker &Tracker);
+  void revert() final;
+  void accept() final {}
+#ifndef NDEBUG
+  void dump(raw_ostream &OS) const final {
+    dumpCommon(OS);
+    OS << "MoveInstr";
+  }
+  LLVM_DUMP_METHOD void dump() const final;
+#endif // NDEBUG
+};
+
 /// The tracker collects all the change objects and implements the main API for
 /// saving / reverting / accepting.
 class Tracker {
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 249ebd7366204..87134995a1538 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -371,6 +371,11 @@ void Instruction::moveBefore(BasicBlock &BB, const BBIterator &WhereIt) {
   if (std::next(getIterator()) == WhereIt)
     // Destination is same as origin, nothing to do.
     return;
+
+  auto &Tracker = Ctx.getTracker();
+  if (Tracker.isTracking())
+    Tracker.track(std::make_unique<MoveInstr>(this, Tracker));
+
   auto *LLVMBB = cast<llvm::BasicBlock>(BB.Val);
   llvm::BasicBlock::iterator It;
   if (WhereIt == BB.end()) {
diff --git a/llvm/lib/SandboxIR/Tracker.cpp b/llvm/lib/SandboxIR/Tracker.cpp
index 08ddb7b5c64c9..626c9c27d05e5 100644
--- a/llvm/lib/SandboxIR/Tracker.cpp
+++ b/llvm/lib/SandboxIR/Tracker.cpp
@@ -124,6 +124,30 @@ void RemoveFromParent::dump() const {
 }
 #endif
 
+MoveInstr::MoveInstr(Instruction *MovedI, Tracker &Tracker)
+    : IRChangeBase(Tracker), MovedI(MovedI) {
+  if (auto *NextI = MovedI->getNextNode())
+    NextInstrOrBB = NextI;
+  else
+    NextInstrOrBB = MovedI->getParent();
+}
+
+void MoveInstr::revert() {
+  if (auto *NextI = NextInstrOrBB.dyn_cast<Instruction *>()) {
+    MovedI->moveBefore(NextI);
+  } else {
+    auto *BB = NextInstrOrBB.get<BasicBlock *>();
+    MovedI->moveBefore(*BB, BB->end());
+  }
+}
+
+#ifndef NDEBUG
+void MoveInstr::dump() const {
+  dump(dbgs());
+  dbgs() << "\n";
+}
+#endif
+
 void Tracker::track(std::unique_ptr<IRChangeBase> &&Change) {
   assert(State == TrackerState::Record && "The tracker should be tracking!");
   Changes.push_back(std::move(Change));
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index e5c558345e187..354cd187adb10 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -255,3 +255,118 @@ define i32 @foo(i32 %arg) {
   EXPECT_EQ(&*It++, Ret);
   EXPECT_EQ(It, BB->end());
 }
+
+// TODO: Test multi-instruction patterns.
+TEST_F(TrackerTest, MoveInstr) {
+  parseIR(C, R"IR(
+define i32 @foo(i32 %arg) {
+  %add0 = add i32 %arg, %arg
+  %add1 = add i32 %add0, %arg
+  ret i32 %add1
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto *F = Ctx.createFunction(&LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  sandboxir::Instruction *Add0 = &*It++;
+  sandboxir::Instruction *Add1 = &*It++;
+  sandboxir::Instruction *Ret = &*It++;
+
+  // Check moveBefore(Instruction *) with tracking enabled.
+  Ctx.save();
+  Add1->moveBefore(Add0);
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+  // Check revert().
+  Ctx.revert();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+
+  // Same for the last instruction in the block.
+  Ctx.save();
+  Ret->moveBefore(Add0);
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(It, BB->end());
+  Ctx.revert();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+
+  // Check moveBefore(BasicBlock &, BasicBlock::iterator) with tracking enabled.
+  Ctx.save();
+  Add1->moveBefore(*BB, Add0->getIterator());
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+  // Check revert().
+  Ctx.revert();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+
+  // Same for the last instruction in the block.
+  Ctx.save();
+  Ret->moveBefore(*BB, Add0->getIterator());
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(It, BB->end());
+  // Check revert().
+  Ctx.revert();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+
+  // Check moveAfter(Instruction *) with tracking enabled.
+  Ctx.save();
+  Add0->moveAfter(Add1);
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+  // Check revert().
+  Ctx.revert();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+
+  // Same for the last instruction in the block.
+  Ctx.save();
+  Ret->moveAfter(Add0);
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(It, BB->end());
+  // Check revert().
+  Ctx.revert();
+  It = BB->begin();
+  EXPECT_EQ(&*It++, Add0);
+  EXPECT_EQ(&*It++, Add1);
+  EXPECT_EQ(&*It++, Ret);
+  EXPECT_EQ(It, BB->end());
+}

From 962d018234cb8c94e387fe3950cd030658850541 Mon Sep 17 00:00:00 2001
From: Brian Cain <bcain@quicinc.com>
Date: Thu, 18 Jul 2024 21:12:21 -0500
Subject: [PATCH 122/486] [clang] [hexagon] handle --unwindlib arg (#99552)

Signed-off-by: Brian Cain <bcain@quicinc.com>
---
 .../clang/Basic/DiagnosticDriverKinds.td      |  2 ++
 clang/lib/Driver/ToolChains/Hexagon.cpp       | 18 ++++++++--
 clang/test/Driver/hexagon-toolchain-linux.c   | 33 +++++++++++++++++++
 3 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index cfa897f28b4c0..26a9792e6a608 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -156,6 +156,8 @@ def err_drv_unsupported_rtlib_for_platform : Error<
   "unsupported runtime library '%0' for platform '%1'">;
 def err_drv_invalid_unwindlib_name : Error<
   "invalid unwind library name in argument '%0'">;
+def err_drv_unsupported_unwind_for_platform : Error<
+  "unsupported unwind library '%0' for platform '%1'">;
 def err_drv_incompatible_unwindlib : Error<
   "--rtlib=libgcc requires --unwindlib=libgcc">;
 def err_drv_incompatible_options : Error<
diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp
index 12b3b99df7ca1..29781399cbab4 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -366,11 +366,14 @@ constructHexagonLinkArgs(Compilation &C, const JobAction &JA,
                               options::OPT_t, options::OPT_u_Group});
     AddLinkerInputs(HTC, Inputs, Args, CmdArgs, JA);
 
+    ToolChain::UnwindLibType UNW = HTC.GetUnwindLibType(Args);
+
     if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
       if (NeedsSanitizerDeps) {
         linkSanitizerRuntimeDeps(HTC, Args, CmdArgs);
 
-        CmdArgs.push_back("-lunwind");
+        if (UNW != ToolChain::UNW_None)
+          CmdArgs.push_back("-lunwind");
       }
       if (NeedsXRayDeps)
         linkXRayRuntimeDeps(HTC, Args, CmdArgs);
@@ -618,13 +621,24 @@ HexagonToolChain::~HexagonToolChain() {}
 void HexagonToolChain::AddCXXStdlibLibArgs(const ArgList &Args,
                                            ArgStringList &CmdArgs) const {
   CXXStdlibType Type = GetCXXStdlibType(Args);
+  ToolChain::UnwindLibType UNW = GetUnwindLibType(Args);
+  if (UNW != ToolChain::UNW_None && UNW != ToolChain::UNW_CompilerRT) {
+    const Arg *A = Args.getLastArg(options::OPT_unwindlib_EQ);
+    if (A) {
+      getDriver().Diag(diag::err_drv_unsupported_unwind_for_platform)
+          << A->getValue() << getTriple().normalize();
+      return;
+    }
+  }
+
   switch (Type) {
   case ToolChain::CST_Libcxx:
     CmdArgs.push_back("-lc++");
     if (Args.hasArg(options::OPT_fexperimental_library))
       CmdArgs.push_back("-lc++experimental");
     CmdArgs.push_back("-lc++abi");
-    CmdArgs.push_back("-lunwind");
+    if (UNW != ToolChain::UNW_None)
+      CmdArgs.push_back("-lunwind");
     break;
 
   case ToolChain::CST_Libstdcxx:
diff --git a/clang/test/Driver/hexagon-toolchain-linux.c b/clang/test/Driver/hexagon-toolchain-linux.c
index fe32638417ea4..86cc9a30e932c 100644
--- a/clang/test/Driver/hexagon-toolchain-linux.c
+++ b/clang/test/Driver/hexagon-toolchain-linux.c
@@ -119,3 +119,36 @@
 // CHECK010:   crt1.o
 // CHECK010:   "-L/tmp"
 // CHECK010-NOT:  "-lstandalone"
+
+// -----------------------------------------------------------------------------
+// unwindlib
+// -----------------------------------------------------------------------------
+// RUN: %clangxx --unwindlib=none \
+// RUN:    --target=hexagon-unknown-linux-musl %s -### 2>&1 \
+// RUN:    | FileCheck -check-prefix=CHECK011 %s
+// CHECK011:   InstalledDir: [[INSTALLED_DIR:.+]]
+// CHECK011:   crt1.o
+// CHECK011-NOT:  "-lunwind"
+// CHECK011-NOT:  "-lgcc_eh"
+// CHECK012-NOT:  "-lgcc_s"
+
+
+// RUN: %clangxx --rtlib=compiler-rt --unwindlib=libunwind \
+// RUN:    --target=hexagon-unknown-linux-musl %s -### 2>&1 \
+// RUN:    | FileCheck -check-prefix=CHECK012 %s
+// RUN: %clangxx \
+// RUN:    --target=hexagon-unknown-linux-musl %s -### 2>&1 \
+// RUN:    | FileCheck -check-prefix=CHECK012 %s
+// CHECK012:   InstalledDir: [[INSTALLED_DIR:.+]]
+// CHECK012:   crt1.o
+// CHECK012:  "-lunwind"
+// CHECK012-NOT:  "-lgcc_eh"
+// CHECK012-NOT:  "-lgcc_s"
+
+// RUN: not %clangxx --rtlib=compiler-rt --unwindlib=libgcc \
+// RUN:    --target=hexagon-unknown-linux-musl %s -### 2>&1 \
+// RUN:    | FileCheck -check-prefix=CHECK013 %s
+// CHECK013:  error: unsupported unwind library 'libgcc' for platform 'hexagon-unknown-linux-musl'
+// CHECK013-NOT:  "-lgcc_eh"
+// CHECK013-NOT:  "-lgcc_s"
+// CHECK013-NOT:  "-lunwind"

From e1d0913a85d4034e875cd93ab531c2001afe6620 Mon Sep 17 00:00:00 2001
From: Jordan Rupprecht <rupprecht@google.com>
Date: Thu, 18 Jul 2024 21:56:37 -0500
Subject: [PATCH 123/486] [bazel] Replace git_repository with http_archive.
 (#99422)

The current git_repository usage points to tags, which leads to warnings
that the build may not be reproducable due to not using a git sha.

The docs for
[git_repository](https://bazel.build/rules/lib/repo/git#git_repository)
recommend using `http_archive`, so switch to that instead. Also bump to
newer versions for these two repos.
---
 utils/bazel/WORKSPACE | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/utils/bazel/WORKSPACE b/utils/bazel/WORKSPACE
index 3822bcff61f47..7baca11eed3d3 100644
--- a/utils/bazel/WORKSPACE
+++ b/utils/bazel/WORKSPACE
@@ -2,7 +2,6 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
@@ -84,10 +83,11 @@ apple_support_dependencies()
 # Note: that building from source requires `m4` to be installed on the host machine.
 # This is a known issue: https://github.com/bazelbuild/rules_foreign_cc/issues/755.
 
-git_repository(
+http_archive(
     name = "rules_foreign_cc",
-    remote = "https://github.com/bazelbuild/rules_foreign_cc.git",
-    tag = "0.9.0",
+    sha256 = "4b33d62cf109bcccf286b30ed7121129cc34cf4f4ed9d8a11f38d9108f40ba74",
+    strip_prefix = "rules_foreign_cc-0.11.1",
+    url = "https://github.com/bazelbuild/rules_foreign_cc/releases/download/0.11.1/rules_foreign_cc-0.11.1.tar.gz",
 )
 
 load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
@@ -120,11 +120,12 @@ maybe(
 )
 
 maybe(
-    new_git_repository,
+    http_archive,
     name = "pfm",
     build_file = "@llvm-raw//utils/bazel/third_party_build:pfm.BUILD",
-    remote = "https://git.code.sf.net/p/perfmon2/libpfm4",
-    tag = "v4.12.1",
+    sha256 = "d18b97764c755528c1051d376e33545d0eb60c6ebf85680436813fa5b04cc3d1",
+    strip_prefix = "libpfm-4.13.0",
+    urls = ["https://versaweb.dl.sourceforge.net/project/perfmon2/libpfm4/libpfm-4.13.0.tar.gz"],
 )
 
 maybe(

From 1df2e0c344f0ddf7e09a9c89eba6bbee52142344 Mon Sep 17 00:00:00 2001
From: Allen <zhongyunde@huawei.com>
Date: Fri, 19 Jul 2024 11:19:21 +0800
Subject: [PATCH 124/486] [clang codegen] Emit int TBAA metadata on FP math
 libcall expf (#96025)

Base on the discussion
https://discourse.llvm.org/t/fp-can-we-add-pure-attribute-for-math-library-functions-default/79459,
math libcalls set errno, so it should emit "int" TBAA metadata on FP
libcalls to solve the alias issue.

Note: Only add support for expf in this PR

Fix https://github.com/llvm/llvm-project/issues/86635
---
 clang/lib/CodeGen/CGBuiltin.cpp           | 29 ++++++++++++++++++++++-
 clang/test/CodeGen/math-libcalls-tbaa.cpp |  9 ++++---
 2 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index f426570b17e15..2d32a8582cfdf 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -688,7 +688,34 @@ static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
                               const CallExpr *E, llvm::Constant *calleeValue) {
   CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
   CGCallee callee = CGCallee::forDirect(calleeValue, GlobalDecl(FD));
-  return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
+  RValue Call =
+      CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
+
+  // Check the supported intrinsic.
+  if (unsigned BuiltinID = FD->getBuiltinID()) {
+    auto IsErrnoIntrinsic = [&]() -> unsigned {
+      switch (BuiltinID) {
+      case Builtin::BIexpf:
+      case Builtin::BI__builtin_expf:
+      case Builtin::BI__builtin_expf128:
+        return true;
+      }
+      // TODO: support more FP math libcalls
+      return false;
+    }();
+
+    // Restrict to target with errno, for example, MacOS doesn't set errno.
+    if (IsErrnoIntrinsic && CGF.CGM.getLangOpts().MathErrno &&
+        !CGF.Builder.getIsFPConstrained()) {
+      ASTContext &Context = CGF.getContext();
+      // Emit "int" TBAA metadata on FP math libcalls.
+      clang::QualType IntTy = Context.IntTy;
+      TBAAAccessInfo TBAAInfo = CGF.CGM.getTBAAAccessInfo(IntTy);
+      Instruction *Inst = cast<llvm::Instruction>(Call.getScalarVal());
+      CGF.CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
+    }
+  }
+  return Call;
 }
 
 /// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
diff --git a/clang/test/CodeGen/math-libcalls-tbaa.cpp b/clang/test/CodeGen/math-libcalls-tbaa.cpp
index 0b231d474df77..f15938dee0272 100644
--- a/clang/test/CodeGen/math-libcalls-tbaa.cpp
+++ b/clang/test/CodeGen/math-libcalls-tbaa.cpp
@@ -12,9 +12,8 @@ extern "C" float expf(float);
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[NUM]], i64 40
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2:![0-9]+]]
-// CHECK-NEXT:    [[CALL:%.*]] = tail call float @expf(float noundef [[TMP0]]) #[[ATTR2:[0-9]+]]
-// CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[CALL]], [[TMP1]]
+// CHECK-NEXT:    [[CALL:%.*]] = tail call float @expf(float noundef [[TMP0]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA6:![0-9]+]]
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP0]], [[CALL]]
 // CHECK-NEXT:    ret float [[MUL]]
 //
 extern "C" float foo (float num[], float r2inv, int n) {
@@ -27,11 +26,15 @@ extern "C" float foo (float num[], float r2inv, int n) {
 // NoNewStructPathTBAA: [[META3]] = !{!"float", [[META4:![0-9]+]], i64 0}
 // NoNewStructPathTBAA: [[META4]] = !{!"omnipotent char", [[META5:![0-9]+]], i64 0}
 // NoNewStructPathTBAA: [[META5]] = !{!"Simple C++ TBAA"}
+// NoNewStructPathTBAA: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+// NoNewStructPathTBAA: [[META7]] = !{!"int", [[META4]], i64 0}
 //.
 // NewStructPathTBAA: [[TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0, i64 4}
 // NewStructPathTBAA: [[META3]] = !{[[META4:![0-9]+]], i64 4, !"float"}
 // NewStructPathTBAA: [[META4]] = !{[[META5:![0-9]+]], i64 1, !"omnipotent char"}
 // NewStructPathTBAA: [[META5]] = !{!"Simple C++ TBAA"}
+// NewStructPathTBAA: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0, i64 4}
+// NewStructPathTBAA: [[META7]] = !{[[META4]], i64 4, !"int"}
 //.
 //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 // NewStructPathTBAA: {{.*}}

From e475bb7ac33d7bc0446fe86858f5c3073cd48c97 Mon Sep 17 00:00:00 2001
From: Hristo Hristov <hghristov.rmm@gmail.com>
Date: Fri, 19 Jul 2024 06:38:02 +0300
Subject: [PATCH 125/486] [libc++][memory] P1132R8: `out_ptr` - a scalable
 output pointer abstraction (#73618)

Differential Revision: https://reviews.llvm.org/D150525

Implements:
- https://wg21.link/P1132R8 - `out_ptr` - a scalable output pointer
abstraction
- https://eel.is/c++draft/smartptr.adapt - 20.3.4 Smart pointer adaptors
- https://wg21.link/LWG3734 - Inconsistency in `inout_ptr` and `out_ptr`
for empty case
- https://wg21.link/LWG3897- `inout_ptr` will not update raw pointer to
0

---------

Co-authored-by: Hristo Hristov <zingam@outlook.com>
---
 libcxx/docs/FeatureTestMacroTable.rst         |   4 +-
 libcxx/docs/ReleaseNotes/19.rst               |   1 +
 libcxx/docs/Status/Cxx23Issues.csv            |   4 +-
 libcxx/docs/Status/Cxx23Papers.csv            |   2 +-
 libcxx/docs/Status/Cxx2cIssues.csv            |   2 +-
 libcxx/include/CMakeLists.txt                 |   2 +
 libcxx/include/__memory/allocator_traits.h    |   1 -
 libcxx/include/__memory/inout_ptr.h           | 109 +++++++++
 libcxx/include/__memory/out_ptr.h             | 101 ++++++++
 libcxx/include/__memory/pointer_traits.h      |  70 +++++-
 libcxx/include/memory                         |  18 ++
 libcxx/include/module.modulemap               |   2 +
 libcxx/include/version                        |   4 +-
 libcxx/modules/std/memory.inc                 |  10 +-
 .../memory.version.compile.pass.cpp           |  32 +--
 .../version.version.compile.pass.cpp          |  32 +--
 .../inout_ptr/inout_ptr.general.pass.cpp      | 206 ++++++++++++++++
 .../adapt/inout_ptr/inout_ptr.verify.cpp      |  34 +++
 .../inout_ptr/inout_ptr_t.convert.pass.cpp    |  61 +++++
 .../adapt/inout_ptr/inout_ptr_t.ctor.pass.cpp |  61 +++++
 .../adapt/inout_ptr/inout_ptr_t.verify.cpp    |  29 +++
 .../adapt/out_ptr/out_ptr.general.pass.cpp    | 230 ++++++++++++++++++
 .../smartptr/adapt/out_ptr/out_ptr.verify.cpp |  34 +++
 .../adapt/out_ptr/out_ptr_t.convert.pass.cpp  |  61 +++++
 .../adapt/out_ptr/out_ptr_t.ctor.pass.cpp     |  59 +++++
 .../adapt/out_ptr/out_ptr_t.verify.cpp        |  29 +++
 .../test/std/utilities/smartptr/adapt/types.h |  86 +++++++
 .../generate_feature_test_macro_components.py |   1 -
 28 files changed, 1223 insertions(+), 62 deletions(-)
 create mode 100644 libcxx/include/__memory/inout_ptr.h
 create mode 100644 libcxx/include/__memory/out_ptr.h
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.general.pass.cpp
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.verify.cpp
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.convert.pass.cpp
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.ctor.pass.cpp
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.verify.cpp
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.general.pass.cpp
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.verify.cpp
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.convert.pass.cpp
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.ctor.pass.cpp
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.verify.cpp
 create mode 100644 libcxx/test/std/utilities/smartptr/adapt/types.h

diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 53cfc3739d2be..b548322b1b2b8 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -346,7 +346,7 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_optional``                                     ``202110L``
     ---------------------------------------------------------- -----------------
-    ``__cpp_lib_out_ptr``                                      *unimplemented*
+    ``__cpp_lib_out_ptr``                                      ``202106L``
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_print``                                        ``202207L``
     ---------------------------------------------------------- -----------------
@@ -450,7 +450,7 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_optional_range_support``                       *unimplemented*
     ---------------------------------------------------------- -----------------
-    ``__cpp_lib_out_ptr``                                      *unimplemented*
+    ``__cpp_lib_out_ptr``                                      ``202311L``
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_philox_engine``                                *unimplemented*
     ---------------------------------------------------------- -----------------
diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index ccafa1a1456e0..784934eafe9f7 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -38,6 +38,7 @@ What's New in Libc++ 19.0.0?
 Implemented Papers
 ------------------
 
+- P1132R8 - ``out_ptr`` - a scalable output pointer abstraction
 - P2637R3 - Member ``visit``
 - P2652R2 - Disallow User Specialization of ``allocator_traits``
 - P2819R2 - Add ``tuple`` protocol to ``complex``
diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index cc601b3cd3c96..547d38c25c1da 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -192,7 +192,7 @@
 "`3515 <https://wg21.link/LWG3515>`__","§[stacktrace.basic.nonmem]: ``operator<<`` should be less templatized", "November 2022","","",""
 "`3545 <https://wg21.link/LWG3545>`__","``std::pointer_traits`` should be SFINAE-friendly", "November 2022","|Complete|","18.0",""
 "`3569 <https://wg21.link/LWG3569>`__","``join_view`` fails to support ranges of ranges with non-default_initializable iterators", "November 2022","","","|ranges|"
-"`3594 <https://wg21.link/LWG3594>`__","``inout_ptr`` — inconsistent ``release()`` in destructor", "November 2022","","",""
+"`3594 <https://wg21.link/LWG3594>`__","``inout_ptr`` — inconsistent ``release()`` in destructor", "November 2022","|Complete|","19.0",""
 "`3597 <https://wg21.link/LWG3597>`__","Unsigned integer types don't model advanceable", "November 2022","","","|ranges|"
 "`3600 <https://wg21.link/LWG3600>`__","Making ``istream_iterator`` copy constructor trivial is an ABI break", "November 2022","","",""
 "`3629 <https://wg21.link/LWG3629>`__","``make_error_code`` and ``make_error_condition`` are customization points","November 2022","|Complete|","16.0",""
@@ -282,7 +282,7 @@
 "`3645 <https://wg21.link/LWG3645>`__","``resize_and_overwrite`` is overspecified to call its callback with lvalues","February 2023","|Complete|","14.0",""
 "`3655 <https://wg21.link/LWG3655>`__","The ``INVOKE`` operation and union types","February 2023","|Complete|","18.0",""
 "`3723 <https://wg21.link/LWG3723>`__","``priority_queue::push_range`` needs to ``append_range``","February 2023","","","|ranges|"
-"`3734 <https://wg21.link/LWG3734>`__","Inconsistency in ``inout_ptr`` and ``out_ptr`` for empty case","February 2023","","",""
+"`3734 <https://wg21.link/LWG3734>`__","Inconsistency in ``inout_ptr`` and ``out_ptr`` for empty case","February 2023","|Complete|","19.0",""
 "`3772 <https://wg21.link/LWG3772>`__","``repeat_view``'s ``piecewise`` constructor is missing Postconditions","February 2023","|Complete|","17.0","|ranges|"
 "`3786 <https://wg21.link/LWG3786>`__","Flat maps' deduction guide needs to default ``Allocator`` to be useful","February 2023","","",""
 "`3803 <https://wg21.link/LWG3803>`__","``flat_foo`` constructors taking ``KeyContainer`` lack ``KeyCompare`` parameter","February 2023","","",""
diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index 4f589cd938d7c..6dd6bdad07658 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -13,7 +13,7 @@
 "","","","","","",""
 "`P0401R6 <https://wg21.link/P0401R6>`__","LWG","Providing size feedback in the Allocator interface","June 2021","|Complete|","15.0"
 "`P0448R4 <https://wg21.link/P0448R4>`__","LWG","A strstream replacement using span<charT> as buffer","June 2021","",""
-"`P1132R8 <https://wg21.link/P1132R8>`__","LWG","out_ptr - a scalable output pointer abstraction","June 2021","",""
+"`P1132R8 <https://wg21.link/P1132R8>`__","LWG","out_ptr - a scalable output pointer abstraction","June 2021","|Complete|","19.0"
 "`P1328R1 <https://wg21.link/P1328R1>`__","LWG","Making std::type_info::operator== constexpr","June 2021","|Complete|","17.0"
 "`P1425R4 <https://wg21.link/P1425R4>`__","LWG","Iterators pair constructors for stack and queue","June 2021","|Complete|","14.0","|ranges|"
 "`P1518R2 <https://wg21.link/P1518R2>`__","LWG","Stop overconstraining allocators in container deduction guides","June 2021","|Complete|","13.0"
diff --git a/libcxx/docs/Status/Cxx2cIssues.csv b/libcxx/docs/Status/Cxx2cIssues.csv
index b5732ee981ffb..dec9af1c12656 100644
--- a/libcxx/docs/Status/Cxx2cIssues.csv
+++ b/libcxx/docs/Status/Cxx2cIssues.csv
@@ -24,7 +24,7 @@
 "`3749 <https://wg21.link/LWG3749>`__","``common_iterator`` should handle integer-class difference types","Kona November 2023","","",""
 "`3809 <https://wg21.link/LWG3809>`__","Is ``std::subtract_with_carry_engine<uint16_t>`` supposed to work","Kona November 2023","","",""
 "`3892 <https://wg21.link/LWG3892>`__","Incorrect formatting of nested ranges and tuples","Kona November 2023","|Complete|","17.0","|format|"
-"`3897 <https://wg21.link/LWG3897>`__","``inout_ptr`` will not update raw pointer to 0","Kona November 2023","","",""
+"`3897 <https://wg21.link/LWG3897>`__","``inout_ptr`` will not update raw pointer to 0","Kona November 2023","|Complete|","19.0",""
 "`3946 <https://wg21.link/LWG3946>`__","The definition of ``const_iterator_t`` should be reworked","Kona November 2023","","",""
 "`3947 <https://wg21.link/LWG3947>`__","Unexpected constraints on ``adjacent_transform_view::base()``","Kona November 2023","","","|ranges|"
 "`3948 <https://wg21.link/LWG3948>`__","``possibly-const-range and as-const-pointer`` should be ``noexcept``","Kona November 2023","","","|ranges|"
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 26bad4f656a07..4b036ec639507 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -533,6 +533,8 @@ set(files
   __memory/concepts.h
   __memory/construct_at.h
   __memory/destruct_n.h
+  __memory/inout_ptr.h
+  __memory/out_ptr.h
   __memory/pointer_traits.h
   __memory/ranges_construct_at.h
   __memory/ranges_uninitialized_algorithms.h
diff --git a/libcxx/include/__memory/allocator_traits.h b/libcxx/include/__memory/allocator_traits.h
index ac564f0e6fa0c..c5fcc89327b8f 100644
--- a/libcxx/include/__memory/allocator_traits.h
+++ b/libcxx/include/__memory/allocator_traits.h
@@ -41,7 +41,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD
   struct NAME<_Tp, __void_t<typename _Tp::PROPERTY > > : true_type {}
 
 // __pointer
-_LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_pointer, pointer);
 template <class _Tp,
           class _Alloc,
           class _RawAlloc = __libcpp_remove_reference_t<_Alloc>,
diff --git a/libcxx/include/__memory/inout_ptr.h b/libcxx/include/__memory/inout_ptr.h
new file mode 100644
index 0000000000000..72e1a21ad6867
--- /dev/null
+++ b/libcxx/include/__memory/inout_ptr.h
@@ -0,0 +1,109 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___INOUT_PTR_H
+#define _LIBCPP___INOUT_PTR_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/shared_ptr.h>
+#include <__memory/unique_ptr.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_specialization.h>
+#include <__type_traits/is_void.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Smart, class _Pointer, class... _Args>
+class _LIBCPP_TEMPLATE_VIS inout_ptr_t {
+  static_assert(!__is_specialization_v<_Smart, shared_ptr>, "std::shared_ptr<> is not supported with std::inout_ptr.");
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit inout_ptr_t(_Smart& __smart, _Args... __args)
+      : __s_(__smart), __a_(std::forward<_Args>(__args)...), __p_([&__smart] {
+          if constexpr (is_pointer_v<_Smart>) {
+            return __smart;
+          } else {
+            return __smart.get();
+          }
+        }()) {
+    if constexpr (requires { __s_.release(); }) {
+      __s_.release();
+    } else {
+      __s_ = _Smart();
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI inout_ptr_t(const inout_ptr_t&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI ~inout_ptr_t() {
+    // LWG-3897 inout_ptr will not update raw pointer to null
+    if constexpr (!is_pointer_v<_Smart>) {
+      if (!__p_) {
+        return;
+      }
+    }
+
+    using _SP = __pointer_of_or_t<_Smart, _Pointer>;
+    if constexpr (is_pointer_v<_Smart>) {
+      std::apply([&](auto&&... __args) { __s_ = _Smart(static_cast<_SP>(__p_), std::forward<_Args>(__args)...); },
+                 std::move(__a_));
+    } else if constexpr (__resettable_smart_pointer_with_args<_Smart, _Pointer, _Args...>) {
+      std::apply([&](auto&&... __args) { __s_.reset(static_cast<_SP>(__p_), std::forward<_Args>(__args)...); },
+                 std::move(__a_));
+    } else {
+      static_assert(is_constructible_v<_Smart, _SP, _Args...>,
+                    "The smart pointer must be constructible from arguments of types _Smart, _Pointer, _Args...");
+      std::apply([&](auto&&... __args) { __s_ = _Smart(static_cast<_SP>(__p_), std::forward<_Args>(__args)...); },
+                 std::move(__a_));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI operator _Pointer*() const noexcept { return std::addressof(const_cast<_Pointer&>(__p_)); }
+
+  _LIBCPP_HIDE_FROM_ABI operator void**() const noexcept
+    requires(!is_same_v<_Pointer, void*>)
+  {
+    static_assert(is_pointer_v<_Pointer>, "The conversion to void** requires _Pointer to be a raw pointer.");
+
+    return reinterpret_cast<void**>(static_cast<_Pointer*>(*this));
+  }
+
+private:
+  _Smart& __s_;
+  tuple<_Args...> __a_;
+  _Pointer __p_;
+};
+
+template <class _Pointer = void, class _Smart, class... _Args>
+_LIBCPP_HIDE_FROM_ABI auto inout_ptr(_Smart& __s, _Args&&... __args) {
+  using _Ptr = conditional_t<is_void_v<_Pointer>, __pointer_of_t<_Smart>, _Pointer>;
+  return std::inout_ptr_t<_Smart, _Ptr, _Args&&...>(__s, std::forward<_Args>(__args)...);
+}
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___INOUT_PTR_H
diff --git a/libcxx/include/__memory/out_ptr.h b/libcxx/include/__memory/out_ptr.h
new file mode 100644
index 0000000000000..95aa2029c9231
--- /dev/null
+++ b/libcxx/include/__memory/out_ptr.h
@@ -0,0 +1,101 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___OUT_PTR_H
+#define _LIBCPP___OUT_PTR_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/shared_ptr.h>
+#include <__memory/unique_ptr.h>
+#include <__type_traits/is_specialization.h>
+#include <__type_traits/is_void.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Smart, class _Pointer, class... _Args>
+class _LIBCPP_TEMPLATE_VIS out_ptr_t {
+  static_assert(!__is_specialization_v<_Smart, shared_ptr> || sizeof...(_Args) > 0,
+                "Using std::shared_ptr<> without a deleter in std::out_ptr is not supported.");
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit out_ptr_t(_Smart& __smart, _Args... __args)
+      : __s_(__smart), __a_(std::forward<_Args>(__args)...), __p_() {
+    using _Ptr = decltype(__smart);
+    if constexpr (__resettable_smart_pointer<_Ptr>) {
+      __s_.reset();
+    } else if constexpr (is_constructible_v<_Smart>) {
+      __s_ = _Smart();
+    } else {
+      static_assert(__resettable_smart_pointer<_Ptr> || is_constructible_v<_Smart>,
+                    "The adapted pointer type must have a reset() member function or be default constructible.");
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI out_ptr_t(const out_ptr_t&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI ~out_ptr_t() {
+    if (!__p_) {
+      return;
+    }
+
+    using _SP = __pointer_of_or_t<_Smart, _Pointer>;
+    if constexpr (__resettable_smart_pointer_with_args<_Smart, _Pointer, _Args...>) {
+      std::apply([&](auto&&... __args) { __s_.reset(static_cast<_SP>(__p_), std::forward<_Args>(__args)...); },
+                 std::move(__a_));
+    } else {
+      static_assert(is_constructible_v<_Smart, _SP, _Args...>,
+                    "The smart pointer must be constructible from arguments of types _Smart, _Pointer, _Args...");
+      std::apply([&](auto&&... __args) { __s_ = _Smart(static_cast<_SP>(__p_), std::forward<_Args>(__args)...); },
+                 std::move(__a_));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI operator _Pointer*() const noexcept { return std::addressof(const_cast<_Pointer&>(__p_)); }
+
+  _LIBCPP_HIDE_FROM_ABI operator void**() const noexcept
+    requires(!is_same_v<_Pointer, void*>)
+  {
+    static_assert(is_pointer_v<_Pointer>, "The conversion to void** requires _Pointer to be a raw pointer.");
+
+    return reinterpret_cast<void**>(static_cast<_Pointer*>(*this));
+  }
+
+private:
+  _Smart& __s_;
+  tuple<_Args...> __a_;
+  _Pointer __p_ = _Pointer();
+};
+
+template <class _Pointer = void, class _Smart, class... _Args>
+_LIBCPP_HIDE_FROM_ABI auto out_ptr(_Smart& __s, _Args&&... __args) {
+  using _Ptr = conditional_t<is_void_v<_Pointer>, __pointer_of_t<_Smart>, _Pointer>;
+  return std::out_ptr_t<_Smart, _Ptr, _Args&&...>(__s, std::forward<_Args>(__args)...);
+}
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___OUT_PTR_H
diff --git a/libcxx/include/__memory/pointer_traits.h b/libcxx/include/__memory/pointer_traits.h
index fcd2c3edd9f6a..0914aceb318b7 100644
--- a/libcxx/include/__memory/pointer_traits.h
+++ b/libcxx/include/__memory/pointer_traits.h
@@ -20,19 +20,28 @@
 #include <__type_traits/is_void.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
+#include <__utility/forward.h>
 #include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp, class = void>
-struct __has_element_type : false_type {};
+// clang-format off
+#define _LIBCPP_CLASS_TRAITS_HAS_XXX(NAME, PROPERTY)                                                                   \
+  template <class _Tp, class = void>                                                                                   \
+  struct NAME : false_type {};                                                                                         \
+  template <class _Tp>                                                                                                 \
+  struct NAME<_Tp, __void_t<typename _Tp::PROPERTY> > : true_type {}
+// clang-format on
 
-template <class _Tp>
-struct __has_element_type<_Tp, __void_t<typename _Tp::element_type> > : true_type {};
+_LIBCPP_CLASS_TRAITS_HAS_XXX(__has_pointer, pointer);
+_LIBCPP_CLASS_TRAITS_HAS_XXX(__has_element_type, element_type);
 
 template <class _Ptr, bool = __has_element_type<_Ptr>::value>
 struct __pointer_traits_element_type {};
@@ -240,6 +249,59 @@ to_address(const _Pointer& __p) noexcept -> decltype(std::__to_address(__p)) {
 }
 #endif
 
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Tp>
+struct __pointer_of {};
+
+template <class _Tp>
+  requires(__has_pointer<_Tp>::value)
+struct __pointer_of<_Tp> {
+  using type = typename _Tp::pointer;
+};
+
+template <class _Tp>
+  requires(!__has_pointer<_Tp>::value && __has_element_type<_Tp>::value)
+struct __pointer_of<_Tp> {
+  using type = typename _Tp::element_type*;
+};
+
+template <class _Tp>
+  requires(!__has_pointer<_Tp>::value && !__has_element_type<_Tp>::value &&
+           __has_element_type<pointer_traits<_Tp>>::value)
+struct __pointer_of<_Tp> {
+  using type = typename pointer_traits<_Tp>::element_type*;
+};
+
+template <typename _Tp>
+using __pointer_of_t = typename __pointer_of<_Tp>::type;
+
+template <class _Tp, class _Up>
+struct __pointer_of_or {
+  using type _LIBCPP_NODEBUG = _Up;
+};
+
+template <class _Tp, class _Up>
+  requires requires { typename __pointer_of_t<_Tp>; }
+struct __pointer_of_or<_Tp, _Up> {
+  using type _LIBCPP_NODEBUG = __pointer_of_t<_Tp>;
+};
+
+template <typename _Tp, typename _Up>
+using __pointer_of_or_t = typename __pointer_of_or<_Tp, _Up>::type;
+
+template <class _Smart>
+concept __resettable_smart_pointer = requires(_Smart __s) { __s.reset(); };
+
+template <class _Smart, class _Pointer, class... _Args>
+concept __resettable_smart_pointer_with_args = requires(_Smart __s, _Pointer __p, _Args... __args) {
+  __s.reset(static_cast<__pointer_of_or_t<_Smart, _Pointer>>(__p), std::forward<_Args>(__args)...);
+};
+
+#endif
+
 _LIBCPP_END_NAMESPACE_STD
 
+_LIBCPP_POP_MACROS
+
 #endif // _LIBCPP___MEMORY_POINTER_TRAITS_H
diff --git a/libcxx/include/memory b/libcxx/include/memory
index d52ee7b4c8eee..e1b8462191f84 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -911,6 +911,22 @@ void* align(size_t alignment, size_t size, void*& ptr, size_t& space);
 template<size_t N, class T>
 [[nodiscard]] constexpr T* assume_aligned(T* ptr); // since C++20
 
+// [out.ptr.t], class template out_ptr_t
+template<class Smart, class Pointer, class... Args>
+  class out_ptr_t;                                          // since c++23
+
+// [out.ptr], function template out_ptr
+template<class Pointer = void, class Smart, class... Args>
+  auto out_ptr(Smart& s, Args&&... args);                   // since c++23
+
+// [inout.ptr.t], class template inout_ptr_t
+template<class Smart, class Pointer, class... Args>
+  class inout_ptr_t;                                        // since c++23
+
+// [inout.ptr], function template inout_ptr
+template<class Pointer = void, class Smart, class... Args>
+  auto inout_ptr(Smart& s, Args&&... args);                 // since c++23
+
 }  // std
 
 */
@@ -924,6 +940,8 @@ template<size_t N, class T>
 #include <__memory/allocator_arg_t.h>
 #include <__memory/allocator_traits.h>
 #include <__memory/auto_ptr.h>
+#include <__memory/inout_ptr.h>
+#include <__memory/out_ptr.h>
 #include <__memory/pointer_traits.h>
 #include <__memory/raw_storage_iterator.h>
 #include <__memory/shared_ptr.h>
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 5ed284d80f35e..43ab9c6987c84 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1521,6 +1521,8 @@ module std_private_memory_concepts                        [system] {
 module std_private_memory_construct_at                    [system] { header "__memory/construct_at.h" }
 module std_private_memory_destruct_n                      [system] { header "__memory/destruct_n.h" }
 module std_private_memory_fwd                             [system] { header "__fwd/memory.h" }
+module std_private_memory_inout_ptr                       [system] { header "__memory/inout_ptr.h" }
+module std_private_memory_out_ptr                         [system] { header "__memory/out_ptr.h" }
 module std_private_memory_pointer_traits                  [system] { header "__memory/pointer_traits.h" }
 module std_private_memory_ranges_construct_at             [system] { header "__memory/ranges_construct_at.h" }
 module std_private_memory_ranges_uninitialized_algorithms [system] {
diff --git a/libcxx/include/version b/libcxx/include/version
index 7d9fad1cb1eb2..68aa88a48fa8f 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -477,7 +477,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 // # define __cpp_lib_move_only_function                   202110L
 # undef  __cpp_lib_optional
 # define __cpp_lib_optional                             202110L
-// # define __cpp_lib_out_ptr                              202106L
+# define __cpp_lib_out_ptr                              202106L
 # define __cpp_lib_print                                202207L
 // # define __cpp_lib_ranges_as_const                      202207L
 # define __cpp_lib_ranges_as_rvalue                     202207L
@@ -536,7 +536,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # define __cpp_lib_mdspan                               202406L
 // # define __cpp_lib_optional_range_support               202406L
 # undef  __cpp_lib_out_ptr
-// # define __cpp_lib_out_ptr                              202311L
+# define __cpp_lib_out_ptr                              202311L
 // # define __cpp_lib_philox_engine                        202406L
 // # define __cpp_lib_ranges_concat                        202403L
 # define __cpp_lib_ratio                                202306L
diff --git a/libcxx/modules/std/memory.inc b/libcxx/modules/std/memory.inc
index 56c621c0cf17f..c15f37a21239c 100644
--- a/libcxx/modules/std/memory.inc
+++ b/libcxx/modules/std/memory.inc
@@ -177,17 +177,19 @@ export namespace std {
   // [util.smartptr.atomic], atomic smart pointers
   // using std::atomic;
 
+#if _LIBCPP_STD_VER >= 23
   // [out.ptr.t], class template out_ptr_t
-  //  using std::out_ptr_t;
+  using std::out_ptr_t;
 
   // [out.ptr], function template out_ptr
-  //  using std::out_ptr;
+  using std::out_ptr;
 
   // [inout.ptr.t], class template inout_ptr_t
-  //  using std::inout_ptr_t;
+  using std::inout_ptr_t;
 
   // [inout.ptr], function template inout_ptr
-  //  using std::inout_ptr;
+  using std::inout_ptr;
+#endif // _LIBCPP_STD_VER >= 23
 
 #ifndef _LIBCPP_HAS_NO_THREADS
   // [depr.util.smartptr.shared.atomic]
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.compile.pass.cpp
index 45d9271faa578..aa1170658b103 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.compile.pass.cpp
@@ -485,17 +485,11 @@
 #   error "__cpp_lib_make_unique should have the value 201304L in c++23"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_out_ptr
-#     error "__cpp_lib_out_ptr should be defined in c++23"
-#   endif
-#   if __cpp_lib_out_ptr != 202106L
-#     error "__cpp_lib_out_ptr should have the value 202106L in c++23"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_out_ptr
-#     error "__cpp_lib_out_ptr should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_out_ptr
+#   error "__cpp_lib_out_ptr should be defined in c++23"
+# endif
+# if __cpp_lib_out_ptr != 202106L
+#   error "__cpp_lib_out_ptr should have the value 202106L in c++23"
 # endif
 
 # ifndef __cpp_lib_ranges
@@ -622,17 +616,11 @@
 #   error "__cpp_lib_make_unique should have the value 201304L in c++26"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_out_ptr
-#     error "__cpp_lib_out_ptr should be defined in c++26"
-#   endif
-#   if __cpp_lib_out_ptr != 202311L
-#     error "__cpp_lib_out_ptr should have the value 202311L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_out_ptr
-#     error "__cpp_lib_out_ptr should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_out_ptr
+#   error "__cpp_lib_out_ptr should be defined in c++26"
+# endif
+# if __cpp_lib_out_ptr != 202311L
+#   error "__cpp_lib_out_ptr should have the value 202311L in c++26"
 # endif
 
 # ifndef __cpp_lib_ranges
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index e1af3061725df..945de95df67e1 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -5542,17 +5542,11 @@
 #   error "__cpp_lib_optional_range_support should not be defined before c++26"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_out_ptr
-#     error "__cpp_lib_out_ptr should be defined in c++23"
-#   endif
-#   if __cpp_lib_out_ptr != 202106L
-#     error "__cpp_lib_out_ptr should have the value 202106L in c++23"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_out_ptr
-#     error "__cpp_lib_out_ptr should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_out_ptr
+#   error "__cpp_lib_out_ptr should be defined in c++23"
+# endif
+# if __cpp_lib_out_ptr != 202106L
+#   error "__cpp_lib_out_ptr should have the value 202106L in c++23"
 # endif
 
 # if !defined(_LIBCPP_VERSION)
@@ -7383,17 +7377,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_out_ptr
-#     error "__cpp_lib_out_ptr should be defined in c++26"
-#   endif
-#   if __cpp_lib_out_ptr != 202311L
-#     error "__cpp_lib_out_ptr should have the value 202311L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_out_ptr
-#     error "__cpp_lib_out_ptr should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_out_ptr
+#   error "__cpp_lib_out_ptr should be defined in c++26"
+# endif
+# if __cpp_lib_out_ptr != 202311L
+#   error "__cpp_lib_out_ptr should have the value 202311L in c++26"
 # endif
 
 # if !defined(_LIBCPP_VERSION)
diff --git a/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.general.pass.cpp b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.general.pass.cpp
new file mode 100644
index 0000000000000..d55a0c2abd464
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.general.pass.cpp
@@ -0,0 +1,206 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <memory>
+
+// [inout.ptr], function template inout_ptr
+// template<class Pointer = void, class Smart, class... Args>
+//   auto inout_ptr(Smart& s, Args&&... args);                 // since c++23
+
+#include <cassert>
+#include <memory>
+#include <utility>
+
+#include "../types.h"
+
+// Test updating the ownership of an `inout_ptr_t`-managed pointer for an API with a non-void pointer type.
+// The API returns a new valid object.
+void test_replace_int_p() {
+  auto replace_int_p = [](int** pp) {
+    assert(**pp == 90);
+
+    delete *pp;
+    *pp = new int{84};
+  };
+
+  // raw pointer
+  {
+    auto rPtr = new int{90};
+
+    replace_int_p(std::inout_ptr<int*>(rPtr));
+    assert(*rPtr == 84);
+
+    delete rPtr;
+  }
+
+  // std::unique_ptr
+  {
+    auto uPtr = std::make_unique<int>(90);
+
+    replace_int_p(std::inout_ptr(uPtr));
+    assert(*uPtr == 84);
+  }
+
+  {
+    MoveOnlyDeleter<int> del;
+    std::unique_ptr<int, MoveOnlyDeleter<int>> uPtr{new int{90}};
+
+    replace_int_p(std::inout_ptr(uPtr, std::move(del)));
+    assert(*uPtr == 84);
+    assert(uPtr.get_deleter().wasMoveInitilized == true);
+  }
+
+  // pointer-like ConstructiblePtr
+  {
+    ConstructiblePtr<int> cPtr(new int{90});
+
+    replace_int_p(std::inout_ptr(cPtr));
+    assert(cPtr == 84);
+  }
+
+  // pointer-like ResettablePtr
+  {
+    ResettablePtr<int> rPtr(new int{90});
+
+    replace_int_p(std::inout_ptr(rPtr));
+    assert(rPtr == 84);
+  }
+
+  // pointer-like NonConstructiblePtr
+  {
+    NonConstructiblePtr<int> nPtr;
+    nPtr.reset(new int{90});
+
+    replace_int_p(std::inout_ptr(nPtr));
+    assert(nPtr == 84);
+  }
+}
+
+// Test updating the ownership of an `inout_ptr_t`-managed pointer for an API with a non-void pointer type.
+// The API returns `nullptr`.
+void test_replace_int_p_with_nullptr() {
+  auto replace_int_p_with_nullptr = [](int** pp) -> void {
+    assert(**pp == 90);
+
+    delete *pp;
+    *pp = nullptr;
+  };
+
+  // raw pointer
+  {
+    // LWG-3897 inout_ptr will not update raw pointer to null
+    auto rPtr = new int{90};
+
+    replace_int_p_with_nullptr(std::inout_ptr<int*>(rPtr));
+    assert(rPtr == nullptr);
+  }
+
+  // std::unique_ptr
+  {
+    auto uPtr = std::make_unique<int>(90);
+
+    replace_int_p_with_nullptr(std::inout_ptr(uPtr));
+    assert(uPtr == nullptr);
+  }
+}
+
+// Test updating the ownership of an `inout_ptr_t`-managed pointer for an API with a void pointer type.
+// The API returns a new valid object.
+void test_replace_int_void_p() {
+  auto replace_int_void_p = [](void** pp) {
+    assert(*(static_cast<int*>(*pp)) == 90);
+
+    delete static_cast<int*>(*pp);
+    *pp = new int{84};
+  };
+
+  // raw pointer
+  {
+    auto rPtr = new int{90};
+
+    replace_int_void_p(std::inout_ptr<int*>(rPtr));
+    assert(*rPtr == 84);
+
+    delete rPtr;
+  }
+
+  // std::unique_ptr
+  {
+    auto uPtr = std::make_unique<int>(90);
+
+    replace_int_void_p(std::inout_ptr(uPtr));
+    assert(*uPtr == 84);
+  }
+}
+
+// Test updating the ownership of an `inout_ptr_t`-managed pointer for an API with a non-void pointer type.
+// The API returns `nullptr`.
+void test_replace_int_void_p_with_nullptr() {
+  auto replace_int_void_p_with_nullptr = [](void** pp) {
+    assert(*(static_cast<int*>(*pp)) == 90);
+
+    delete static_cast<int*>(*pp);
+    *pp = nullptr;
+  };
+
+  // raw pointer
+  {
+    auto rPtr = new int{90};
+
+    replace_int_void_p_with_nullptr(std::inout_ptr<int*>(rPtr));
+    assert(rPtr == nullptr);
+  }
+
+  // std::unique_ptr
+  {
+    auto uPtr = std::make_unique<int>(90);
+
+    replace_int_void_p_with_nullptr(std::inout_ptr(uPtr));
+    assert(uPtr == nullptr);
+  }
+}
+
+// Test updating the ownership of an `inout_ptr_t`-managed pointer for an API with a void pointer type.
+// The API returns a new valid object.
+void test_replace_nullptr_with_int_p() {
+  auto replace_nullptr_with_int_p = [](int** pp) {
+    assert(*pp == nullptr);
+
+    *pp = new int{84};
+  };
+
+  // raw pointer
+  {
+    int* rPtr = nullptr;
+
+    replace_nullptr_with_int_p(std::inout_ptr<int*>(rPtr));
+    assert(*rPtr == 84);
+
+    delete rPtr;
+  }
+
+  // std::unique_ptr
+  {
+    std::unique_ptr<int> uPtr;
+
+    replace_nullptr_with_int_p(std::inout_ptr(uPtr));
+    assert(*uPtr == 84);
+  }
+}
+
+int main(int, char**) {
+  test_replace_int_p();
+  test_replace_int_p_with_nullptr();
+  test_replace_int_void_p();
+  test_replace_int_void_p_with_nullptr();
+  test_replace_nullptr_with_int_p();
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.verify.cpp b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.verify.cpp
new file mode 100644
index 0000000000000..d6d70782c5eb5
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.verify.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <memory>
+
+// [inout.ptr], function template inout_ptr
+// template<class Pointer = void, class Smart, class... Args>
+//   auto inout_ptr(Smart& s, Args&&... args);                 // since c++23
+
+#include <memory>
+#include <tuple>
+
+#include "../types.h"
+
+int main(int, char**) {
+  // `std::inout_ptr<>` does not support `std::shared_ptr<>`.
+  {
+    std::shared_ptr<int> sPtr;
+
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.*}}std::shared_ptr<> is not supported}}
+    std::ignore = std::inout_ptr(sPtr);
+    // expected-error@*:* {{no matching conversion for functional-style cast from 'std::shared_ptr<int>' to 'std::inout_ptr_t<shared_ptr<int>, _Ptr>' (aka 'inout_ptr_t<std::shared_ptr<int>, int *>'}}
+    std::ignore = std::inout_ptr<int*>(sPtr);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.convert.pass.cpp b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.convert.pass.cpp
new file mode 100644
index 0000000000000..230402af07cfe
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.convert.pass.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <memory>
+
+// [inout.ptr.t], class template inout_ptr_t
+// template<class Smart, class Pointer, class... Args>
+//   class inout_ptr_t;                                        // since c++23
+
+// operator Pointer*() const noexcept;
+// operator void**() const noexcept;
+
+#include <cassert>
+#include <concepts>
+#include <memory>
+
+int main(int, char**) {
+  // operator Pointer*()
+  {
+    std::unique_ptr<int> uPtr = std::make_unique<int>(84);
+
+    const std::inout_ptr_t<std::unique_ptr<int>, int*> ioPtr{uPtr};
+
+    static_assert(noexcept(ioPtr.operator int**()));
+    std::same_as<int**> decltype(auto) pPtr = ioPtr.operator int**();
+
+    assert(**pPtr == 84);
+  }
+
+  {
+    std::unique_ptr<int, std::default_delete<int>> uPtr = std::make_unique<int>(84);
+
+    const std::inout_ptr_t<decltype(uPtr), int*, std::default_delete<int>> ioPtr{uPtr, std::default_delete<int>{}};
+
+    static_assert(noexcept(ioPtr.operator int**()));
+    std::same_as<int**> decltype(auto) pPtr = ioPtr.operator int**();
+
+    assert(**pPtr == 84);
+  }
+
+  // operator void**()
+  {
+    std::unique_ptr<int> uPtr = std::make_unique<int>(84);
+
+    const std::inout_ptr_t<std::unique_ptr<int>, void*> ioPtr{uPtr};
+
+    static_assert(noexcept(ioPtr.operator void**()));
+    std::same_as<void**> decltype(auto) pPtr = ioPtr.operator void**();
+
+    assert(**reinterpret_cast<int**>(pPtr) == 84);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.ctor.pass.cpp b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.ctor.pass.cpp
new file mode 100644
index 0000000000000..c7e173a76e728
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.ctor.pass.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <memory>
+
+// [inout.ptr.t], class template inout_ptr_t
+// template<class Smart, class Pointer, class... Args>
+//   class inout_ptr_t;                                        // since c++23
+
+// explicit inout_ptr_t(Smart&, Args...);
+
+#include <cassert>
+#include <memory>
+
+#include "test_convertible.h"
+#include "../types.h"
+
+int main(int, char**) {
+  {
+    std::unique_ptr<int> uPtr;
+
+    std::inout_ptr_t<std::unique_ptr<int>, int*>{uPtr};
+
+    static_assert(
+        !test_convertible<std::inout_ptr_t<std::unique_ptr<int>, int*>>(), "This constructor must be explicit");
+
+    // Test the state of the pointer after construction. Complete tests are available in inout_ptr.general.pass.cpp.
+    assert(uPtr == nullptr);
+  }
+
+  {
+    auto deleter = [](auto* p) { delete p; };
+    std::unique_ptr<int, decltype(deleter)> uPtr;
+
+    std::inout_ptr_t<std::unique_ptr<int, decltype(deleter)>, int*>{uPtr};
+
+    static_assert(!test_convertible<std::inout_ptr_t<std::unique_ptr<int, decltype(deleter)>, int*>>(),
+                  "This constructor must be explicit");
+
+    // Test the state of the pointer after construction. Complete tests are available in inout_ptr.general.pass.cpp.
+    assert(uPtr == nullptr);
+  }
+
+  {
+    std::unique_ptr<int, MoveOnlyDeleter<int>> uPtr;
+
+    std::inout_ptr_t<decltype(uPtr), int*, MoveOnlyDeleter<int>>{uPtr, MoveOnlyDeleter<int>{}};
+
+    // Test the state of the pointer after construction. Complete tests are available in inout_ptr.general.pass.cpp.
+    assert(uPtr == nullptr);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.verify.cpp b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.verify.cpp
new file mode 100644
index 0000000000000..2270c4c3fc785
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr_t.verify.cpp
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <memory>
+
+// [inout.ptr.t], class template inout_ptr_t
+// template<class Smart, class Pointer, class... Args>
+//   class inout_ptr_t;                                        // since c++23
+
+#include <memory>
+
+int main(int, char**) {
+  // `std::inout_ptr<>` does not support `std::shared_ptr<>`.
+  {
+    std::shared_ptr<int> sPtr;
+
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.*}}std::shared_ptr<> is not supported with std::inout_ptr.}}
+    std::inout_ptr_t<std::shared_ptr<int>, int*>{sPtr};
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.general.pass.cpp b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.general.pass.cpp
new file mode 100644
index 0000000000000..a78e22f69abc0
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.general.pass.cpp
@@ -0,0 +1,230 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <memory>
+
+// [out.ptr], function template out_ptr
+// template<class Pointer = void, class Smart, class... Args>
+//   auto out_ptr(Smart& s, Args&&... args);                   // since c++23
+
+#include <cassert>
+#include <memory>
+#include <utility>
+
+#include "../types.h"
+
+// Test updating an `out_ptr_t`-managed pointer for an API with a non-void pointer type.
+// The API returns a new valid object.
+void test_get_int_p() {
+  auto get_int_p = [](int** pp) { *pp = new int{84}; };
+
+  // raw pointer
+  {
+    int* rPtr;
+
+    get_int_p(std::out_ptr<int*>(rPtr));
+    assert(*rPtr == 84);
+
+    delete rPtr;
+  }
+
+  // std::unique_ptr
+  {
+    std::unique_ptr<int> uPtr;
+
+    get_int_p(std::out_ptr(uPtr));
+    assert(*uPtr == 84);
+  }
+
+  {
+    MoveOnlyDeleter<int> del;
+    std::unique_ptr<int, MoveOnlyDeleter<int>> uPtr;
+
+    get_int_p(std::out_ptr(uPtr, std::move(del)));
+    assert(*uPtr == 84);
+    assert(uPtr.get_deleter().wasMoveInitilized == true);
+  }
+
+  // std::shared_ptr
+  {
+    std::shared_ptr<int> sPtr;
+
+    get_int_p(std::out_ptr(sPtr, [](auto* p) {
+      assert(*p == 84);
+
+      delete p;
+    }));
+    assert(*sPtr == 84);
+  }
+
+  // pointer-like ConstructiblePtr
+  {
+    ConstructiblePtr<int> cPtr;
+
+    get_int_p(std::out_ptr(cPtr));
+    assert(cPtr == 84);
+  }
+
+  // pointer-like ResettablePtr
+  {
+    ResettablePtr<int> rPtr{nullptr};
+
+    get_int_p(std::out_ptr(rPtr));
+    assert(rPtr == 84);
+  }
+
+  // NonConstructiblePtr
+  {
+    NonConstructiblePtr<int> nPtr;
+
+    get_int_p(std::out_ptr(nPtr));
+    assert(nPtr == 84);
+  }
+}
+
+// Test updating an `out_ptr_t`-managed pointer for an API with a non-void pointer type.
+// The API returns `nullptr`.
+void test_get_int_p_nullptr() {
+  auto get_int_p_nullptr = [](int** pp) { *pp = nullptr; };
+  // raw pointer
+  {
+    int* rPtr;
+
+    get_int_p_nullptr(std::out_ptr<int*>(rPtr));
+    assert(rPtr == nullptr);
+
+    delete rPtr;
+  }
+
+  // std::unique_ptr
+  {
+    std::unique_ptr<int> uPtr;
+
+    get_int_p_nullptr(std::out_ptr(uPtr));
+    assert(uPtr == nullptr);
+  }
+
+  // std::shared_ptr
+  {
+    std::shared_ptr<int> sPtr;
+
+    get_int_p_nullptr(std::out_ptr(sPtr, [](auto* p) {
+      assert(p == nullptr);
+
+      delete p;
+    }));
+    assert(sPtr == nullptr);
+  }
+}
+
+// Test updating an `out_ptr_t`-managed pointer for an API with a void pointer type.
+// The API returns a new valid object.
+void test_get_int_void_p() {
+  auto get_int_void_p = [](void** pp) { *(reinterpret_cast<int**>(pp)) = new int{84}; };
+
+  // raw pointer
+  {
+    int* rPtr;
+
+    get_int_void_p(std::out_ptr(rPtr));
+    assert(*rPtr == 84);
+
+    delete rPtr;
+  }
+
+  // std::unique_ptr
+  {
+    std::unique_ptr<int> uPtr;
+
+    get_int_void_p(std::out_ptr(uPtr));
+    assert(*uPtr == 84);
+  }
+
+  // std::shared_ptr
+  {
+    std::shared_ptr<int> sPtr;
+
+    get_int_void_p(std::out_ptr(sPtr, [](auto* p) {
+      assert(*p == 84);
+
+      delete p;
+    }));
+    assert(*sPtr == 84);
+  }
+
+  // pointer-like ConstructiblePtr
+  {
+    ConstructiblePtr<int> cPtr;
+
+    get_int_void_p(std::out_ptr(cPtr));
+    assert(cPtr == 84);
+  }
+
+  // pointer-like ResettablePtr
+  {
+    ResettablePtr<int> rPtr{nullptr};
+
+    get_int_void_p(std::out_ptr(rPtr));
+    assert(rPtr == 84);
+  }
+
+  // NonConstructiblePtr
+  {
+    NonConstructiblePtr<int> nPtr;
+
+    get_int_void_p(std::out_ptr(nPtr));
+    assert(nPtr == 84);
+  }
+}
+
+// Test updating an `out_ptr_t`-managed pointer for an API with a void pointer type.
+// The API returns `nullptr`.
+void test_get_int_void_p_nullptr() {
+  auto get_int_void_p_nullptr = [](void** pp) { *pp = nullptr; };
+
+  // raw pointer
+  {
+    int* rPtr;
+
+    get_int_void_p_nullptr(std::out_ptr<int*>(rPtr));
+    assert(rPtr == nullptr);
+
+    delete rPtr;
+  }
+
+  // std::unique_ptr
+  {
+    std::unique_ptr<int> uPtr;
+
+    get_int_void_p_nullptr(std::out_ptr(uPtr));
+    assert(uPtr == nullptr);
+  }
+
+  // std::shared_ptr
+  {
+    std::shared_ptr<int> sPtr;
+
+    get_int_void_p_nullptr(std::out_ptr(sPtr, [](auto* p) {
+      assert(p == nullptr);
+
+      delete p;
+    }));
+    assert(sPtr == nullptr);
+  }
+}
+
+int main(int, char**) {
+  test_get_int_p();
+  test_get_int_p_nullptr();
+  test_get_int_void_p();
+  test_get_int_void_p_nullptr();
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.verify.cpp b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.verify.cpp
new file mode 100644
index 0000000000000..1fe78ecb22789
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.verify.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <memory>
+
+// [out.ptr], function template out_ptr
+// template<class Pointer = void, class Smart, class... Args>
+//   auto out_ptr(Smart& s, Args&&... args);                   // since c++23
+
+#include <memory>
+#include <tuple>
+
+#include "../types.h"
+
+int main(int, char**) {
+  // `std::out_ptr<>` requires `std::shared_ptr<>` with a deleter.
+  {
+    std::shared_ptr<int> sPtr;
+
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.*}}Using std::shared_ptr<> without a deleter in std::out_ptr is not supported.}}
+    std::ignore = std::out_ptr(sPtr);
+    // expected-error@*:* {{no matching conversion for functional-style cast from 'std::shared_ptr<int>' to 'std::out_ptr_t<shared_ptr<int>, _Ptr>' (aka 'out_ptr_t<std::shared_ptr<int>, int *>')}}
+    std::ignore = std::out_ptr<int*>(sPtr);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.convert.pass.cpp b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.convert.pass.cpp
new file mode 100644
index 0000000000000..828a5d0ba31ea
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.convert.pass.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <memory>
+
+// [out.ptr.t], class template out_ptr_t
+// template<class Smart, class Pointer, class... Args>
+//   class out_ptr_t;                                          // since c++23
+
+// operator Pointer*() const noexcept;
+// operator void**() const noexcept;
+
+#include <cassert>
+#include <concepts>
+#include <memory>
+
+int main(int, char**) {
+  // operator Pointer*()
+  {
+    std::unique_ptr<int> uPtr;
+
+    const std::out_ptr_t<std::unique_ptr<int>, int*> oPtr{uPtr};
+
+    static_assert(noexcept(oPtr.operator int**()));
+    std::same_as<int**> decltype(auto) pPtr = oPtr.operator int**();
+
+    assert(*pPtr == nullptr);
+  }
+
+  {
+    std::unique_ptr<int, std::default_delete<int>> uPtr;
+
+    const std::out_ptr_t<decltype(uPtr), int*, std::default_delete<int>> oPtr{uPtr, std::default_delete<int>{}};
+
+    static_assert(noexcept(oPtr.operator int**()));
+    std::same_as<int**> decltype(auto) pPtr = oPtr.operator int**();
+
+    assert(*pPtr == nullptr);
+  }
+
+  // operator void**()
+  {
+    std::unique_ptr<int> uPtr;
+
+    const std::out_ptr_t<std::unique_ptr<int>, void*> oPtr{uPtr};
+
+    static_assert(noexcept(oPtr.operator void**()));
+    std::same_as<void**> decltype(auto) pPtr = oPtr.operator void**();
+
+    assert(*pPtr == nullptr);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.ctor.pass.cpp b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.ctor.pass.cpp
new file mode 100644
index 0000000000000..29e1258845bc9
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.ctor.pass.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <memory>
+
+// [out.ptr.t], class template out_ptr_t
+// template<class Smart, class Pointer, class... Args>
+//   class out_ptr_t;                                          // since c++23
+
+// explicit out_ptr_t(Smart&, Args...);
+
+#include <cassert>
+#include <memory>
+
+#include "test_convertible.h"
+#include "../types.h"
+
+int main(int, char**) {
+  {
+    std::unique_ptr<int> uPtr;
+
+    std::out_ptr_t<std::unique_ptr<int>, int*>{uPtr};
+
+    static_assert(!test_convertible<std::out_ptr_t<std::unique_ptr<int>, int*>>(), "This constructor must be explicit");
+
+    // Test the state of the pointer after construction. Complete tests are available in out_ptr.general.pass.cpp
+    assert(uPtr == nullptr);
+  }
+
+  {
+    std::unique_ptr<int, std::default_delete<int>> uPtr;
+
+    std::out_ptr_t<decltype(uPtr), int*, std::default_delete<int>>{uPtr, std::default_delete<int>{}};
+
+    static_assert(!test_convertible<std::out_ptr_t<decltype(uPtr), int*, std::default_delete<int>>>(),
+                  "This constructor must be explicit");
+
+    // Test the state of the pointer after construction. Complete tests are available in out_ptr.general.pass.cpp
+    assert(uPtr == nullptr);
+  }
+
+  {
+    std::unique_ptr<int, MoveOnlyDeleter<int>> uPtr;
+
+    std::out_ptr_t<decltype(uPtr), int*, MoveOnlyDeleter<int>>{uPtr, MoveOnlyDeleter<int>{}};
+
+    // Test the state of the pointer after construction. Complete tests are available in out_ptr.general.pass.cpp
+    assert(uPtr == nullptr);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.verify.cpp b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.verify.cpp
new file mode 100644
index 0000000000000..cbe96e752ac92
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr_t.verify.cpp
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <memory>
+
+// [out.ptr.t], class template out_ptr_t
+// template<class Smart, class Pointer, class... Args>
+//   class out_ptr_t;                                          // since c++23
+
+#include <memory>
+
+int main(int, char**) {
+  // `std::out_ptr_t<>` requires `std::shared_ptr<>` with a deleter.
+  {
+    std::shared_ptr<int> sPtr;
+
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.*}}Using std::shared_ptr<> without a deleter in std::out_ptr is not supported.}}
+    std::out_ptr_t<std::shared_ptr<int>, int*>{sPtr};
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/types.h b/libcxx/test/std/utilities/smartptr/adapt/types.h
new file mode 100644
index 0000000000000..0da6007ba394f
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/adapt/types.h
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_LIBCXX_UTILITIES_SMARTPTR_ADAPT_TYPES_H
+#define TEST_LIBCXX_UTILITIES_SMARTPTR_ADAPT_TYPES_H
+
+#include <type_traits>
+#include <memory>
+
+#include "test_macros.h"
+
+// Custom deleters.
+
+template <typename T>
+struct MoveOnlyDeleter {
+  MoveOnlyDeleter()                                  = default;
+  MoveOnlyDeleter(const MoveOnlyDeleter&)            = delete;
+  MoveOnlyDeleter& operator=(const MoveOnlyDeleter&) = delete;
+  MoveOnlyDeleter(MoveOnlyDeleter&&) : wasMoveInitilized{true} {}
+  MoveOnlyDeleter& operator=(MoveOnlyDeleter&&) = default;
+
+  void operator()(T* p) const { delete p; }
+
+  bool wasMoveInitilized = false;
+};
+
+// Custom pointer types.
+
+template <typename T>
+struct ConstructiblePtr {
+  using pointer = T*;
+  std::unique_ptr<T> ptr;
+
+  ConstructiblePtr() = default;
+  explicit ConstructiblePtr(T* p) : ptr{p} {}
+
+  auto operator==(T val) { return *ptr == val; }
+
+  auto* get() const { return ptr.get(); }
+
+  void release() { ptr.release(); }
+};
+
+LIBCPP_STATIC_ASSERT(std::is_same_v<std::__pointer_of_t< ConstructiblePtr<int>>, int* >);
+static_assert(std::is_constructible_v< ConstructiblePtr<int>, int* >);
+
+struct ResetArg {};
+
+template <typename T>
+struct ResettablePtr {
+  using element_type = T;
+  std::unique_ptr<T> ptr;
+
+  explicit ResettablePtr(T* p) : ptr{p} {}
+
+  auto operator*() const { return *ptr; }
+
+  auto operator==(T val) { return *ptr == val; }
+
+  void reset() { ptr.reset(); }
+  void reset(T* p, ResetArg) { ptr.reset(p); }
+
+  auto* get() const { return ptr.get(); }
+
+  void release() { ptr.release(); }
+};
+
+LIBCPP_STATIC_ASSERT(std::is_same_v<std::__pointer_of_t< ResettablePtr<int>>, int* >);
+static_assert(std::is_constructible_v< ResettablePtr<int>, int* >);
+
+template <typename T>
+struct NonConstructiblePtr : public ResettablePtr<T> {
+  NonConstructiblePtr() : NonConstructiblePtr::ResettablePtr(nullptr) {};
+
+  void reset(T* p) { ResettablePtr<T>::ptr.reset(p); }
+};
+
+LIBCPP_STATIC_ASSERT(std::is_same_v<std::__pointer_of_t< NonConstructiblePtr<int>>, int* >);
+static_assert(!std::is_constructible_v< NonConstructiblePtr<int>, int* >);
+
+#endif // TEST_LIBCXX_UTILITIES_SMARTPTR_ADAPT_TYPES_H
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index 65ba2f8455f1b..a09beb29565ad 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -949,7 +949,6 @@ def add_version_header(tc):
                 "c++26": 202311,  # P2833R2 Freestanding Library: inout expected span
             },
             "headers": ["memory"],
-            "unimplemented": True,
         },
         {
             "name": "__cpp_lib_parallel_algorithm",

From 401d7bcabc0affc38a6e52ffad782eedf0a5ec5a Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 19 Jul 2024 03:38:30 +0000
Subject: [PATCH 126/486] [gn build] Port e475bb7ac33d

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 0349e636f8009..27f29c3344bfa 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -604,6 +604,8 @@ if (current_toolchain == default_toolchain) {
       "__memory/concepts.h",
       "__memory/construct_at.h",
       "__memory/destruct_n.h",
+      "__memory/inout_ptr.h",
+      "__memory/out_ptr.h",
       "__memory/pointer_traits.h",
       "__memory/ranges_construct_at.h",
       "__memory/ranges_uninitialized_algorithms.h",

From 3023b15fb1ec00dbe6a1cb630236125f500978ef Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Thu, 9 May 2024 12:35:45 -0700
Subject: [PATCH 127/486] [BOLT] Support POSSIBLE_PIC_FIXED_BRANCH

Detect and support fixed PIC indirect jumps of the following form:
```
movslq  En(%rip), %r1
leaq  PIC_JUMP_TABLE(%rip), %r2
addq  %r2, %r1
jmpq  *%r1
```

with PIC_JUMP_TABLE that looks like following:

```
  JT:  ----------
   E1:| L1 - JT  |
      |----------|
   E2:| L2 - JT  |
      |----------|
      |          |
         ......
   En:| Ln - JT  |
       ----------
```

The code could be produced by compilers, see
https://github.com/llvm/llvm-project/issues/91648.

Test Plan: updated jump-table-fixed-ref-pic.test

Reviewers: maksfb, ayermolo, dcci, rafaelauler

Reviewed By: rafaelauler

Pull Request: https://github.com/llvm/llvm-project/pull/91667
---
 bolt/include/bolt/Core/BinaryContext.h        |   3 +
 bolt/include/bolt/Core/MCPlusBuilder.h        |  13 ++-
 bolt/lib/Core/BinaryContext.cpp               |  10 ++
 bolt/lib/Core/BinaryFunction.cpp              |  47 +++++++-
 bolt/lib/Passes/IndirectCallPromotion.cpp     |   4 +-
 .../Target/AArch64/AArch64MCPlusBuilder.cpp   |  13 ++-
 bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp  |   3 +-
 bolt/lib/Target/X86/X86MCPlusBuilder.cpp      | 105 +++++++++++-------
 .../X86/Inputs/jump-table-fixed-ref-pic.s     |   2 +-
 bolt/test/X86/jump-table-fixed-ref-pic.test   |  12 +-
 10 files changed, 153 insertions(+), 59 deletions(-)

diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 73932c4ca2fb3..befdc9974c683 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -431,6 +431,9 @@ class BinaryContext {
     return nullptr;
   }
 
+  /// Deregister JumpTable registered at a given \p Address and delete it.
+  void deleteJumpTable(uint64_t Address);
+
   unsigned getDWARFEncodingSize(unsigned Encoding) {
     if (Encoding == dwarf::DW_EH_PE_omit)
       return 0;
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index c916c6f95751f..32eda0b283b88 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -58,6 +58,8 @@ enum class IndirectBranchType : char {
   POSSIBLE_PIC_JUMP_TABLE, /// Possibly a jump table for PIC.
   POSSIBLE_GOTO,           /// Possibly a gcc's computed goto.
   POSSIBLE_FIXED_BRANCH,   /// Possibly an indirect branch to a fixed location.
+  POSSIBLE_PIC_FIXED_BRANCH, /// Possibly an indirect jump to a fixed entry in a
+                             /// PIC jump table.
 };
 
 class MCPlusBuilder {
@@ -1474,12 +1476,11 @@ class MCPlusBuilder {
   /// will be set to the different components of the branch.  \p MemLocInstr
   /// is the instruction that loads up the indirect function pointer.  It may
   /// or may not be same as \p Instruction.
-  virtual IndirectBranchType
-  analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin,
-                        InstructionIterator End, const unsigned PtrSize,
-                        MCInst *&MemLocInstr, unsigned &BaseRegNum,
-                        unsigned &IndexRegNum, int64_t &DispValue,
-                        const MCExpr *&DispExpr, MCInst *&PCRelBaseOut) const {
+  virtual IndirectBranchType analyzeIndirectBranch(
+      MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
+      const unsigned PtrSize, MCInst *&MemLocInstr, unsigned &BaseRegNum,
+      unsigned &IndexRegNum, int64_t &DispValue, const MCExpr *&DispExpr,
+      MCInst *&PCRelBaseOut, MCInst *&FixedEntryLoadInst) const {
     llvm_unreachable("not implemented");
     return IndirectBranchType::UNKNOWN;
   }
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 0a1f1bb9e0d20..035f68e39751b 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -2523,6 +2523,16 @@ BinaryFunction *BinaryContext::getBinaryFunctionAtAddress(uint64_t Address) {
   return nullptr;
 }
 
+/// Deregister JumpTable registered at a given \p Address and delete it.
+void BinaryContext::deleteJumpTable(uint64_t Address) {
+  assert(JumpTables.count(Address) && "Must have a jump table at address");
+  JumpTable *JT = JumpTables.at(Address);
+  for (BinaryFunction *Parent : JT->Parents)
+    Parent->JumpTables.erase(Address);
+  JumpTables.erase(Address);
+  delete JT;
+}
+
 DebugAddressRangesVector BinaryContext::translateModuleAddressRanges(
     const DWARFAddressRangesVector &InputRanges) const {
   DebugAddressRangesVector OutputRanges;
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 282638f9dc76f..ea09371b57e8a 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -780,6 +780,9 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
   // setting the value of the register used by the branch.
   MCInst *MemLocInstr;
 
+  // The instruction loading the fixed PIC jump table entry value.
+  MCInst *FixedEntryLoadInstr;
+
   // Address of the table referenced by MemLocInstr. Could be either an
   // array of function pointers, or a jump table.
   uint64_t ArrayStart = 0;
@@ -811,7 +814,7 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
 
   IndirectBranchType BranchType = BC.MIB->analyzeIndirectBranch(
       Instruction, Begin, Instructions.end(), PtrSize, MemLocInstr, BaseRegNum,
-      IndexRegNum, DispValue, DispExpr, PCRelBaseInstr);
+      IndexRegNum, DispValue, DispExpr, PCRelBaseInstr, FixedEntryLoadInstr);
 
   if (BranchType == IndirectBranchType::UNKNOWN && !MemLocInstr)
     return BranchType;
@@ -877,6 +880,43 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
   if (BaseRegNum == BC.MRI->getProgramCounter())
     ArrayStart += getAddress() + Offset + Size;
 
+  if (FixedEntryLoadInstr) {
+    assert(BranchType == IndirectBranchType::POSSIBLE_PIC_FIXED_BRANCH &&
+           "Invalid IndirectBranch type");
+    MCInst::iterator FixedEntryDispOperand =
+        BC.MIB->getMemOperandDisp(*FixedEntryLoadInstr);
+    assert(FixedEntryDispOperand != FixedEntryLoadInstr->end() &&
+           "Invalid memory instruction");
+    const MCExpr *FixedEntryDispExpr = FixedEntryDispOperand->getExpr();
+    const uint64_t EntryAddress = getExprValue(FixedEntryDispExpr);
+    uint64_t EntrySize = BC.getJumpTableEntrySize(JumpTable::JTT_PIC);
+    ErrorOr<int64_t> Value =
+        BC.getSignedValueAtAddress(EntryAddress, EntrySize);
+    if (!Value)
+      return IndirectBranchType::UNKNOWN;
+
+    BC.outs() << "BOLT-INFO: fixed PIC indirect branch detected in " << *this
+              << " at 0x" << Twine::utohexstr(getAddress() + Offset)
+              << " referencing data at 0x" << Twine::utohexstr(EntryAddress)
+              << " the destination value is 0x"
+              << Twine::utohexstr(ArrayStart + *Value) << '\n';
+
+    TargetAddress = ArrayStart + *Value;
+
+    // Remove spurious JumpTable at EntryAddress caused by PIC reference from
+    // the load instruction.
+    BC.deleteJumpTable(EntryAddress);
+
+    // Replace FixedEntryDispExpr used in target address calculation with outer
+    // jump table reference.
+    JumpTable *JT = BC.getJumpTableContainingAddress(ArrayStart);
+    assert(JT && "Must have a containing jump table for PIC fixed branch");
+    BC.MIB->replaceMemOperandDisp(*FixedEntryLoadInstr, JT->getFirstLabel(),
+                                  EntryAddress - ArrayStart, &*BC.Ctx);
+
+    return BranchType;
+  }
+
   LLVM_DEBUG(dbgs() << "BOLT-DEBUG: addressed memory is 0x"
                     << Twine::utohexstr(ArrayStart) << '\n');
 
@@ -1126,6 +1166,7 @@ void BinaryFunction::handleIndirectBranch(MCInst &Instruction, uint64_t Size,
   }
   case IndirectBranchType::POSSIBLE_JUMP_TABLE:
   case IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE:
+  case IndirectBranchType::POSSIBLE_PIC_FIXED_BRANCH:
     if (opts::JumpTables == JTS_NONE)
       IsSimple = false;
     break;
@@ -1878,9 +1919,11 @@ bool BinaryFunction::postProcessIndirectBranches(
         int64_t DispValue;
         const MCExpr *DispExpr;
         MCInst *PCRelBaseInstr;
+        MCInst *FixedEntryLoadInstr;
         IndirectBranchType Type = BC.MIB->analyzeIndirectBranch(
             Instr, BB.begin(), II, PtrSize, MemLocInstr, BaseRegNum,
-            IndexRegNum, DispValue, DispExpr, PCRelBaseInstr);
+            IndexRegNum, DispValue, DispExpr, PCRelBaseInstr,
+            FixedEntryLoadInstr);
         if (Type != IndirectBranchType::UNKNOWN || MemLocInstr != nullptr)
           continue;
 
diff --git a/bolt/lib/Passes/IndirectCallPromotion.cpp b/bolt/lib/Passes/IndirectCallPromotion.cpp
index a73eb867b3777..2b5a591f4c7a2 100644
--- a/bolt/lib/Passes/IndirectCallPromotion.cpp
+++ b/bolt/lib/Passes/IndirectCallPromotion.cpp
@@ -386,13 +386,15 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB,
   JumpTableInfoType HotTargets;
   MCInst *MemLocInstr;
   MCInst *PCRelBaseOut;
+  MCInst *FixedEntryLoadInstr;
   unsigned BaseReg, IndexReg;
   int64_t DispValue;
   const MCExpr *DispExpr;
   MutableArrayRef<MCInst> Insts(&BB.front(), &CallInst);
   const IndirectBranchType Type = BC.MIB->analyzeIndirectBranch(
       CallInst, Insts.begin(), Insts.end(), BC.AsmInfo->getCodePointerSize(),
-      MemLocInstr, BaseReg, IndexReg, DispValue, DispExpr, PCRelBaseOut);
+      MemLocInstr, BaseReg, IndexReg, DispValue, DispExpr, PCRelBaseOut,
+      FixedEntryLoadInstr);
 
   assert(MemLocInstr && "There should always be a load for jump tables");
   if (!MemLocInstr)
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 1a2327df1d323..f58f7857e28ae 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -852,16 +852,19 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     return Uses;
   }
 
-  IndirectBranchType analyzeIndirectBranch(
-      MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
-      const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
-      unsigned &IndexRegNumOut, int64_t &DispValueOut,
-      const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override {
+  IndirectBranchType
+  analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin,
+                        InstructionIterator End, const unsigned PtrSize,
+                        MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
+                        unsigned &IndexRegNumOut, int64_t &DispValueOut,
+                        const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut,
+                        MCInst *&FixedEntryLoadInstr) const override {
     MemLocInstrOut = nullptr;
     BaseRegNumOut = AArch64::NoRegister;
     IndexRegNumOut = AArch64::NoRegister;
     DispValueOut = 0;
     DispExprOut = nullptr;
+    FixedEntryLoadInstr = nullptr;
 
     // An instruction referencing memory used by jump instruction (directly or
     // via register). This location could be an array of function pointers
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index eb3f38a0b8f4a..f8c83b09395f5 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -176,13 +176,14 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
       MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
       const unsigned PtrSize, MCInst *&MemLocInstr, unsigned &BaseRegNum,
       unsigned &IndexRegNum, int64_t &DispValue, const MCExpr *&DispExpr,
-      MCInst *&PCRelBaseOut) const override {
+      MCInst *&PCRelBaseOut, MCInst *&FixedEntryLoadInst) const override {
     MemLocInstr = nullptr;
     BaseRegNum = 0;
     IndexRegNum = 0;
     DispValue = 0;
     DispExpr = nullptr;
     PCRelBaseOut = nullptr;
+    FixedEntryLoadInst = nullptr;
 
     // Check for the following long tail call sequence:
     // 1: auipc xi, %pcrel_hi(sym)
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index e46c42533031c..63086c06d74fd 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -1866,8 +1866,11 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     return true;
   }
 
+  /// Analyzes PIC-style jump table code template and return identified
+  /// IndirectBranchType, MemLocInstr (all cases) and FixedEntryLoadInstr
+  /// (POSSIBLE_PIC_FIXED_BRANCH case).
   template <typename Itr>
-  std::pair<IndirectBranchType, MCInst *>
+  std::tuple<IndirectBranchType, MCInst *, MCInst *>
   analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const {
     // Analyze PIC-style jump table code template:
     //
@@ -1876,6 +1879,13 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     //    add %r2, %r1
     //    jmp *%r1
     //
+    // or a fixed indirect jump template:
+    //
+    //    movslq En(%rip), {%r2|%r1}              <- FixedEntryLoadInstr
+    //    lea PIC_JUMP_TABLE(%rip), {%r1|%r2}     <- MemLocInstr
+    //    add %r2, %r1
+    //    jmp *%r1
+    //
     // (with any irrelevant instructions in-between)
     //
     // When we call this helper we've already determined %r1 and %r2, and
@@ -1916,8 +1926,13 @@ class X86MCPlusBuilder : public MCPlusBuilder {
              MO.SegRegNum == X86::NoRegister;
     };
     LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n");
-    MCInst *MemLocInstr = nullptr;
-    const MCInst *MovInstr = nullptr;
+    MCInst *FirstInstr = nullptr;
+    MCInst *SecondInstr = nullptr;
+    enum {
+      NOMATCH = 0,
+      MATCH_JUMP_TABLE,
+      MATCH_FIXED_BRANCH,
+    } MatchingState = NOMATCH;
     while (++II != IE) {
       MCInst &Instr = *II;
       const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode());
@@ -1926,68 +1941,76 @@ class X86MCPlusBuilder : public MCPlusBuilder {
         // Ignore instructions that don't affect R1, R2 registers.
         continue;
       }
-      if (!MovInstr) {
-        // Expect to see MOV instruction.
-        if (!isMOVSX64rm32(Instr)) {
-          LLVM_DEBUG(dbgs() << "MOV instruction expected.\n");
+      const bool IsMOVSXInstr = isMOVSX64rm32(Instr);
+      const bool IsLEAInstr = isLEA64r(Instr);
+      if (MatchingState == NOMATCH) {
+        if (IsMOVSXInstr)
+          MatchingState = MATCH_JUMP_TABLE;
+        else if (IsLEAInstr)
+          MatchingState = MATCH_FIXED_BRANCH;
+        else
           break;
-        }
 
-        // Check if it's setting %r1 or %r2. In canonical form it sets %r2.
-        // If it sets %r1 - rename the registers so we have to only check
-        // a single form.
-        unsigned MovDestReg = Instr.getOperand(0).getReg();
-        if (MovDestReg != R2)
+        // Check if the first instruction is setting %r1 or %r2. In canonical
+        // form lea sets %r1 and mov sets %r2. If it's the opposite - rename so
+        // we have to only check a single form.
+        unsigned DestReg = Instr.getOperand(0).getReg();
+        MCPhysReg &ExpectReg = MatchingState == MATCH_JUMP_TABLE ? R2 : R1;
+        if (DestReg != ExpectReg)
           std::swap(R1, R2);
-        if (MovDestReg != R2) {
-          LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n");
+        if (DestReg != ExpectReg)
           break;
-        }
 
-        // Verify operands for MOV.
+        // Verify operands
         std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr);
         if (!MO)
           break;
-        if (!isIndexed(*MO, R1))
-          // POSSIBLE_PIC_JUMP_TABLE
+        if ((MatchingState == MATCH_JUMP_TABLE && isIndexed(*MO, R1)) ||
+            (MatchingState == MATCH_FIXED_BRANCH && isRIPRel(*MO)))
+          FirstInstr = &Instr;
+        else
           break;
-        MovInstr = &Instr;
       } else {
-        if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo))
+        unsigned ExpectReg = MatchingState == MATCH_JUMP_TABLE ? R1 : R2;
+        if (!InstrDesc.hasDefOfPhysReg(Instr, ExpectReg, *RegInfo))
           continue;
-        if (!isLEA64r(Instr)) {
-          LLVM_DEBUG(dbgs() << "LEA instruction expected\n");
+        if ((MatchingState == MATCH_JUMP_TABLE && !IsLEAInstr) ||
+            (MatchingState == MATCH_FIXED_BRANCH && !IsMOVSXInstr))
           break;
-        }
-        if (Instr.getOperand(0).getReg() != R1) {
-          LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n");
+        if (Instr.getOperand(0).getReg() != ExpectReg)
           break;
-        }
 
-        // Verify operands for LEA.
+        // Verify operands.
         std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr);
         if (!MO)
           break;
         if (!isRIPRel(*MO))
           break;
-        MemLocInstr = &Instr;
+        SecondInstr = &Instr;
         break;
       }
     }
 
-    if (!MemLocInstr)
-      return std::make_pair(IndirectBranchType::UNKNOWN, nullptr);
+    if (!SecondInstr)
+      return std::make_tuple(IndirectBranchType::UNKNOWN, nullptr, nullptr);
 
+    if (MatchingState == MATCH_FIXED_BRANCH) {
+      LLVM_DEBUG(dbgs() << "checking potential fixed indirect branch\n");
+      return std::make_tuple(IndirectBranchType::POSSIBLE_PIC_FIXED_BRANCH,
+                             FirstInstr, SecondInstr);
+    }
     LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n");
-    return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE,
-                          MemLocInstr);
+    return std::make_tuple(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE,
+                           SecondInstr, nullptr);
   }
 
-  IndirectBranchType analyzeIndirectBranch(
-      MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
-      const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
-      unsigned &IndexRegNumOut, int64_t &DispValueOut,
-      const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override {
+  IndirectBranchType
+  analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin,
+                        InstructionIterator End, const unsigned PtrSize,
+                        MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
+                        unsigned &IndexRegNumOut, int64_t &DispValueOut,
+                        const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut,
+                        MCInst *&FixedEntryLoadInst) const override {
     // Try to find a (base) memory location from where the address for
     // the indirect branch is loaded. For X86-64 the memory will be specified
     // in the following format:
@@ -2014,6 +2037,7 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     IndexRegNumOut = X86::NoRegister;
     DispValueOut = 0;
     DispExprOut = nullptr;
+    FixedEntryLoadInst = nullptr;
 
     std::reverse_iterator<InstructionIterator> II(End);
     std::reverse_iterator<InstructionIterator> IE(Begin);
@@ -2046,7 +2070,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
           unsigned R2 = PrevInstr.getOperand(2).getReg();
           if (R1 == R2)
             return IndirectBranchType::UNKNOWN;
-          std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2);
+          std::tie(Type, MemLocInstr, FixedEntryLoadInst) =
+              analyzePICJumpTable(PrevII, IE, R1, R2);
           break;
         }
         return IndirectBranchType::UNKNOWN;
@@ -2090,6 +2115,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
       if (MO->ScaleImm != 1 || MO->BaseRegNum != RIPRegister)
         return IndirectBranchType::UNKNOWN;
       break;
+    case IndirectBranchType::POSSIBLE_PIC_FIXED_BRANCH:
+      break;
     default:
       if (MO->ScaleImm != PtrSize)
         return IndirectBranchType::UNKNOWN;
diff --git a/bolt/test/X86/Inputs/jump-table-fixed-ref-pic.s b/bolt/test/X86/Inputs/jump-table-fixed-ref-pic.s
index 66629a4880e64..6407964593e2d 100644
--- a/bolt/test/X86/Inputs/jump-table-fixed-ref-pic.s
+++ b/bolt/test/X86/Inputs/jump-table-fixed-ref-pic.s
@@ -6,7 +6,7 @@ main:
   jae .L4
   cmpq $0x1, %rdi
   jne .L4
-  mov .Ljt_pic+8(%rip), %rax
+  movslq .Ljt_pic+8(%rip), %rax
   lea .Ljt_pic(%rip), %rdx
   add %rdx, %rax
   jmpq *%rax
diff --git a/bolt/test/X86/jump-table-fixed-ref-pic.test b/bolt/test/X86/jump-table-fixed-ref-pic.test
index c8b6eda2278b9..d215c565b31e5 100644
--- a/bolt/test/X86/jump-table-fixed-ref-pic.test
+++ b/bolt/test/X86/jump-table-fixed-ref-pic.test
@@ -1,9 +1,13 @@
 ## Verify that BOLT detects fixed destination of indirect jump for PIC
 ## case.
 
-XFAIL: *
-
 RUN: %clang %cflags -no-pie %S/Inputs/jump-table-fixed-ref-pic.s -Wl,-q -o %t
-RUN: llvm-bolt %t --relocs -o %t.null 2>&1 | FileCheck %s
+RUN: llvm-bolt %t --relocs -o %t.null -print-cfg 2>&1 | FileCheck %s
+
+CHECK: BOLT-INFO: fixed PIC indirect branch detected in main {{.*}} the destination value is 0x[[#TGT:]]
+CHECK: Binary Function "main" after building cfg
 
-CHECK: BOLT-INFO: fixed indirect branch detected in main
+CHECK:      movslq ".rodata/1"+8(%rip), %rax
+CHECK-NEXT: leaq ".rodata/1"(%rip), %rdx
+CHECK-NEXT: addq %rdx, %rax
+CHECK-NEXT: jmpq *%rax # UNKNOWN CONTROL FLOW

From 9b007a199d650f47072fe112b8315f3b3bebb27d Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Thu, 18 Jul 2024 20:58:16 -0700
Subject: [PATCH 128/486] [BOLT] Expose pseudo probe function checksum and GUID
 (#99389)

Add a BinaryFunction field for pseudo probe function GUID.
Populate it during pseudo probe section parsing, and emit it in YAML
profile (both regular and BAT), along with function checksum.

To be used for stale function matching.

Test Plan: update pseudoprobe-decoding-inline.test
---
 bolt/include/bolt/Core/BinaryContext.h        | 13 +++++++++
 bolt/include/bolt/Core/BinaryFunction.h       |  8 +++++
 .../include/bolt/Profile/ProfileYAMLMapping.h |  5 ++++
 bolt/lib/Profile/DataAggregator.cpp           |  9 ++++++
 bolt/lib/Profile/YAMLProfileWriter.cpp        |  8 +++++
 bolt/lib/Rewrite/PseudoProbeRewriter.cpp      | 29 +++++++++++++++++--
 bolt/lib/Rewrite/RewriteInstance.cpp          |  1 +
 .../test/X86/pseudoprobe-decoding-inline.test | 23 ++++++++++++++-
 8 files changed, 92 insertions(+), 4 deletions(-)

diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index befdc9974c683..de9ba09a5bb49 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -32,6 +32,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCPseudoProbe.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
@@ -246,6 +247,9 @@ class BinaryContext {
   /// DWP Context.
   std::shared_ptr<DWARFContext> DWPContext;
 
+  /// Decoded pseudo probes.
+  std::shared_ptr<MCPseudoProbeDecoder> PseudoProbeDecoder;
+
   /// A map of DWO Ids to CUs.
   using DWOIdToCUMapType = std::unordered_map<uint64_t, DWARFUnit *>;
   DWOIdToCUMapType DWOCUs;
@@ -377,6 +381,15 @@ class BinaryContext {
     RtLibrary = std::move(Lib);
   }
 
+  const MCPseudoProbeDecoder *getPseudoProbeDecoder() const {
+    return PseudoProbeDecoder.get();
+  }
+
+  void setPseudoProbeDecoder(std::shared_ptr<MCPseudoProbeDecoder> Decoder) {
+    assert(!PseudoProbeDecoder && "Cannot set pseudo probe decoder twice.");
+    PseudoProbeDecoder = Decoder;
+  }
+
   /// Return BinaryFunction containing a given \p Address or nullptr if
   /// no registered function contains the \p Address.
   ///
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 7d0afee4d1b78..da3fc433b7a3b 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -416,6 +416,9 @@ class BinaryFunction {
   /// different parameters by every pass.
   mutable uint64_t Hash{0};
 
+  /// Function GUID assigned externally.
+  uint64_t GUID{0};
+
   /// For PLT functions it contains a symbol associated with a function
   /// reference. It is nullptr for non-PLT functions.
   const MCSymbol *PLTSymbol{nullptr};
@@ -2256,6 +2259,11 @@ class BinaryFunction {
   /// Returns the last computed hash value of the function.
   size_t getHash() const { return Hash; }
 
+  /// Returns the function GUID.
+  uint64_t getGUID() const { return GUID; }
+
+  void setGUID(uint64_t Id) { GUID = Id; }
+
   using OperandHashFuncTy =
       function_ref<typename std::string(const MCOperand &)>;
 
diff --git a/bolt/include/bolt/Profile/ProfileYAMLMapping.h b/bolt/include/bolt/Profile/ProfileYAMLMapping.h
index 9dd3920dbf094..c4e5f2b284a86 100644
--- a/bolt/include/bolt/Profile/ProfileYAMLMapping.h
+++ b/bolt/include/bolt/Profile/ProfileYAMLMapping.h
@@ -151,6 +151,8 @@ struct BinaryFunctionProfile {
   llvm::yaml::Hex64 Hash{0};
   uint64_t ExecCount{0};
   std::vector<BinaryBasicBlockProfile> Blocks;
+  llvm::yaml::Hex64 GUID{0};
+  llvm::yaml::Hex64 PseudoProbeDescHash{0};
   bool Used{false};
 };
 } // end namespace bolt
@@ -164,6 +166,9 @@ template <> struct MappingTraits<bolt::BinaryFunctionProfile> {
     YamlIO.mapRequired("nblocks", BFP.NumBasicBlocks);
     YamlIO.mapOptional("blocks", BFP.Blocks,
                        std::vector<bolt::BinaryBasicBlockProfile>());
+    YamlIO.mapOptional("guid", BFP.GUID, (uint64_t)0);
+    YamlIO.mapOptional("pseudo_probe_desc_hash", BFP.PseudoProbeDescHash,
+                       (uint64_t)0);
   }
 };
 
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index ce6ec0a04ac16..fd0a526e31b79 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -2298,6 +2298,8 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
 
   yaml::bolt::BinaryProfile BP;
 
+  const MCPseudoProbeDecoder *PseudoProbeDecoder = BC.getPseudoProbeDecoder();
+
   // Fill out the header info.
   BP.Header.Version = 1;
   BP.Header.FileName = std::string(BC.getFilename());
@@ -2398,6 +2400,13 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
         const unsigned BlockIndex = BlockMap.getBBIndex(BI.To.Offset);
         YamlBF.Blocks[BlockIndex].ExecCount += BI.Branches;
       }
+      if (PseudoProbeDecoder) {
+        if ((YamlBF.GUID = BF->getGUID())) {
+          const MCPseudoProbeFuncDesc *FuncDesc =
+              PseudoProbeDecoder->getFuncDescForGUID(YamlBF.GUID);
+          YamlBF.PseudoProbeDescHash = FuncDesc->FuncHash;
+        }
+      }
       // Drop blocks without a hash, won't be useful for stale matching.
       llvm::erase_if(YamlBF.Blocks,
                      [](const yaml::bolt::BinaryBasicBlockProfile &YamlBB) {
diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp
index 9adbfdc5ff089..82d70e540e5e1 100644
--- a/bolt/lib/Profile/YAMLProfileWriter.cpp
+++ b/bolt/lib/Profile/YAMLProfileWriter.cpp
@@ -57,6 +57,7 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
                            const BoltAddressTranslation *BAT) {
   yaml::bolt::BinaryFunctionProfile YamlBF;
   const BinaryContext &BC = BF.getBinaryContext();
+  const MCPseudoProbeDecoder *PseudoProbeDecoder = BC.getPseudoProbeDecoder();
 
   const uint16_t LBRProfile = BF.getProfileFlags() & BinaryFunction::PF_LBR;
 
@@ -69,6 +70,13 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
   YamlBF.Hash = BF.getHash();
   YamlBF.NumBasicBlocks = BF.size();
   YamlBF.ExecCount = BF.getKnownExecutionCount();
+  if (PseudoProbeDecoder) {
+    if ((YamlBF.GUID = BF.getGUID())) {
+      const MCPseudoProbeFuncDesc *FuncDesc =
+          PseudoProbeDecoder->getFuncDescForGUID(YamlBF.GUID);
+      YamlBF.PseudoProbeDescHash = FuncDesc->FuncHash;
+    }
+  }
 
   BinaryFunction::BasicBlockOrderType Order;
   llvm::copy(UseDFS ? BF.dfs() : BF.getLayout().blocks(),
diff --git a/bolt/lib/Rewrite/PseudoProbeRewriter.cpp b/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
index 51038dbead330..3704a9ba452b9 100644
--- a/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
+++ b/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/LEB128.h"
+#include <memory>
 
 #undef DEBUG_TYPE
 #define DEBUG_TYPE "pseudo-probe-rewriter"
@@ -72,23 +73,35 @@ class PseudoProbeRewriter final : public MetadataRewriter {
   void parsePseudoProbe();
 
   /// PseudoProbe decoder
-  MCPseudoProbeDecoder ProbeDecoder;
+  std::shared_ptr<MCPseudoProbeDecoder> ProbeDecoderPtr;
 
 public:
   PseudoProbeRewriter(BinaryContext &BC)
-      : MetadataRewriter("pseudo-probe-rewriter", BC) {}
+      : MetadataRewriter("pseudo-probe-rewriter", BC),
+        ProbeDecoderPtr(std::make_shared<MCPseudoProbeDecoder>()) {
+    BC.setPseudoProbeDecoder(ProbeDecoderPtr);
+  }
 
+  Error preCFGInitializer() override;
   Error postEmitFinalizer() override;
+
+  ~PseudoProbeRewriter() override { ProbeDecoderPtr.reset(); }
 };
 
-Error PseudoProbeRewriter::postEmitFinalizer() {
+Error PseudoProbeRewriter::preCFGInitializer() {
   parsePseudoProbe();
+
+  return Error::success();
+}
+
+Error PseudoProbeRewriter::postEmitFinalizer() {
   updatePseudoProbes();
 
   return Error::success();
 }
 
 void PseudoProbeRewriter::parsePseudoProbe() {
+  MCPseudoProbeDecoder &ProbeDecoder(*ProbeDecoderPtr);
   PseudoProbeDescSection = BC.getUniqueSectionByName(".pseudo_probe_desc");
   PseudoProbeSection = BC.getUniqueSectionByName(".pseudo_probe");
 
@@ -138,9 +151,18 @@ void PseudoProbeRewriter::parsePseudoProbe() {
     ProbeDecoder.printGUID2FuncDescMap(outs());
     ProbeDecoder.printProbesForAllAddresses(outs());
   }
+
+  for (const auto &[GUID, FuncDesc] : ProbeDecoder.getGUID2FuncDescMap()) {
+    if (!FuncStartAddrs.contains(GUID))
+      continue;
+    BinaryFunction *BF = BC.getBinaryFunctionAtAddress(FuncStartAddrs[GUID]);
+    assert(BF);
+    BF->setGUID(GUID);
+  }
 }
 
 void PseudoProbeRewriter::updatePseudoProbes() {
+  MCPseudoProbeDecoder &ProbeDecoder(*ProbeDecoderPtr);
   // check if there is pseudo probe section decoded
   if (ProbeDecoder.getAddress2ProbesMap().empty())
     return;
@@ -241,6 +263,7 @@ void PseudoProbeRewriter::updatePseudoProbes() {
 }
 
 void PseudoProbeRewriter::encodePseudoProbes() {
+  MCPseudoProbeDecoder &ProbeDecoder(*ProbeDecoderPtr);
   // Buffer for new pseudo probes section
   SmallString<8> Contents;
   MCDecodedPseudoProbe *LastProbe = nullptr;
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 4ae802dc97ccd..33ebae3b6e6de 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -669,6 +669,7 @@ Error RewriteInstance::run() {
         opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) {
       selectFunctionsToProcess();
       disassembleFunctions();
+      processMetadataPreCFG();
       buildFunctionsCFG();
     }
     processProfileData();
diff --git a/bolt/test/X86/pseudoprobe-decoding-inline.test b/bolt/test/X86/pseudoprobe-decoding-inline.test
index 15e93b1630a66..c32660c41a09c 100644
--- a/bolt/test/X86/pseudoprobe-decoding-inline.test
+++ b/bolt/test/X86/pseudoprobe-decoding-inline.test
@@ -1,5 +1,26 @@
 # REQUIRES: system-linux
-# RUN: llvm-bolt  %S/../../../llvm/test/tools/llvm-profgen/Inputs/inline-cs-pseudoprobe.perfbin --print-pseudo-probes=all -o %t.bolt 2>&1 | FileCheck %s
+# RUN: llvm-bolt  %S/../../../llvm/test/tools/llvm-profgen/Inputs/inline-cs-pseudoprobe.perfbin --print-pseudo-probes=all -o %t.bolt --lite=0 --enable-bat 2>&1 | FileCheck %s
+
+# PREAGG: B X:0 #foo# 1 0
+# PREAGG: B X:0 #bar# 1 0
+# PREAGG: B X:0 #main# 1 0
+## Check pseudo-probes in regular YAML profile (non-BOLTed binary)
+# RUN: link_fdata %s %S/../../../llvm/test/tools/llvm-profgen/Inputs/inline-cs-pseudoprobe.perfbin %t.preagg PREAGG
+# RUN: perf2bolt %S/../../../llvm/test/tools/llvm-profgen/Inputs/inline-cs-pseudoprobe.perfbin -p %t.preagg --pa -w %t.yaml -o %t.fdata
+# RUN: FileCheck --input-file %t.yaml %s --check-prefix CHECK-YAML
+## Check pseudo-probes in BAT YAML profile (BOLTed binary)
+# RUN: link_fdata %s %t.bolt %t.preagg2 PREAGG
+# RUN: perf2bolt %t.bolt -p %t.preagg2 --pa -w %t.yaml2 -o %t.fdata2
+# RUN: FileCheck --input-file %t.yaml2 %s --check-prefix CHECK-YAML
+# CHECK-YAML: name: bar
+# CHECK-YAML: guid: 0xE413754A191DB537
+# CHECK-YAML: pseudo_probe_desc_hash: 0x10E852DA94
+# CHECK-YAML: name: foo
+# CHECK-YAML: guid: 0x5CF8C24CDB18BDAC
+# CHECK-YAML: pseudo_probe_desc_hash: 0x200205A19C5B4
+# CHECK-YAML: name: main
+# CHECK-YAML: guid: 0xDB956436E78DD5FA
+# CHECK-YAML: pseudo_probe_desc_hash: 0x10000FFFFFFFF
 
 CHECK: Report of decoding input pseudo probe binaries
 

From 6c3aa626b28d60411b14c593f3e0443cf37fc84d Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Thu, 18 Jul 2024 21:00:26 -0700
Subject: [PATCH 129/486] [MC][NFC] Use std::map for AddressProbesMap

AddressProbesMap is keyed by binary addresses, and it makes sense to
treat them as ordered. This also enables slicing by binary function/
binary basic block, to be used in BOLT
(https://github.com/llvm/llvm-project/pull/99554).

Test Plan: NFC

Reviewers: wlei-llvm

Reviewed By: wlei-llvm

Pull Request: https://github.com/llvm/llvm-project/pull/99553
---
 llvm/include/llvm/MC/MCPseudoProbe.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h
index 4d82da70f3df8..5344dea4141b3 100644
--- a/llvm/include/llvm/MC/MCPseudoProbe.h
+++ b/llvm/include/llvm/MC/MCPseudoProbe.h
@@ -61,6 +61,7 @@
 #include "llvm/IR/PseudoProbe.h"
 #include "llvm/Support/ErrorOr.h"
 #include <list>
+#include <map>
 #include <memory>
 #include <string>
 #include <tuple>
@@ -102,8 +103,7 @@ using MCPseudoProbeInlineStack = SmallVector<InlineSite, 8>;
 using GUIDProbeFunctionMap =
     std::unordered_map<uint64_t, MCPseudoProbeFuncDesc>;
 // Address to pseudo probes map.
-using AddressProbesMap =
-    std::unordered_map<uint64_t, std::list<MCDecodedPseudoProbe>>;
+using AddressProbesMap = std::map<uint64_t, std::list<MCDecodedPseudoProbe>>;
 
 class MCDecodedPseudoProbeInlineTree;
 

From c905db67a05c4455548ea322c780a8d74380fd86 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Thu, 18 Jul 2024 21:01:40 -0700
Subject: [PATCH 130/486] [BOLT] Attach pseudo probes to blocks in YAML profile

Read pseudo probes in regular and BAT YAML profile generation, and
attach them to YAML profile basic blocks. This exposes GUID, probe id,
and probe type in profile for future use in stale profile matching.

Test Plan: updated pseudoprobe-decoding-inline.test

Reviewers: dcci, rafaelauler, ayermolo, maksfb

Reviewed By: rafaelauler

Pull Request: https://github.com/llvm/llvm-project/pull/99554
---
 .../include/bolt/Profile/ProfileYAMLMapping.h | 28 +++++++++++++++++++
 bolt/lib/Profile/DataAggregator.cpp           | 20 +++++++++++++
 bolt/lib/Profile/YAMLProfileWriter.cpp        | 15 ++++++++++
 .../test/X86/pseudoprobe-decoding-inline.test |  8 ++++++
 4 files changed, 71 insertions(+)

diff --git a/bolt/include/bolt/Profile/ProfileYAMLMapping.h b/bolt/include/bolt/Profile/ProfileYAMLMapping.h
index c4e5f2b284a86..2a0514d7d9304 100644
--- a/bolt/include/bolt/Profile/ProfileYAMLMapping.h
+++ b/bolt/include/bolt/Profile/ProfileYAMLMapping.h
@@ -93,11 +93,36 @@ template <> struct MappingTraits<bolt::SuccessorInfo> {
   static const bool flow = true;
 };
 
+namespace bolt {
+struct PseudoProbeInfo {
+  llvm::yaml::Hex64 GUID;
+  uint64_t Index;
+  uint8_t Type;
+
+  bool operator==(const PseudoProbeInfo &Other) const {
+    return GUID == Other.GUID && Index == Other.Index;
+  }
+  bool operator!=(const PseudoProbeInfo &Other) const {
+    return !(*this == Other);
+  }
+};
+} // end namespace bolt
+
+template <> struct MappingTraits<bolt::PseudoProbeInfo> {
+  static void mapping(IO &YamlIO, bolt::PseudoProbeInfo &PI) {
+    YamlIO.mapRequired("guid", PI.GUID);
+    YamlIO.mapRequired("id", PI.Index);
+    YamlIO.mapRequired("type", PI.Type);
+  }
+
+  static const bool flow = true;
+};
 } // end namespace yaml
 } // end namespace llvm
 
 LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::bolt::CallSiteInfo)
 LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::bolt::SuccessorInfo)
+LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::bolt::PseudoProbeInfo)
 
 namespace llvm {
 namespace yaml {
@@ -111,6 +136,7 @@ struct BinaryBasicBlockProfile {
   uint64_t EventCount{0};
   std::vector<CallSiteInfo> CallSites;
   std::vector<SuccessorInfo> Successors;
+  std::vector<PseudoProbeInfo> PseudoProbes;
 
   bool operator==(const BinaryBasicBlockProfile &Other) const {
     return Index == Other.Index;
@@ -132,6 +158,8 @@ template <> struct MappingTraits<bolt::BinaryBasicBlockProfile> {
                        std::vector<bolt::CallSiteInfo>());
     YamlIO.mapOptional("succ", BBP.Successors,
                        std::vector<bolt::SuccessorInfo>());
+    YamlIO.mapOptional("pseudo_probes", BBP.PseudoProbes,
+                       std::vector<bolt::PseudoProbeInfo>());
   }
 };
 
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index fd0a526e31b79..c9668bddd0d7e 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -2406,6 +2406,26 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
               PseudoProbeDecoder->getFuncDescForGUID(YamlBF.GUID);
           YamlBF.PseudoProbeDescHash = FuncDesc->FuncHash;
         }
+        // Fetch probes belonging to all fragments
+        const AddressProbesMap &ProbeMap =
+            PseudoProbeDecoder->getAddress2ProbesMap();
+        BinaryFunction::FragmentsSetTy Fragments(BF->Fragments);
+        Fragments.insert(BF);
+        for (const BinaryFunction *F : Fragments) {
+          const uint64_t FuncAddr = F->getAddress();
+          const auto &FragmentProbes =
+              llvm::make_range(ProbeMap.lower_bound(FuncAddr),
+                               ProbeMap.lower_bound(FuncAddr + F->getSize()));
+          for (const auto &[OutputAddress, Probes] : FragmentProbes) {
+            const uint32_t InputOffset = BAT->translate(
+                FuncAddr, OutputAddress - FuncAddr, /*IsBranchSrc=*/true);
+            const unsigned BlockIndex = getBlock(InputOffset).second;
+            for (const MCDecodedPseudoProbe &Probe : Probes)
+              YamlBF.Blocks[BlockIndex].PseudoProbes.emplace_back(
+                  yaml::bolt::PseudoProbeInfo{Probe.getGuid(), Probe.getIndex(),
+                                              Probe.getType()});
+          }
+        }
       }
       // Drop blocks without a hash, won't be useful for stale matching.
       llvm::erase_if(YamlBF.Blocks,
diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp
index 82d70e540e5e1..b9fcb5df29fdb 100644
--- a/bolt/lib/Profile/YAMLProfileWriter.cpp
+++ b/bolt/lib/Profile/YAMLProfileWriter.cpp
@@ -185,6 +185,21 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
       ++BranchInfo;
     }
 
+    if (PseudoProbeDecoder) {
+      const AddressProbesMap &ProbeMap =
+          PseudoProbeDecoder->getAddress2ProbesMap();
+      const uint64_t FuncAddr = BF.getAddress();
+      const std::pair<uint64_t, uint64_t> &BlockRange =
+          BB->getInputAddressRange();
+      const auto &BlockProbes =
+          llvm::make_range(ProbeMap.lower_bound(FuncAddr + BlockRange.first),
+                           ProbeMap.lower_bound(FuncAddr + BlockRange.second));
+      for (const auto &[_, Probes] : BlockProbes)
+        for (const MCDecodedPseudoProbe &Probe : Probes)
+          YamlBB.PseudoProbes.emplace_back(yaml::bolt::PseudoProbeInfo{
+              Probe.getGuid(), Probe.getIndex(), Probe.getType()});
+    }
+
     YamlBF.Blocks.emplace_back(YamlBB);
   }
   return YamlBF;
diff --git a/bolt/test/X86/pseudoprobe-decoding-inline.test b/bolt/test/X86/pseudoprobe-decoding-inline.test
index c32660c41a09c..924331e34c76a 100644
--- a/bolt/test/X86/pseudoprobe-decoding-inline.test
+++ b/bolt/test/X86/pseudoprobe-decoding-inline.test
@@ -13,12 +13,20 @@
 # RUN: perf2bolt %t.bolt -p %t.preagg2 --pa -w %t.yaml2 -o %t.fdata2
 # RUN: FileCheck --input-file %t.yaml2 %s --check-prefix CHECK-YAML
 # CHECK-YAML: name: bar
+# CHECK-YAML: - bid: 0
+# CHECK-YAML:   pseudo_probes: [ { guid: 0xE413754A191DB537, id: 1, type: 0 }, { guid: 0xE413754A191DB537, id: 4, type: 0 } ]
 # CHECK-YAML: guid: 0xE413754A191DB537
 # CHECK-YAML: pseudo_probe_desc_hash: 0x10E852DA94
+#
 # CHECK-YAML: name: foo
+# CHECK-YAML: - bid: 0
+# CHECK-YAML:   pseudo_probes: [ { guid: 0x5CF8C24CDB18BDAC, id: 1, type: 0 }, { guid: 0x5CF8C24CDB18BDAC, id: 2, type: 0 } ]
 # CHECK-YAML: guid: 0x5CF8C24CDB18BDAC
 # CHECK-YAML: pseudo_probe_desc_hash: 0x200205A19C5B4
+#
 # CHECK-YAML: name: main
+# CHECK-YAML: - bid: 0
+# CHECK-YAML:   pseudo_probes: [ { guid: 0xDB956436E78DD5FA, id: 1, type: 0 }, { guid: 0x5CF8C24CDB18BDAC, id: 1, type: 0 }, { guid: 0x5CF8C24CDB18BDAC, id: 2, type: 0 } ]
 # CHECK-YAML: guid: 0xDB956436E78DD5FA
 # CHECK-YAML: pseudo_probe_desc_hash: 0x10000FFFFFFFF
 

From 871740761f15df9436fc5e31c0c0ef17b9294891 Mon Sep 17 00:00:00 2001
From: AtariDreams <gfunni234@gmail.com>
Date: Fri, 19 Jul 2024 00:15:36 -0400
Subject: [PATCH 131/486] [CodeGen] Remove checks for vectors in unsigned
 division prior to computing leading zeros (#99524)

It turns out we can safely use
DAG.computeKnownBits(N0).countMinLeadingZeros() with constant legal
vectors, so remove the check for it.
---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   | 13 +---
 llvm/test/CodeGen/X86/combine-udiv.ll         | 61 +++++++++++++++++++
 ...of-two-or-zero-when-comparing-with-zero.ll |  9 +--
 3 files changed, 67 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index c3a20b5044c5f..140c97ccd90ba 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6483,15 +6483,7 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
 
   // Try to use leading zeros of the dividend to reduce the multiplier and
   // avoid expensive fixups.
-  // TODO: Support vectors.
-  unsigned LeadingZeros = 0;
-  if (!VT.isVector() && isa<ConstantSDNode>(N1)) {
-    assert(!isOneConstant(N1) && "Unexpected divisor");
-    LeadingZeros = DAG.computeKnownBits(N0).countMinLeadingZeros();
-    // UnsignedDivisionByConstantInfo doesn't work correctly if leading zeros in
-    // the dividend exceeds the leading zeros for the divisor.
-    LeadingZeros = std::min(LeadingZeros, N1->getAsAPIntVal().countl_zero());
-  }
+  unsigned KnownLeadingZeros = DAG.computeKnownBits(N0).countMinLeadingZeros();
 
   bool UseNPQ = false, UsePreShift = false, UsePostShift = false;
   SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
@@ -6510,7 +6502,8 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
       MagicFactor = NPQFactor = DAG.getUNDEF(SVT);
     } else {
       UnsignedDivisionByConstantInfo magics =
-          UnsignedDivisionByConstantInfo::get(Divisor, LeadingZeros);
+          UnsignedDivisionByConstantInfo::get(
+              Divisor, std::min(KnownLeadingZeros, Divisor.countl_zero()));
 
       MagicFactor = DAG.getConstant(magics.Magic, dl, SVT);
 
diff --git a/llvm/test/CodeGen/X86/combine-udiv.ll b/llvm/test/CodeGen/X86/combine-udiv.ll
index e429ac0c63c2d..d5a481549f851 100644
--- a/llvm/test/CodeGen/X86/combine-udiv.ll
+++ b/llvm/test/CodeGen/X86/combine-udiv.ll
@@ -755,3 +755,64 @@ define <4 x i1> @boolvec_udiv(<4 x i1> %x, <4 x i1> %y) {
   %r = udiv <4 x i1> %x, %y
   ret <4 x i1> %r
 }
+
+define <4 x i32> @vector_div_leading_zeros(<4 x i32> %x) {
+; SSE2-LABEL: vector_div_leading_zeros:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: vector_div_leading_zeros:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
+; SSE41-NEXT:    pmuludq %xmm2, %xmm1
+; SSE41-NEXT:    pmuludq %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: vector_div_leading_zeros:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
+; AVX1-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: vector_div_leading_zeros:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; AVX2-NEXT:    retq
+;
+; XOP-LABEL: vector_div_leading_zeros:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; XOP-NEXT:    vbroadcastss {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
+; XOP-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; XOP-NEXT:    vpmuludq %xmm2, %xmm0, %xmm0
+; XOP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; XOP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; XOP-NEXT:    retq
+  %a = and <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
+  %b = udiv <4 x i32> %a, <i32 7, i32 7, i32 7, i32 7>
+  ret <4 x i32> %b
+}
diff --git a/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll b/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll
index 886c3ae10324d..9e398096bfcc5 100644
--- a/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll
+++ b/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll
@@ -230,7 +230,7 @@ define <4 x i1> @p7_vector_urem_by_const__nonsplat_undef2(<4 x i32> %x, <4 x i32
 ; SSE2-LABEL: p7_vector_urem_by_const__nonsplat_undef2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [715827883,715827883,715827883,715827883]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
@@ -238,7 +238,6 @@ define <4 x i1> @p7_vector_urem_by_const__nonsplat_undef2(<4 x i32> %x, <4 x i32
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    psrld $2, %xmm2
 ; SSE2-NEXT:    pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [6,0,6,0,6,0,6,0]
 ; SSE2-NEXT:    psubd %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -249,12 +248,11 @@ define <4 x i1> @p7_vector_urem_by_const__nonsplat_undef2(<4 x i32> %x, <4 x i32
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [715827883,715827883,715827883,715827883]
 ; SSE4-NEXT:    pmuludq %xmm2, %xmm1
 ; SSE4-NEXT:    pmuludq %xmm0, %xmm2
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; SSE4-NEXT:    psrld $2, %xmm2
 ; SSE4-NEXT:    pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [6,0,6,0,6,0,6,0]
 ; SSE4-NEXT:    psubd %xmm2, %xmm0
 ; SSE4-NEXT:    pxor %xmm1, %xmm1
@@ -266,12 +264,11 @@ define <4 x i1> @p7_vector_urem_by_const__nonsplat_undef2(<4 x i32> %x, <4 x i32
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [715827883,715827883,715827883,715827883]
 ; AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; AVX2-NEXT:    vpsrld $2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [6,0,6,0,6,0,6,0]
 ; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1

From f554dd7e7690f96ecc130065972c306cf0decd7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Fri, 19 Jul 2024 06:16:12 +0200
Subject: [PATCH 132/486] [GlobalIsel] import G_SCMP and G_UCMP (#99518)

See https://github.com/llvm/llvm-project/pull/98894
---
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  | 10 ++++
 .../AArch64/GlobalISel/irtranslator-sucmp.ll  | 54 +++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-sucmp.ll

diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 72dff12423ced..40a691af22748 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2561,6 +2561,16 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     MIRBuilder.buildVScale(getOrCreateVReg(CI), 1);
     return true;
   }
+  case Intrinsic::scmp:
+    MIRBuilder.buildSCmp(getOrCreateVReg(CI),
+                         getOrCreateVReg(*CI.getOperand(0)),
+                         getOrCreateVReg(*CI.getOperand(1)));
+    return true;
+  case Intrinsic::ucmp:
+    MIRBuilder.buildUCmp(getOrCreateVReg(CI),
+                         getOrCreateVReg(*CI.getOperand(0)),
+                         getOrCreateVReg(*CI.getOperand(1)));
+    return true;
   case Intrinsic::prefetch: {
     Value *Addr = CI.getOperand(0);
     unsigned RW = cast<ConstantInt>(CI.getOperand(1))->getZExtValue();
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-sucmp.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-sucmp.ll
new file mode 100644
index 0000000000000..1fa21bfb733e8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-sucmp.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -O0 -mtriple=aarch64-linux-gnu -global-isel -stop-after=irtranslator %s -o - | FileCheck %s
+
+define void @scmp_i32(i32 %arg1, i32 %arg2) {
+  ; CHECK-LABEL: name: scmp_i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $w0, $w1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK-NEXT:   [[SCMP:%[0-9]+]]:_(s4) = G_SCMP [[COPY]](s32), [[COPY1]]
+  ; CHECK-NEXT:   RET_ReallyLR
+  %res4 = call i4 @llvm.scmp.i4.i32(i32 %arg1, i32 %arg2)
+  ret void
+}
+
+define void @scmp_4_32i(<4 x i32> %arg1, <4 x i32> %arg2) {
+  ; CHECK-LABEL: name: scmp_4_32i
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $q0, $q1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+  ; CHECK-NEXT:   [[SCMP:%[0-9]+]]:_(<4 x s32>) = G_SCMP [[COPY]](<4 x s32>), [[COPY1]]
+  ; CHECK-NEXT:   RET_ReallyLR
+  %res4 = call <4 x i32> @llvm.scmp.v4i32.i32(<4 x i32> %arg1, <4 x i32> %arg2)
+  ret void
+}
+
+define void @ucmp_i32(i32 %arg1, i32 %arg2) {
+  ; CHECK-LABEL: name: ucmp_i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $w0, $w1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK-NEXT:   [[UCMP:%[0-9]+]]:_(s4) = G_UCMP [[COPY]](s32), [[COPY1]]
+  ; CHECK-NEXT:   RET_ReallyLR
+  %res4 = call i4 @llvm.ucmp.i4.i32(i32 %arg1, i32 %arg2)
+  ret void
+}
+
+define void @ucmp_4_32i(<4 x i32> %arg1, <4 x i32> %arg2) {
+  ; CHECK-LABEL: name: ucmp_4_32i
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $q0, $q1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+  ; CHECK-NEXT:   [[UCMP:%[0-9]+]]:_(<4 x s32>) = G_UCMP [[COPY]](<4 x s32>), [[COPY1]]
+  ; CHECK-NEXT:   RET_ReallyLR
+  %res4 = call <4 x i32> @llvm.ucmp.v4i32.i32(<4 x i32> %arg1, <4 x i32> %arg2)
+  ret void
+}

From 687fc08e7c7755f68838040edb00d4f64e0bebe7 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 18 Jul 2024 21:29:47 -0700
Subject: [PATCH 133/486] [ADT] Use UnorderedElementsAre in
 SetOperationsTest.cpp (NFC) (#99596)

---
 llvm/unittests/ADT/SetOperationsTest.cpp | 166 +++++++++--------------
 1 file changed, 65 insertions(+), 101 deletions(-)

diff --git a/llvm/unittests/ADT/SetOperationsTest.cpp b/llvm/unittests/ADT/SetOperationsTest.cpp
index da84c8afbd2ba..f99f5c9b2af10 100644
--- a/llvm/unittests/ADT/SetOperationsTest.cpp
+++ b/llvm/unittests/ADT/SetOperationsTest.cpp
@@ -25,50 +25,43 @@ namespace {
 TEST(SetOperationsTest, SetUnion) {
   std::set<int> Set1 = {1, 2, 3, 4};
   std::set<int> Set2 = {5, 6, 7, 8};
-  // Set1 should be the union of input sets Set1 and Set2.
-  std::set<int> ExpectedSet1 = {1, 2, 3, 4, 5, 6, 7, 8};
-  // Set2 should not be touched.
-  std::set<int> ExpectedSet2 = Set2;
 
   set_union(Set1, Set2);
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  // Set1 should be the union of input sets Set1 and Set2.
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2, 3, 4, 5, 6, 7, 8));
+  // Set2 should not be touched.
+  EXPECT_THAT(Set2, UnorderedElementsAre(5, 6, 7, 8));
 
   Set1.clear();
   Set2 = {1, 2};
+
+  set_union(Set1, Set2);
   // Set1 should be the union of input sets Set1 and Set2, which in this case
   // will be Set2.
-  ExpectedSet1 = Set2;
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2));
   // Set2 should not be touched.
-  ExpectedSet2 = Set2;
-
-  set_union(Set1, Set2);
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  EXPECT_THAT(Set2, UnorderedElementsAre(1, 2));
 }
 
 TEST(SetOperationsTest, SetIntersect) {
   std::set<int> Set1 = {1, 2, 3, 4};
   std::set<int> Set2 = {3, 4, 5, 6};
-  // Set1 should be the intersection of sets Set1 and Set2.
-  std::set<int> ExpectedSet1 = {3, 4};
-  // Set2 should not be touched.
-  std::set<int> ExpectedSet2 = Set2;
 
   set_intersect(Set1, Set2);
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  // Set1 should be the intersection of sets Set1 and Set2.
+  EXPECT_THAT(Set1, UnorderedElementsAre(3, 4));
+  // Set2 should not be touched.
+  EXPECT_THAT(Set2, UnorderedElementsAre(3, 4, 5, 6));
 
   Set1 = {1, 2, 3, 4};
   Set2 = {5, 6};
-  // Set2 should not be touched.
-  ExpectedSet2 = Set2;
 
   set_intersect(Set1, Set2);
   // Set1 should be the intersection of sets Set1 and Set2, which
   // is empty as they are non-overlapping.
   EXPECT_THAT(Set1, IsEmpty());
-  EXPECT_EQ(ExpectedSet2, Set2);
+  // Set2 should not be touched.
+  EXPECT_THAT(Set2, UnorderedElementsAre(5, 6));
 
   // Check that set_intersect works on SetVector via remove_if.
   SmallSetVector<int, 4> SV;
@@ -85,35 +78,27 @@ TEST(SetOperationsTest, SetIntersection) {
   std::set<int> Set1 = {1, 2, 3, 4};
   std::set<int> Set2 = {3, 4, 5, 6};
   std::set<int> Result;
-  // Result should be the intersection of sets Set1 and Set2.
-  std::set<int> ExpectedResult = {3, 4};
-  // Set1 and Set2 should not be touched.
-  std::set<int> ExpectedSet1 = Set1;
-  std::set<int> ExpectedSet2 = Set2;
 
   Result = set_intersection(Set1, Set2);
-  EXPECT_EQ(ExpectedResult, Result);
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  // Result should be the intersection of sets Set1 and Set2.
+  EXPECT_THAT(Result, UnorderedElementsAre(3, 4));
+  // Set1 and Set2 should not be touched.
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2, 3, 4));
+  EXPECT_THAT(Set2, UnorderedElementsAre(3, 4, 5, 6));
 
   Set1 = {1, 2, 3, 4};
   Set2 = {5, 6};
-  // Set1 and Set2 should not be touched.
-  ExpectedSet1 = Set1;
-  ExpectedSet2 = Set2;
 
   Result = set_intersection(Set1, Set2);
   // Result should be the intersection of sets Set1 and Set2, which
   // is empty as they are non-overlapping.
   EXPECT_THAT(Result, IsEmpty());
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  // Set1 and Set2 should not be touched.
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2, 3, 4));
+  EXPECT_THAT(Set2, UnorderedElementsAre(5, 6));
 
   Set1 = {5, 6};
   Set2 = {1, 2, 3, 4};
-  // Set1 and Set2 should not be touched.
-  ExpectedSet1 = Set1;
-  ExpectedSet2 = Set2;
 
   Result = set_intersection(Set1, Set2);
   // Result should be the intersection of sets Set1 and Set2, which
@@ -121,85 +106,73 @@ TEST(SetOperationsTest, SetIntersection) {
   // reversed, since the code takes a different path depending on which input
   // set is smaller.
   EXPECT_THAT(Result, IsEmpty());
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  // Set1 and Set2 should not be touched.
+  EXPECT_THAT(Set1, UnorderedElementsAre(5, 6));
+  EXPECT_THAT(Set2, UnorderedElementsAre(1, 2, 3, 4));
 }
 
 TEST(SetOperationsTest, SetDifference) {
   std::set<int> Set1 = {1, 2, 3, 4};
   std::set<int> Set2 = {3, 4, 5, 6};
   std::set<int> Result;
-  // Result should be Set1 - Set2, leaving only {1, 2}.
-  std::set<int> ExpectedResult = {1, 2};
-  // Set1 and Set2 should not be touched.
-  std::set<int> ExpectedSet1 = Set1;
-  std::set<int> ExpectedSet2 = Set2;
 
   Result = set_difference(Set1, Set2);
-  EXPECT_EQ(ExpectedResult, Result);
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  // Result should be Set1 - Set2, leaving only {1, 2}.
+  EXPECT_THAT(Result, UnorderedElementsAre(1, 2));
+  // Set1 and Set2 should not be touched.
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2, 3, 4));
+  EXPECT_THAT(Set2, UnorderedElementsAre(3, 4, 5, 6));
 
   Set1 = {1, 2, 3, 4};
   Set2 = {1, 2, 3, 4};
-  // Set1 and Set2 should not be touched.
-  ExpectedSet1 = Set1;
-  ExpectedSet2 = Set2;
 
   Result = set_difference(Set1, Set2);
   // Result should be Set1 - Set2, which should be empty.
   EXPECT_THAT(Result, IsEmpty());
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  // Set1 and Set2 should not be touched.
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2, 3, 4));
+  EXPECT_THAT(Set2, UnorderedElementsAre(1, 2, 3, 4));
 
   Set1 = {1, 2, 3, 4};
   Set2 = {5, 6};
+
+  Result = set_difference(Set1, Set2);
   // Result should be Set1 - Set2, which should be Set1 as they are
   // non-overlapping.
-  ExpectedResult = Set1;
+  EXPECT_THAT(Result, UnorderedElementsAre(1, 2, 3, 4));
   // Set1 and Set2 should not be touched.
-  ExpectedSet1 = Set1;
-  ExpectedSet2 = Set2;
-
-  Result = set_difference(Set1, Set2);
-  EXPECT_EQ(ExpectedResult, Result);
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2, 3, 4));
+  EXPECT_THAT(Set2, UnorderedElementsAre(5, 6));
 }
 
 TEST(SetOperationsTest, SetSubtract) {
   std::set<int> Set1 = {1, 2, 3, 4};
   std::set<int> Set2 = {3, 4, 5, 6};
-  // Set1 should get Set1 - Set2, leaving only {1, 2}.
-  std::set<int> ExpectedSet1 = {1, 2};
-  // Set2 should not be touched.
-  std::set<int> ExpectedSet2 = Set2;
 
   set_subtract(Set1, Set2);
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  // Set1 should get Set1 - Set2, leaving only {1, 2}.
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2));
+  // Set2 should not be touched.
+  EXPECT_THAT(Set2, UnorderedElementsAre(3, 4, 5, 6));
 
   Set1 = {1, 2, 3, 4};
   Set2 = {1, 2, 3, 4};
-  // Set2 should not be touched.
-  ExpectedSet2 = Set2;
 
   set_subtract(Set1, Set2);
   // Set1 should get Set1 - Set2, which should be empty.
   EXPECT_THAT(Set1, IsEmpty());
-  EXPECT_EQ(ExpectedSet2, Set2);
+  // Set2 should not be touched.
+  EXPECT_THAT(Set2, UnorderedElementsAre(1, 2, 3, 4));
 
   Set1 = {1, 2, 3, 4};
   Set2 = {5, 6};
+
+  set_subtract(Set1, Set2);
   // Set1 should get Set1 - Set2, which should be Set1 as they are
   // non-overlapping.
-  ExpectedSet1 = Set1;
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2, 3, 4));
   // Set2 should not be touched.
-  ExpectedSet2 = Set2;
-
-  set_subtract(Set1, Set2);
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
+  EXPECT_THAT(Set2, UnorderedElementsAre(5, 6));
 }
 
 TEST(SetOperationsTest, SetSubtractSmallPtrSet) {
@@ -239,56 +212,47 @@ TEST(SetOperationsTest, SetSubtractRemovedRemaining) {
 
   std::set<int> Set1 = {1, 2, 3, 4};
   std::set<int> Set2 = {3, 4, 5, 6};
+
+  set_subtract(Set1, Set2, Removed, Remaining);
   // Set1 should get Set1 - Set2, leaving only {1, 2}.
-  std::set<int> ExpectedSet1 = {1, 2};
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2));
   // Set2 should not be touched.
-  std::set<int> ExpectedSet2 = Set2;
+  EXPECT_THAT(Set2, UnorderedElementsAre(3, 4, 5, 6));
   // We should get back that {3, 4} from Set2 were removed from Set1, and {5, 6}
   // were not removed from Set1.
-  std::set<int> ExpectedRemoved = {3, 4};
-  std::set<int> ExpectedRemaining = {5, 6};
-
-  set_subtract(Set1, Set2, Removed, Remaining);
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
-  EXPECT_EQ(ExpectedRemoved, Removed);
-  EXPECT_EQ(ExpectedRemaining, Remaining);
+  EXPECT_THAT(Removed, UnorderedElementsAre(3, 4));
+  EXPECT_THAT(Remaining, UnorderedElementsAre(5, 6));
 
   Set1 = {1, 2, 3, 4};
   Set2 = {1, 2, 3, 4};
   Removed.clear();
   Remaining.clear();
-  // Set2 should not be touched.
-  ExpectedSet2 = Set2;
-  // Set should get back that all of Set2 was removed from Set1, and nothing
-  // left in Set2 was not removed from Set1.
-  ExpectedRemoved = Set2;
 
   set_subtract(Set1, Set2, Removed, Remaining);
   // Set1 should get Set1 - Set2, which should be empty.
   EXPECT_THAT(Set1, IsEmpty());
-  EXPECT_EQ(ExpectedSet2, Set2);
-  EXPECT_EQ(ExpectedRemoved, Removed);
+  // Set2 should not be touched.
+  EXPECT_THAT(Set2, UnorderedElementsAre(1, 2, 3, 4));
+  // Set should get back that all of Set2 was removed from Set1, and nothing
+  // left in Set2 was not removed from Set1.
+  EXPECT_THAT(Removed, UnorderedElementsAre(1, 2, 3, 4));
   EXPECT_THAT(Remaining, IsEmpty());
 
   Set1 = {1, 2, 3, 4};
   Set2 = {5, 6};
   Removed.clear();
   Remaining.clear();
+
+  set_subtract(Set1, Set2, Removed, Remaining);
   // Set1 should get Set1 - Set2, which should be Set1 as they are
   // non-overlapping.
-  ExpectedSet1 = {1, 2, 3, 4};
+  EXPECT_THAT(Set1, UnorderedElementsAre(1, 2, 3, 4));
   // Set2 should not be touched.
-  ExpectedSet2 = Set2;
+  EXPECT_THAT(Set2, UnorderedElementsAre(5, 6));
+  EXPECT_THAT(Removed, IsEmpty());
   // Set should get back that none of Set2 was removed from Set1, and all
   // of Set2 was not removed from Set1.
-  ExpectedRemaining = Set2;
-
-  set_subtract(Set1, Set2, Removed, Remaining);
-  EXPECT_EQ(ExpectedSet1, Set1);
-  EXPECT_EQ(ExpectedSet2, Set2);
-  EXPECT_THAT(Removed, IsEmpty());
-  EXPECT_EQ(ExpectedRemaining, Remaining);
+  EXPECT_THAT(Remaining, UnorderedElementsAre(5, 6));
 }
 
 TEST(SetOperationsTest, SetIsSubset) {

From 88e9bd822fe088eff74f49081b890071538fa40c Mon Sep 17 00:00:00 2001
From: Shengchen Kan <shengchen.kan@intel.com>
Date: Fri, 19 Jul 2024 12:32:23 +0800
Subject: [PATCH 134/486] [X86][Driver] Enable feature zu for -mapxf

This is follow-up for #78901 after validation.
Drop the comments for stability since zu is the last feature for cpuid APX_F.
---
 clang/include/clang/Driver/Options.td         | 5 ++---
 clang/lib/Basic/Targets/X86.cpp               | 4 ++--
 clang/test/Driver/x86-target-features.c       | 4 ++--
 clang/test/Preprocessor/x86_target_features.c | 2 +-
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 1675e435d210c..12b20fe04675b 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -6356,9 +6356,8 @@ def mapx_features_EQ : CommaJoined<["-"], "mapx-features=">, Group<m_x86_Feature
     HelpText<"Enable features of APX">, Values<"egpr,push2pop2,ppx,ndd,ccmp,nf,cf,zu">,  Visibility<[ClangOption, CLOption, FlangOption]>;
 def mno_apx_features_EQ : CommaJoined<["-"], "mno-apx-features=">, Group<m_x86_Features_Group>,
     HelpText<"Disable features of APX">, Values<"egpr,push2pop2,ppx,ndd,ccmp,nf,cf,zu">, Visibility<[ClangOption, CLOption, FlangOption]>;
-// For stability, we only add a feature to -mapxf after it passes the validation of llvm-test-suite && cpu2017 on Intel SDE.
-def mapxf : Flag<["-"], "mapxf">, Alias<mapx_features_EQ>, AliasArgs<["egpr","push2pop2","ppx","ndd","ccmp","nf","cf"]>;
-def mno_apxf : Flag<["-"], "mno-apxf">, Alias<mno_apx_features_EQ>, AliasArgs<["egpr","push2pop2","ppx","ndd","ccmp","nf","cf"]>;
+def mapxf : Flag<["-"], "mapxf">, Alias<mapx_features_EQ>, AliasArgs<["egpr","push2pop2","ppx","ndd","ccmp","nf","cf","zu"]>;
+def mno_apxf : Flag<["-"], "mno-apxf">, Alias<mno_apx_features_EQ>, AliasArgs<["egpr","push2pop2","ppx","ndd","ccmp","nf","cf","zu"]>;
 def mapx_inline_asm_use_gpr32 : Flag<["-"], "mapx-inline-asm-use-gpr32">, Group<m_Group>,
                                 HelpText<"Enable use of GPR32 in inline assembly for APX">;
 } // let Flags = [TargetSpecific]
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 121a2c2d795fe..18e6dbf03e00d 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -963,8 +963,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__CF__");
   if (HasZU)
     Builder.defineMacro("__ZU__");
-  // Condition here is aligned with the feature set of mapxf in Options.td
-  if (HasEGPR && HasPush2Pop2 && HasPPX && HasNDD && HasCCMP && HasNF && HasCF)
+  if (HasEGPR && HasPush2Pop2 && HasPPX && HasNDD && HasCCMP && HasNF &&
+      HasCF && HasZU)
     Builder.defineMacro("__APX_F__");
   if (HasEGPR && HasInlineAsmUseGPR32)
     Builder.defineMacro("__APX_INLINE_ASM_USE_GPR32__");
diff --git a/clang/test/Driver/x86-target-features.c b/clang/test/Driver/x86-target-features.c
index ba61457102a57..63237cbc3e7ef 100644
--- a/clang/test/Driver/x86-target-features.c
+++ b/clang/test/Driver/x86-target-features.c
@@ -428,8 +428,8 @@
 // RUN: %clang -target x86_64-unknown-linux-gnu -mno-apxf -mapxf %s -### -o %t.o 2>&1 | FileCheck -check-prefix=APXF %s
 // RUN: %clang -target x86_64-unknown-linux-gnu -mapxf -mno-apxf %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-APXF %s
 //
-// APXF: "-target-feature" "+egpr" "-target-feature" "+push2pop2" "-target-feature" "+ppx" "-target-feature" "+ndd" "-target-feature" "+ccmp" "-target-feature" "+nf" "-target-feature" "+cf"
-// NO-APXF: "-target-feature" "-egpr" "-target-feature" "-push2pop2" "-target-feature" "-ppx" "-target-feature" "-ndd" "-target-feature" "-ccmp" "-target-feature" "-nf" "-target-feature" "-cf"
+// APXF: "-target-feature" "+egpr" "-target-feature" "+push2pop2" "-target-feature" "+ppx" "-target-feature" "+ndd" "-target-feature" "+ccmp" "-target-feature" "+nf" "-target-feature" "+cf" "-target-feature" "+zu"
+// NO-APXF: "-target-feature" "-egpr" "-target-feature" "-push2pop2" "-target-feature" "-ppx" "-target-feature" "-ndd" "-target-feature" "-ccmp" "-target-feature" "-nf" "-target-feature" "-cf" "-target-feature" "-zu"
 
 // RUN: %clang -target x86_64-unknown-linux-gnu -mapx-features=egpr %s -### -o %t.o 2>&1 | FileCheck -check-prefix=EGPR %s
 // RUN: %clang -target x86_64-unknown-linux-gnu -mapx-features=push2pop2 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=PUSH2POP2 %s
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 5c0b815c8ae6f..1babf4747ba15 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -755,7 +755,7 @@
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=nf -x c -E -dM -o - %s | FileCheck --check-prefix=NF %s
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=cf -x c -E -dM -o - %s | FileCheck --check-prefix=CF %s
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=zu -x c -E -dM -o - %s | FileCheck --check-prefix=ZU %s
-// RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapxf -x c -E -dM -o - %s | FileCheck --check-prefixes=EGPR,PUSH2POP2,PPX,NDD,CCMP,NF,CF,APXF %s
+// RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapxf -x c -E -dM -o - %s | FileCheck --check-prefixes=EGPR,PUSH2POP2,PPX,NDD,CCMP,NF,CF,ZU,APXF %s
 // APXF: #define __APX_F__ 1
 // CCMP: #define __CCMP__ 1
 // CF: #define __CF__ 1

From 592233a962fc870b5270d4e55aab5fe8941ac676 Mon Sep 17 00:00:00 2001
From: Brandon Wu <brandon.wu@sifive.com>
Date: Fri, 19 Jul 2024 12:36:47 +0800
Subject: [PATCH 135/486] [TableGen][SelectionDAG] Make CheckValueTypeMatcher
 use MVT::SimpleValueType (#99537)

The original `CheckValueTypeMatcher` stores StringRef as the member
variable type, however it's more efficient to use use
MVT::SimpleValueType since it prevents string comparison in isEqualImpl,
it also reduce the memory consumption in each object.
---
 llvm/utils/TableGen/Common/DAGISelMatcher.cpp |  4 ++--
 llvm/utils/TableGen/Common/DAGISelMatcher.h   | 10 +++++-----
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp |  4 ++--
 llvm/utils/TableGen/DAGISelMatcherGen.cpp     |  2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/utils/TableGen/Common/DAGISelMatcher.cpp b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp
index 3298965ab41d7..71bbeaaba9a91 100644
--- a/llvm/utils/TableGen/Common/DAGISelMatcher.cpp
+++ b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp
@@ -218,7 +218,7 @@ void CheckChild2CondCodeMatcher::printImpl(raw_ostream &OS,
 }
 
 void CheckValueTypeMatcher::printImpl(raw_ostream &OS, unsigned indent) const {
-  OS.indent(indent) << "CheckValueType MVT::" << TypeName << '\n';
+  OS.indent(indent) << "CheckValueType " << getEnumName(VT) << '\n';
 }
 
 void CheckComplexPatMatcher::printImpl(raw_ostream &OS, unsigned indent) const {
@@ -409,7 +409,7 @@ bool CheckChildIntegerMatcher::isContradictoryImpl(const Matcher *M) const {
 
 bool CheckValueTypeMatcher::isContradictoryImpl(const Matcher *M) const {
   if (const CheckValueTypeMatcher *CVT = dyn_cast<CheckValueTypeMatcher>(M))
-    return CVT->getTypeName() != getTypeName();
+    return CVT->getVT() != getVT();
   return false;
 }
 
diff --git a/llvm/utils/TableGen/Common/DAGISelMatcher.h b/llvm/utils/TableGen/Common/DAGISelMatcher.h
index d4fe513e2e967..1d78b93310f1c 100644
--- a/llvm/utils/TableGen/Common/DAGISelMatcher.h
+++ b/llvm/utils/TableGen/Common/DAGISelMatcher.h
@@ -684,13 +684,13 @@ class CheckChild2CondCodeMatcher : public Matcher {
 /// CheckValueTypeMatcher - This checks to see if the current node is a
 /// VTSDNode with the specified type, if not it fails to match.
 class CheckValueTypeMatcher : public Matcher {
-  StringRef TypeName;
+  MVT::SimpleValueType VT;
 
 public:
-  CheckValueTypeMatcher(StringRef type_name)
-      : Matcher(CheckValueType), TypeName(type_name) {}
+  CheckValueTypeMatcher(MVT::SimpleValueType SimpleVT)
+      : Matcher(CheckValueType), VT(SimpleVT) {}
 
-  StringRef getTypeName() const { return TypeName; }
+  MVT::SimpleValueType getVT() const { return VT; }
 
   static bool classof(const Matcher *N) {
     return N->getKind() == CheckValueType;
@@ -699,7 +699,7 @@ class CheckValueTypeMatcher : public Matcher {
 private:
   void printImpl(raw_ostream &OS, unsigned indent) const override;
   bool isEqualImpl(const Matcher *M) const override {
-    return cast<CheckValueTypeMatcher>(M)->TypeName == TypeName;
+    return cast<CheckValueTypeMatcher>(M)->VT == VT;
   }
   bool isContradictoryImpl(const Matcher *M) const override;
 };
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index ff508d6487333..229245ff40504 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -697,8 +697,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
     return 2;
 
   case Matcher::CheckValueType:
-    OS << "OPC_CheckValueType, MVT::"
-       << cast<CheckValueTypeMatcher>(N)->getTypeName() << ",\n";
+    OS << "OPC_CheckValueType, "
+       << getEnumName(cast<CheckValueTypeMatcher>(N)->getVT()) << ",\n";
     return 2;
 
   case Matcher::CheckComplexPat: {
diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
index 99babdf07316f..db3fe8d2993f2 100644
--- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
@@ -235,7 +235,7 @@ void MatcherGen::EmitLeafMatchCode(const TreePatternNode &N) {
     if (N.hasName())
       return;
     // An unnamed ValueType as in (sext_inreg GPR:$foo, i8).
-    return AddMatcher(new CheckValueTypeMatcher(LeafRec->getName()));
+    return AddMatcher(new CheckValueTypeMatcher(llvm::getValueType(LeafRec)));
   }
 
   if ( // Handle register references.  Nothing to do here, they always match.

From 79a0b665934a3de1113d1df90a9d677f39d9389a Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Thu, 18 Jul 2024 21:47:48 -0700
Subject: [PATCH 136/486] [BOLT] Add MC dependency for Profile

---
 bolt/lib/Profile/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bolt/lib/Profile/CMakeLists.txt b/bolt/lib/Profile/CMakeLists.txt
index 39708baac4036..9aa4ba0490b0f 100644
--- a/bolt/lib/Profile/CMakeLists.txt
+++ b/bolt/lib/Profile/CMakeLists.txt
@@ -11,6 +11,7 @@ add_llvm_library(LLVMBOLTProfile
 
   LINK_COMPONENTS
   Demangle
+  MC
   Support
   TransformUtils
   )

From 82af008d9891bc109ba218fb546170d83c5de9a2 Mon Sep 17 00:00:00 2001
From: Rainer Orth <ro@gcc.gnu.org>
Date: Fri, 19 Jul 2024 07:15:10 +0200
Subject: [PATCH 137/486] [safestack] Various 32-bit Linux fixes (#99455)

When enabling 32-bit testing on Linux/i386 and Linux/sparc, many tests
`FAIL`:

- All Linux/i386 tests `FAIL` with ``` safestack CHECK failed:
/vol/llvm/src/llvm-project/local/compiler-rt/lib/safestack/safestack.cpp:95
MAP_FAILED != addr ``` because the safestack `mmap` implementation
doesn't work there. This patch adjusts it to match the
`sanitizer_linux.cpp.c` one.
- On 32-bit Linux/sparc, the `pthread*.c` tests `FAIL` because a `tid_t`
(`uint64_t`) `tid` arg was passed to `syscall(SYS_tgkill)` while `tid`
is actually a `pid_t` (`int`). Fixed by adding a cast.

Tested on `x86_64-pc-linux-gnu` (32 and 64-bit) and
`sparc64-unknown-linux-gnu` (32 and 64-bit).
---
 .../lib/safestack/safestack_platform.h        | 25 +++++++++++++++++--
 compiler-rt/lib/safestack/safestack_util.h    |  4 +++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/safestack/safestack_platform.h b/compiler-rt/lib/safestack/safestack_platform.h
index 77eeb9cda6e15..68d26813bef4a 100644
--- a/compiler-rt/lib/safestack/safestack_platform.h
+++ b/compiler-rt/lib/safestack/safestack_platform.h
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 //
 // This file implements platform specific parts of SafeStack runtime.
+// Don't use equivalent functionality from sanitizer_common to avoid dragging
+// a large codebase into security sensitive code.
 //
 //===----------------------------------------------------------------------===//
 
@@ -45,6 +47,19 @@ extern "C" void *__mmap(void *, size_t, int, int, int, int, off_t);
 #  include <thread.h>
 #endif
 
+// Keep in sync with sanitizer_linux.cpp.
+//
+// Are we using 32-bit or 64-bit Linux syscalls?
+// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32
+// but it still needs to use 64-bit syscalls.
+#if SANITIZER_LINUX &&                                \
+    (defined(__x86_64__) || defined(__powerpc64__) || \
+     SANITIZER_WORDSIZE == 64 || (defined(__mips__) && _MIPS_SIM == _ABIN32))
+#  define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
+#else
+#  define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
+#endif
+
 namespace safestack {
 
 #if SANITIZER_NETBSD
@@ -117,7 +132,8 @@ inline int TgKill(pid_t pid, ThreadId tid, int sig) {
 #elif SANITIZER_FREEBSD
   return syscall(SYS_thr_kill2, pid, tid, sig);
 #else
-  return syscall(SYS_tgkill, pid, tid, sig);
+  // tid is pid_t (int), not ThreadId (uint64_t).
+  return syscall(SYS_tgkill, pid, (pid_t)tid, sig);
 #endif
 }
 
@@ -129,8 +145,13 @@ inline void *Mmap(void *addr, size_t length, int prot, int flags, int fd,
   return (void *)__syscall(SYS_mmap, addr, length, prot, flags, fd, offset);
 #elif SANITIZER_SOLARIS
   return _REAL64(mmap)(addr, length, prot, flags, fd, offset);
-#else
+#elif SANITIZER_LINUX_USES_64BIT_SYSCALLS
   return (void *)syscall(SYS_mmap, addr, length, prot, flags, fd, offset);
+#else
+  // mmap2 specifies file offset in 4096-byte units.
+  SFS_CHECK(IsAligned(offset, 4096));
+  return (void *)syscall(SYS_mmap2, addr, length, prot, flags, fd,
+                         offset / 4096);
 #endif
 }
 
diff --git a/compiler-rt/lib/safestack/safestack_util.h b/compiler-rt/lib/safestack/safestack_util.h
index da3f11b54cabc..ca724312c3eec 100644
--- a/compiler-rt/lib/safestack/safestack_util.h
+++ b/compiler-rt/lib/safestack/safestack_util.h
@@ -33,6 +33,10 @@ inline size_t RoundUpTo(size_t size, size_t boundary) {
   return (size + boundary - 1) & ~(boundary - 1);
 }
 
+inline constexpr bool IsAligned(size_t a, size_t alignment) {
+  return (a & (alignment - 1)) == 0;
+}
+
 class MutexLock {
  public:
   explicit MutexLock(pthread_mutex_t &mutex) : mutex_(&mutex) {

From 2e5b4516b70be9b8d45d4ecd0bd72c41e91ce6fb Mon Sep 17 00:00:00 2001
From: Shan Huang <52285902006@stu.ecnu.edu.cn>
Date: Fri, 19 Jul 2024 13:17:21 +0800
Subject: [PATCH 138/486] [DebugInfo][SimpleLoopUnswitch] Fix missing debug
 location updates for new terminators (#98789)

Fix #98787 .
---
 .../Transforms/Scalar/SimpleLoopUnswitch.cpp  | 21 ++--
 ...preserving-debugloc-trivial-terminators.ll | 95 +++++++++++++++++++
 2 files changed, 109 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/preserving-debugloc-trivial-terminators.ll

diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index 6a3b0e1b43e21..c235d2fb2a5bd 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -632,7 +632,8 @@ static bool unswitchTrivialBranch(Loop &L, BranchInst &BI, DominatorTree &DT,
     } else {
       // Create a new unconditional branch that will continue the loop as a new
       // terminator.
-      BranchInst::Create(ContinueBB, ParentBB);
+      Instruction *NewBI = BranchInst::Create(ContinueBB, ParentBB);
+      NewBI->setDebugLoc(BI.getDebugLoc());
     }
     BI.setSuccessor(LoopExitSuccIdx, UnswitchedBB);
     BI.setSuccessor(1 - LoopExitSuccIdx, NewPH);
@@ -666,10 +667,12 @@ static bool unswitchTrivialBranch(Loop &L, BranchInst &BI, DominatorTree &DT,
   // Finish updating dominator tree and memory ssa for full unswitch.
   if (FullUnswitch) {
     if (MSSAU) {
-      // Remove the cloned branch instruction.
-      ParentBB->getTerminator()->eraseFromParent();
-      // Create unconditional branch now.
-      BranchInst::Create(ContinueBB, ParentBB);
+      Instruction *Term = ParentBB->getTerminator();
+      // Remove the cloned branch instruction and create unconditional branch
+      // now.
+      Instruction *NewBI = BranchInst::Create(ContinueBB, ParentBB);
+      NewBI->setDebugLoc(Term->getDebugLoc());
+      Term->eraseFromParent();
       MSSAU->removeEdge(ParentBB, LoopExitBB);
     }
     DT.deleteEdge(ParentBB, LoopExitBB);
@@ -861,8 +864,11 @@ static bool unswitchTrivialSwitch(Loop &L, SwitchInst &SI, DominatorTree &DT,
   BasicBlock *NewPH = SplitEdge(OldPH, L.getHeader(), &DT, &LI, MSSAU);
   OldPH->getTerminator()->eraseFromParent();
 
-  // Now add the unswitched switch.
+  // Now add the unswitched switch. This new switch instruction inherits the
+  // debug location of the old switch, because it semantically replace the old
+  // one.
   auto *NewSI = SwitchInst::Create(LoopCond, NewPH, ExitCases.size(), OldPH);
+  NewSI->setDebugLoc(SIW->getDebugLoc());
   SwitchInstProfUpdateWrapper NewSIW(*NewSI);
 
   // Rewrite the IR for the unswitched basic blocks. This requires two steps.
@@ -972,8 +978,9 @@ static bool unswitchTrivialSwitch(Loop &L, SwitchInst &SI, DominatorTree &DT,
                                       /*KeepOneInputPHIs*/ true);
     }
     // Now nuke the switch and replace it with a direct branch.
+    Instruction *NewBI = BranchInst::Create(CommonSuccBB, BB);
+    NewBI->setDebugLoc(SIW->getDebugLoc());
     SIW.eraseFromParent();
-    BranchInst::Create(CommonSuccBB, BB);
   } else if (DefaultExitBB) {
     assert(SI.getNumCases() > 0 &&
            "If we had no cases we'd have a common successor!");
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/preserving-debugloc-trivial-terminators.ll b/llvm/test/Transforms/SimpleLoopUnswitch/preserving-debugloc-trivial-terminators.ll
new file mode 100644
index 0000000000000..a17430f2c8012
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/preserving-debugloc-trivial-terminators.ll
@@ -0,0 +1,95 @@
+; RUN: opt -passes='loop(simple-loop-unswitch)' -S < %s | FileCheck %s
+; RUN: opt -passes='loop-mssa(simple-loop-unswitch)' -S < %s | FileCheck %s
+
+; Check that SimpleLoopUnswitch's unswitchTrivialBranch() and unswitchTrivialSwitch()
+; propagates debug locations to the new terminators replacing the old ones.
+
+define i32 @test1(ptr %var, i1 %cond1, i1 %cond2) !dbg !5 {
+; CHECK-LABEL: define i32 @test1(
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %[[CONTINUE:.*]], !dbg [[DBG8:![0-9]+]]
+;
+entry:
+  br label %loop_begin, !dbg !8
+
+loop_begin:                                       ; preds = %do_something, %entry
+  br i1 %cond1, label %continue, label %loop_exit, !dbg !9
+
+continue:                                         ; preds = %loop_begin
+  %var_val = load i32, ptr %var, align 4, !dbg !10
+  br i1 %cond2, label %do_something, label %loop_exit, !dbg !11
+
+do_something:                                     ; preds = %continue
+  call void @some_func(), !dbg !12
+  br label %loop_begin, !dbg !13
+
+loop_exit:                                        ; preds = %continue, %loop_begin
+  ret i32 0, !dbg !14
+}
+
+define i32 @test7(i32 %cond1, i32 %x, i32 %y) !dbg !15 {
+; CHECK-LABEL: define i32 @test7(
+; CHECK-SAME: i32 [[COND1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[COND1]], label %[[ENTRY_SPLIT:.*]] [
+; CHECK-NEXT:      i32 0, label %[[LOOP_EXIT:.*]]
+; CHECK-NEXT:      i32 1, label %[[LOOP_EXIT]]
+; CHECK-NEXT:    ], !dbg [[DBG16:![0-9]+]]
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %[[LATCH:.*]], !dbg [[DBG16]]
+;
+entry:
+  br label %loop_begin, !dbg !16
+
+loop_begin:                                       ; preds = %latch, %entry
+  switch i32 %cond1, label %latch [
+  i32 0, label %loop_exit
+  i32 1, label %loop_exit
+  ], !dbg !17
+
+latch:                                            ; preds = %loop_begin
+  call void @some_func(), !dbg !18
+  br label %loop_begin, !dbg !19
+
+loop_exit:                                        ; preds = %loop_begin, %loop_begin
+  %result1 = phi i32 [ %x, %loop_begin ], [ %x, %loop_begin ], !dbg !20
+  %result2 = phi i32 [ %y, %loop_begin ], [ %y, %loop_begin ], !dbg !21
+  %result = add i32 %result1, %result2, !dbg !22
+  ret i32 %result, !dbg !23
+}
+
+; Function Attrs: noreturn
+declare void @some_func()
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+; CHECK: [[DBG8]] = !DILocation(line: 2,
+
+; CHECK: [[DBG16]] = !DILocation(line: 9,
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "test2.ll", directory: "/")
+!2 = !{i32 15}
+!3 = !{i32 0}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !DILocation(line: 1, column: 1, scope: !5)
+!9 = !DILocation(line: 2, column: 1, scope: !5)
+!10 = !DILocation(line: 3, column: 1, scope: !5)
+!11 = !DILocation(line: 4, column: 1, scope: !5)
+!12 = !DILocation(line: 5, column: 1, scope: !5)
+!13 = !DILocation(line: 6, column: 1, scope: !5)
+!14 = !DILocation(line: 7, column: 1, scope: !5)
+!15 = distinct !DISubprogram(name: "test7", linkageName: "test7", scope: null, file: !1, line: 8, type: !6, scopeLine: 8, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!16 = !DILocation(line: 8, column: 1, scope: !15)
+!17 = !DILocation(line: 9, column: 1, scope: !15)
+!18 = !DILocation(line: 10, column: 1, scope: !15)
+!19 = !DILocation(line: 11, column: 1, scope: !15)
+!20 = !DILocation(line: 12, column: 1, scope: !15)
+!21 = !DILocation(line: 13, column: 1, scope: !15)
+!22 = !DILocation(line: 14, column: 1, scope: !15)
+!23 = !DILocation(line: 15, column: 1, scope: !15)

From 3b78dfa10c4b77581cc29c4510aefe919ae660ba Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Thu, 18 Jul 2024 22:17:42 -0700
Subject: [PATCH 139/486] [libc][libcxx] Support for building libc++ against
 LLVM libc (#99287)

Provide an option to build libc++ against LLVM libc and set the CMake
compile and link options appropriately when the option is enabled.
---
 clang/cmake/caches/Fuchsia-stage2.cmake |  2 ++
 libcxx/CMakeLists.txt                   |  9 ++++++
 libcxx/cmake/Modules/HandleLibC.cmake   | 39 +++++++++++++++++++++++++
 libcxx/cmake/config-ix.cmake            |  3 ++
 libcxx/include/CMakeLists.txt           |  2 +-
 libcxx/src/CMakeLists.txt               |  4 +--
 6 files changed, 56 insertions(+), 3 deletions(-)
 create mode 100644 libcxx/cmake/Modules/HandleLibC.cmake

diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 73ebd36c28496..0218f0b21eb28 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -339,6 +339,7 @@ foreach(target armv6m-unknown-eabi;armv7m-unknown-eabi;armv8m.main-unknown-eabi)
   set(RUNTIMES_${target}_LIBCXX_ENABLE_SHARED OFF CACHE BOOL "")
   set(RUNTIMES_${target}_LIBCXX_ENABLE_STATIC ON CACHE BOOL "")
   set(RUNTIMES_${target}_LIBCXX_ENABLE_NEW_DELETE_DEFINITIONS ON CACHE BOOL "")
+  set(RUNTIMES_${target}_LIBCXX_LIBC "llvm-libc" CACHE STRING "")
   set(RUNTIMES_${target}_LIBCXX_ENABLE_FILESYSTEM OFF CACHE BOOL "")
   set(RUNTIMES_${target}_LIBCXX_ENABLE_RANDOM_DEVICE OFF CACHE BOOL "")
   set(RUNTIMES_${target}_LIBCXX_ENABLE_LOCALIZATION OFF CACHE BOOL "")
@@ -389,6 +390,7 @@ foreach(target riscv32-unknown-elf)
   set(RUNTIMES_${target}_LIBCXX_ENABLE_SHARED OFF CACHE BOOL "")
   set(RUNTIMES_${target}_LIBCXX_ENABLE_STATIC ON CACHE BOOL "")
   set(RUNTIMES_${target}_LIBCXX_ENABLE_NEW_DELETE_DEFINITIONS ON CACHE BOOL "")
+  set(RUNTIMES_${target}_LIBCXX_LIBC "llvm-libc" CACHE STRING "")
   set(RUNTIMES_${target}_LIBCXX_ENABLE_FILESYSTEM OFF CACHE BOOL "")
   set(RUNTIMES_${target}_LIBCXX_ENABLE_RANDOM_DEVICE OFF CACHE BOOL "")
   set(RUNTIMES_${target}_LIBCXX_ENABLE_LOCALIZATION OFF CACHE BOOL "")
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 155f81a74a974..332816b15260a 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -212,6 +212,14 @@ set(LIBCXX_ABI_DEFINES "" CACHE STRING "A semicolon separated list of ABI macros
 set(LIBCXX_EXTRA_SITE_DEFINES "" CACHE STRING "Extra defines to add into __config_site")
 option(LIBCXX_USE_COMPILER_RT "Use compiler-rt instead of libgcc" OFF)
 
+# C Library options -----------------------------------------------------------
+
+set(LIBCXX_SUPPORTED_C_LIBRARIES system llvm-libc)
+set(LIBCXX_LIBC "system" CACHE STRING "Specify C library to use. Supported values are ${LIBCXX_SUPPORTED_C_LIBRARIES}.")
+if (NOT "${LIBCXX_LIBC}" IN_LIST LIBCXX_SUPPORTED_C_LIBRARIES)
+  message(FATAL_ERROR "Unsupported C library: '${LIBCXX_CXX_ABI}'. Supported values are ${LIBCXX_SUPPORTED_C_LIBRARIES}.")
+endif()
+
 # ABI Library options ---------------------------------------------------------
 if (MSVC)
   set(LIBCXX_DEFAULT_ABI_LIBRARY "vcruntime")
@@ -495,6 +503,7 @@ endif()
 # Setup Compiler Flags
 #===============================================================================
 
+include(HandleLibC) # Setup the C library flags
 include(HandleLibCXXABI) # Setup the ABI library flags
 
 # FIXME(EricWF): See the FIXME on LIBCXX_ENABLE_PEDANTIC.
diff --git a/libcxx/cmake/Modules/HandleLibC.cmake b/libcxx/cmake/Modules/HandleLibC.cmake
new file mode 100644
index 0000000000000..1b0564ae6fcc6
--- /dev/null
+++ b/libcxx/cmake/Modules/HandleLibC.cmake
@@ -0,0 +1,39 @@
+#===============================================================================
+# Define targets for linking against the selected C library
+#
+# After including this file, the following targets are defined:
+# - libcxx-libc-headers: An interface target that allows getting access to the
+#                        headers of the selected C library.
+# - libcxx-libc-shared: A target representing the selected shared C library.
+# - libcxx-libc-static: A target representing the selected static C library.
+#===============================================================================
+
+# Link against a system-provided libc
+if (LIBCXX_LIBC STREQUAL "system")
+  add_library(libcxx-libc-headers INTERFACE)
+
+  add_library(libcxx-libc-static INTERFACE)
+  add_library(libcxx-libc-shared INTERFACE)
+
+# Link against the in-tree LLVM libc
+elseif (LIBCXX_LIBC STREQUAL "llvm-libc")
+  add_library(libcxx-libc-headers INTERFACE)
+  target_link_libraries(libcxx-libc-headers INTERFACE libc-headers)
+  if(CXX_SUPPORTS_NOSTDLIBINC_FLAG)
+    target_compile_options(libcxx-libc-headers INTERFACE "-nostdlibinc")
+  endif()
+
+  add_library(libcxx-libc-static INTERFACE)
+  if (TARGET libc)
+    target_link_libraries(libcxx-libc-static INTERFACE libc)
+  endif()
+  if (TARGET libm)
+    target_link_libraries(libcxx-libc-static INTERFACE libm)
+  endif()
+  if (CXX_SUPPORTS_NOLIBC_FLAG)
+    target_link_options(libcxx-libc-static INTERFACE "-nolibc")
+  endif()
+
+  # TODO: There's no support for building LLVM libc as a shared library yet.
+  add_library(libcxx-libc-shared INTERFACE)
+endif()
diff --git a/libcxx/cmake/config-ix.cmake b/libcxx/cmake/config-ix.cmake
index 7406fba482e69..998819604de28 100644
--- a/libcxx/cmake/config-ix.cmake
+++ b/libcxx/cmake/config-ix.cmake
@@ -27,6 +27,9 @@ if (NOT LIBCXX_USE_COMPILER_RT)
   endif()
 endif()
 
+check_cxx_compiler_flag(-nostdlibinc CXX_SUPPORTS_NOSTDLIBINC_FLAG)
+check_cxx_compiler_flag(-nolibc CXX_SUPPORTS_NOLIBC_FLAG)
+
 # libc++ is using -nostdlib++ at the link step when available,
 # otherwise -nodefaultlibs is used. We want all our checks to also
 # use one of these options, otherwise we may end up with an inconsistency between
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 4b036ec639507..fa6736dc11e66 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -1043,7 +1043,7 @@ list(APPEND _all_includes "${LIBCXX_GENERATED_INCLUDE_DIR}/libcxx.imp")
 add_custom_target(generate-cxx-headers ALL DEPENDS ${_all_includes})
 
 add_library(cxx-headers INTERFACE)
-target_link_libraries(cxx-headers INTERFACE libcxx-abi-headers)
+target_link_libraries(cxx-headers INTERFACE libcxx-libc-headers libcxx-abi-headers)
 add_dependencies(cxx-headers generate-cxx-headers)
 # It's important that the arch directory be included first so that its header files
 # which interpose on the default include dir be included instead of the default ones.
diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index 0ae58a10c879c..0dfc9647558d4 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -199,7 +199,7 @@ split_list(LIBCXX_LINK_FLAGS)
 if (LIBCXX_ENABLE_SHARED)
   add_library(cxx_shared SHARED ${exclude_from_all} ${LIBCXX_SOURCES} ${LIBCXX_HEADERS})
   target_include_directories(cxx_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-  target_link_libraries(cxx_shared PUBLIC cxx-headers
+  target_link_libraries(cxx_shared PUBLIC cxx-headers libcxx-libc-shared
                                    PRIVATE ${LIBCXX_LIBRARIES})
   set_target_properties(cxx_shared
     PROPERTIES
@@ -292,7 +292,7 @@ set(CMAKE_STATIC_LIBRARY_PREFIX "lib")
 if (LIBCXX_ENABLE_STATIC)
   add_library(cxx_static STATIC ${exclude_from_all} ${LIBCXX_SOURCES} ${LIBCXX_HEADERS})
   target_include_directories(cxx_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-  target_link_libraries(cxx_static PUBLIC cxx-headers
+  target_link_libraries(cxx_static PUBLIC cxx-headers libcxx-libc-static
                                    PRIVATE ${LIBCXX_LIBRARIES}
                                    PRIVATE libcxx-abi-static)
   set_target_properties(cxx_static

From 2f0910d2d74419ef1ebf814b471af721ee78b464 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Fri, 19 Jul 2024 13:28:37 +0800
Subject: [PATCH 140/486] [C++20] [Modules] Skip ODR checks if either
 declaration comes from GMF

This patch tries to workaround the case that:
- in a module unit that imports another module unit
- both the module units including overlapped headers
- the compiler emits false positive ODR violation diagnostics for the
  overlapped headers if ODR check is enabled
- the current module units enables PCH

For the third point, we disabled ODR check if the declarations comes
from GMF. However, due to the forth point, the check whether the
declaration comes from GMF failed. Then we still going to check it and
then the users get false positive checks.

What's worse is that, this always happens in clangd, where will generate
the PCH automatically before parsing the input files.

The root cause of the problem we mixed the modules in the semantical
level and the module in the serialization level.

The problem is pretty fundamental and we need time to fix that. But 19.x
is going to be branched and I hope to give clangd better user
experience. So I decided to land this workaround even if it is pretyy
niche and may only work for the case of clangd's pattern.
---
 clang/lib/Serialization/ASTReader.cpp       |  1 +
 clang/lib/Serialization/ASTReaderDecl.cpp   |  4 +-
 clang/test/Modules/pch-in-module-units.cppm | 51 +++++++++++++++++++++
 3 files changed, 54 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/Modules/pch-in-module-units.cppm

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index afdeccaf93a9d..3cb96df12e4da 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -9875,6 +9875,7 @@ void ASTReader::finishPendingActions() {
             // We only perform ODR checks for decls not in the explicit
             // global module fragment.
             !shouldSkipCheckingODR(FD) &&
+            !shouldSkipCheckingODR(NonConstDefn) &&
             FD->getODRHash() != NonConstDefn->getODRHash()) {
           if (!isa<CXXMethodDecl>(FD)) {
             PendingFunctionOdrMergeFailures[FD].push_back(NonConstDefn);
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 3de9d327f1b3b..31ab6c651d59f 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -821,7 +821,7 @@ void ASTDeclReader::VisitEnumDecl(EnumDecl *ED) {
       Reader.mergeDefinitionVisibility(OldDef, ED);
       // We don't want to check the ODR hash value for declarations from global
       // module fragment.
-      if (!shouldSkipCheckingODR(ED) &&
+      if (!shouldSkipCheckingODR(ED) && !shouldSkipCheckingODR(OldDef) &&
           OldDef->getODRHash() != ED->getODRHash())
         Reader.PendingEnumOdrMergeFailures[OldDef].push_back(ED);
     } else {
@@ -2134,7 +2134,7 @@ void ASTDeclReader::MergeDefinitionData(
   }
 
   // We don't want to check ODR for decls in the global module fragment.
-  if (shouldSkipCheckingODR(MergeDD.Definition))
+  if (shouldSkipCheckingODR(MergeDD.Definition) || shouldSkipCheckingODR(D))
     return;
 
   if (D->getODRHash() != MergeDD.ODRHash) {
diff --git a/clang/test/Modules/pch-in-module-units.cppm b/clang/test/Modules/pch-in-module-units.cppm
new file mode 100644
index 0000000000000..790a3af09a0ca
--- /dev/null
+++ b/clang/test/Modules/pch-in-module-units.cppm
@@ -0,0 +1,51 @@
+// Test that we will skip ODR checks for declarations from PCH if they
+// were from GMF.
+//
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/A.cppm \
+// RUN:   -o %t/A.pcm -fskip-odr-check-in-gmf
+// RUN: %clang_cc1 -std=c++20 -DDIFF -x c++-header %t/foo.h \
+// RUN:   -emit-pch -o %t/foo.pch -fskip-odr-check-in-gmf
+// RUN: %clang_cc1 -std=c++20 %t/B.cppm -fmodule-file=A=%t/A.pcm -include-pch \
+// RUN:   %t/foo.pch -verify -fsyntax-only -fskip-odr-check-in-gmf
+
+//--- foo.h
+#ifndef FOO_H
+#define FOO_H
+inline int foo() {
+#ifndef DIFF
+  return 43;
+#else
+  return 45;
+#endif
+}
+
+class f {
+public:
+  int mem() {
+#ifndef DIFF
+    return 47;
+#else
+    return 45;
+#endif
+  }
+};
+#endif
+
+//--- A.cppm
+module;
+#include "foo.h"
+export module A;
+export using ::foo;
+export using ::f;
+
+//--- B.cppm
+// expected-no-diagnostics
+module;
+#include "foo.h"
+export module B;
+import A;
+export int b = foo() + f().mem();

From 377e1ebdd4ed64c996566b8cede857c7e4c1fc0f Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 18 Jul 2024 23:02:03 -0700
Subject: [PATCH 141/486] [asan] Consume leading zeroes in a test

---
 compiler-rt/test/asan/TestCases/debug_stacks.cpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/compiler-rt/test/asan/TestCases/debug_stacks.cpp b/compiler-rt/test/asan/TestCases/debug_stacks.cpp
index 7c320bfcb2d98..67a9ac849c3aa 100644
--- a/compiler-rt/test/asan/TestCases/debug_stacks.cpp
+++ b/compiler-rt/test/asan/TestCases/debug_stacks.cpp
@@ -42,9 +42,9 @@ int main() {
   fprintf(stderr, "thread id = %d\n", thread_id);
   // CHECK: thread id = 0
   fprintf(stderr, "0x" PTR "\n", trace[0]);
-  // CHECK: [[ALLOC_FRAME_0:0x[0-9a-f]+]]
+  // CHECK: 0x{{0*}}[[ALLOC_FRAME_0:[0-9a-f]+]]
   fprintf(stderr, "0x" PTR "\n", trace[1]);
-  // CHECK: [[ALLOC_FRAME_1:0x[0-9a-f]+]]
+  // CHECK: 0x{{0*}}[[ALLOC_FRAME_1:[0-9a-f]+]]
 
   num_frames = 100;
   num_frames = __asan_get_free_stack(mem, trace, num_frames, &thread_id);
@@ -55,20 +55,20 @@ int main() {
   fprintf(stderr, "thread id = %d\n", thread_id);
   // CHECK: thread id = 0
   fprintf(stderr, "0x" PTR "\n", trace[0]);
-  // CHECK: [[FREE_FRAME_0:0x[0-9a-f]+]]
+  // CHECK: 0x{{0*}}[[FREE_FRAME_0:[0-9a-f]+]]
   fprintf(stderr, "0x" PTR "\n", trace[1]);
-  // CHECK: [[FREE_FRAME_1:0x[0-9a-f]+]]
+  // CHECK: 0x{{0*}}[[FREE_FRAME_1:[0-9a-f]+]]
 
   mem[0] = 'A'; // BOOM
 
   // CHECK: ERROR: AddressSanitizer: heap-use-after-free
   // CHECK: WRITE of size 1 at 0x{{.*}}
   // CHECK: freed by thread T0 here:
-  // CHECK: #0 [[FREE_FRAME_0]]
-  // CHECK: #1 [[FREE_FRAME_1]]
+  // CHECK: #0 0x{{0*}}[[FREE_FRAME_0]]
+  // CHECK: #1 0x{{0*}}[[FREE_FRAME_1]]
   // CHECK: previously allocated by thread T0 here:
-  // CHECK: #0 [[ALLOC_FRAME_0]]
-  // CHECK: #1 [[ALLOC_FRAME_1]]
+  // CHECK: #0 0x{{0*}}[[ALLOC_FRAME_0]]
+  // CHECK: #1 0x{{0*}}[[ALLOC_FRAME_1]]
 
   return 0;
 }

From 55a7be55a5fcf1c0837d62660157a45f8a52eb4f Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 18 Jul 2024 23:14:16 -0700
Subject: [PATCH 142/486] [AArch64,ELF] Use getCurrentSection().first in
 changeSection

Similar to the NVPTX change 4ae23bcca144b542f16d45acc8f270e156e2fa4e.
And improve the tests.
---
 .../MCTargetDesc/AArch64ELFStreamer.cpp       |  2 +-
 .../test/MC/AArch64/mapping-across-sections.s | 68 +++++++++++--------
 2 files changed, 41 insertions(+), 29 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index 7dba22c066dcd..bfeca4bd5a92d 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -187,7 +187,7 @@ class AArch64ELFStreamer : public MCELFStreamer {
     // We have to keep track of the mapping symbol state of any sections we
     // use. Each one should start off as EMS_None, which is provided as the
     // default constructor by DenseMap::lookup.
-    LastMappingSymbols[getPreviousSection().first] = LastEMS;
+    LastMappingSymbols[getCurrentSection().first] = LastEMS;
     LastEMS = LastMappingSymbols.lookup(Section);
 
     MCELFStreamer::changeSection(Section, Subsection);
diff --git a/llvm/test/MC/AArch64/mapping-across-sections.s b/llvm/test/MC/AArch64/mapping-across-sections.s
index 053deb760dcfd..6bb5a8811b57d 100644
--- a/llvm/test/MC/AArch64/mapping-across-sections.s
+++ b/llvm/test/MC/AArch64/mapping-across-sections.s
@@ -1,28 +1,40 @@
-// RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj < %s | llvm-objdump -t - | FileCheck %s
-
-        .text
-        add w0, w0, w0
-
-// .wibble should *not* inherit .text's mapping symbol. It's a completely different section.
-        .section .wibble
-        add w0, w0, w0
-
-// A setion should be able to start with a $d
-        .section .starts_data
-        .word 42
-
-// Changing back to .text should not emit a redundant $x
-        .text
-        add w0, w0, w0
-
-// With all those constraints, we want:
-//   + .text to have $x at 0 and no others
-//   + .wibble to have $x at 0
-//   + .starts_data to have $d at 0
-
-
-// CHECK:      0000000000000000 l .text        0000000000000000 $x
-// CHECK-NEXT: 0000000000000000 l .wibble      0000000000000000 $x
-// CHECK-NEXT: 0000000000000000 l .starts_data 0000000000000000 $d
-// CHECK-NOT: ${{[adtx]}}
-
+// RUN: llvm-mc -triple=aarch64 -filetype=obj %s | llvm-objdump -t - | FileCheck %s
+
+.section .text1,"ax"
+add w0, w0, w0
+
+.text
+add w0, w0, w0
+.word 42
+
+.pushsection .data,"aw"
+.word 42
+.popsection
+
+.text
+add w1, w1, w1
+
+.section .text1,"ax"
+add w1, w1, w1
+
+.text
+.word 42
+
+.section .rodata,"a"
+.word 42
+add w0, w0, w0
+
+.ident "clang"
+.section ".note.GNU-stack","",@progbits
+
+// CHECK:      SYMBOL TABLE:
+// CHECK-NEXT: 0000000000000000 l       .text1 0000000000000000 $x.0
+// CHECK-NEXT: 0000000000000000 l       .text  0000000000000000 $x.1
+// CHECK-NEXT: 0000000000000004 l       .text  0000000000000000 $d.2
+// CHECK-NEXT: 0000000000000000 l       .data  0000000000000000 $d.3
+// CHECK-NEXT: 0000000000000008 l       .text  0000000000000000 $x.4
+// CHECK-NEXT: 000000000000000c l       .text  0000000000000000 $d.5
+// CHECK-NEXT: 0000000000000000 l       .rodata        0000000000000000 $d.6
+// CHECK-NEXT: 0000000000000004 l       .rodata        0000000000000000 $x.7
+// CHECK-NEXT: 0000000000000000 l       .comment       0000000000000000 $d.8
+// CHECK-NOT:  {{.}}

From 6f710fef838185cbbe11193b30d6adc354b08b28 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Fri, 19 Jul 2024 14:20:48 +0800
Subject: [PATCH 143/486] [Doc] Update documentation for no-transitive-change
 (#96453)

(Some backgrounds, not required to read:
https://discourse.llvm.org/t/rfc-c-20-modules-introduce-thin-bmi-and-decls-hash/74755)

This is the document part for the no-transitive-change
(https://github.com/llvm/llvm-project/pull/86912,
https://github.com/llvm/llvm-project/pull/92083,
https://github.com/llvm/llvm-project/pull/92085,
https://github.com/llvm/llvm-project/pull/92511) to provide the ability
for build system to skip some unnecessary recompilations.

See the patch for examples.
---
 clang/docs/ReleaseNotes.rst             |   7 ++
 clang/docs/StandardCPlusPlusModules.rst | 143 ++++++++++++++++++++++++
 2 files changed, 150 insertions(+)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index e0e86af257a19..b0afadfa8d324 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -174,6 +174,13 @@ here. Generic improvements to Clang as a whole or to its underlying
 infrastructure are described first, followed by language-specific
 sections with improvements to Clang's support for those languages.
 
+- Implemented improvements to BMIs for C++20 Modules that can reduce
+  the number of rebuilds during incremental recompilation. We are seeking
+  feedback from Build System authors and other interested users, especially
+  when you feel Clang changes the BMI and misses an opportunity to avoid
+  recompilations or causes correctness issues. See StandardCPlusPlusModules
+  `StandardCPlusPlusModules <StandardCPlusPlusModules.html>`_ for more details.
+
 - The ``\par`` documentation comment command now supports an optional
   argument, which denotes the header of the paragraph started by
   an instance of the ``\par`` command comment. The implementation
diff --git a/clang/docs/StandardCPlusPlusModules.rst b/clang/docs/StandardCPlusPlusModules.rst
index cf0528e75e7f2..b87491910e222 100644
--- a/clang/docs/StandardCPlusPlusModules.rst
+++ b/clang/docs/StandardCPlusPlusModules.rst
@@ -652,6 +652,149 @@ in the future. The expected roadmap for Reduced BMIs as of Clang 19.x is:
    comes, the term BMI will refer to the Reduced BMI and the Full BMI will only
    be meaningful to build systems which elect to support two-phase compilation.
 
+Experimental Non-Cascading Changes
+----------------------------------
+
+This section is primarily for build system vendors. For end compiler users,
+if you don't want to read it all, this is helpful to reduce recompilations.
+We encourage build system vendors and end users try this out and bring feedback.
+
+Before Clang 19, a change in BMI of any (transitive) dependency would cause the
+outputs of the BMI to change. Starting with Clang 19, changes to non-direct
+dependencies should not directly affect the output BMI, unless they affect the
+results of the compilations. We expect that there are many more opportunities
+for this optimization than we currently have realized and would appreaciate 
+feedback about missed optimization opportunities. For example,
+
+.. code-block:: c++
+
+  // m-partA.cppm
+  export module m:partA;
+
+  // m-partB.cppm
+  export module m:partB;
+  export int getB() { return 44; }
+
+  // m.cppm
+  export module m;
+  export import :partA;
+  export import :partB;
+
+  // useBOnly.cppm
+  export module useBOnly;
+  import m;
+  export int B() {
+    return getB();
+  }
+
+  // Use.cc
+  import useBOnly;
+  int get() {
+    return B();
+  }
+
+To compile the project (for brevity, some commands are omitted.):
+
+.. code-block:: console
+
+  $ clang++ -std=c++20 m-partA.cppm --precompile -o m-partA.pcm
+  $ clang++ -std=c++20 m-partB.cppm --precompile -o m-partB.pcm
+  $ clang++ -std=c++20 m.cppm --precompile -o m.pcm -fprebuilt-module-path=.
+  $ clang++ -std=c++20 useBOnly.cppm --precompile -o useBOnly.pcm -fprebuilt-module-path=.
+  $ md5sum useBOnly.pcm
+  07656bf4a6908626795729295f9608da  useBOnly.pcm
+
+If the interface of ``m-partA.cppm`` is changed to:
+
+.. code-block:: c++
+
+  // m-partA.v1.cppm
+  export module m:partA;
+  export int getA() { return 43; }
+
+and the BMI for ``useBOnly`` is recompiled as in:
+
+.. code-block:: console
+
+  $ clang++ -std=c++20 m-partA.cppm --precompile -o m-partA.pcm
+  $ clang++ -std=c++20 m-partB.cppm --precompile -o m-partB.pcm
+  $ clang++ -std=c++20 m.cppm --precompile -o m.pcm -fprebuilt-module-path=.
+  $ clang++ -std=c++20 useBOnly.cppm --precompile -o useBOnly.pcm -fprebuilt-module-path=.
+  $ md5sum useBOnly.pcm
+  07656bf4a6908626795729295f9608da  useBOnly.pcm
+
+then the contents of ``useBOnly.pcm`` remain unchanged.
+Consequently, if the build system only bases recompilation decisions on directly imported modules,
+it becomes possible to skip the recompilation of ``Use.cc``.
+It should be fine because the altered interfaces do not affect ``Use.cc`` in any way;
+the changes do not cascade.
+
+When ``Clang`` generates a BMI, it records the hash values of all potentially contributory BMIs
+for the BMI being produced. This ensures that build systems are not required to consider
+transitively imported modules when deciding whether to recompile.
+
+What is considered to be a potential contributory BMIs is currently unspecified.
+However, it is a severe bug for a BMI to remain unchanged following an observable change
+that affects its consumers.
+
+Build systems may utilize this optimization by doing an update-if-changed operation to the BMI
+that is consumed from the BMI that is output by the compiler.
+
+We encourage build systems to add an experimental mode that
+reuses the cached BMI when **direct** dependencies did not change,
+even if **transitive** dependencies did change.
+
+Given there are potential compiler bugs, we recommend that build systems
+support this feature as a configurable option so that users
+can go back to the transitive change mode safely at any time.
+
+Interactions with Reduced BMI
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+With reduced BMI, non-cascading changes can be more powerful. For example,
+
+.. code-block:: c++
+
+  // A.cppm
+  export module A;
+  export int a() { return 44; }
+
+  // B.cppm
+  export module B;
+  import A;
+  export int b() { return a(); }
+
+.. code-block:: console
+
+  $ clang++ -std=c++20 A.cppm -c -fmodule-output=A.pcm  -fexperimental-modules-reduced-bmi -o A.o
+  $ clang++ -std=c++20 B.cppm -c -fmodule-output=B.pcm  -fexperimental-modules-reduced-bmi -o B.o -fmodule-file=A=A.pcm
+  $ md5sum B.pcm
+  6c2bd452ca32ab418bf35cd141b060b9  B.pcm
+
+And let's change the implementation for ``A.cppm`` into:
+
+.. code-block:: c++
+
+  export module A;
+  int a_impl() { return 99; }
+  export int a() { return a_impl(); }
+
+and recompile the example:
+
+.. code-block:: console
+
+  $ clang++ -std=c++20 A.cppm -c -fmodule-output=A.pcm  -fexperimental-modules-reduced-bmi -o A.o
+  $ clang++ -std=c++20 B.cppm -c -fmodule-output=B.pcm  -fexperimental-modules-reduced-bmi -o B.o -fmodule-file=A=A.pcm
+  $ md5sum B.pcm
+  6c2bd452ca32ab418bf35cd141b060b9  B.pcm
+
+We should find the contents of ``B.pcm`` remains the same. In this case, the build system is
+allowed to skip recompilations of TUs which solely and directly depend on module ``B``.
+
+This only happens with a reduced BMI. With reduced BMIs, we won't record the function body
+of ``int b()`` in the BMI for ``B`` so that the module ``A`` doesn't contribute to the BMI of ``B``
+and we have less dependencies.
+
 Performance Tips
 ----------------
 

From 90668d240417422827d0468aa32b6aaa88105859 Mon Sep 17 00:00:00 2001
From: Rajat Bajpai <rbajpai@nvidia.com>
Date: Fri, 19 Jul 2024 12:16:41 +0530
Subject: [PATCH 144/486] [CVP][LVI] Add support for InsertElementInst in LVI
 (#99368)

Currently, the LVI analysis pass doesn't support InsertElementInst
vector instruction. Due to this, some optimization opportunities are
missed. For example, in the below example, ICMP instruction can be
folded but it doesn't.

```
...
%ie1 = insertelement <2 x i32> poison, i32 10, i64 0
%ie2 = insertelement <2 x i32> %ie1, i32 20, i64 1
%icmp = icmp <2 x i1> %ie2, <i32 40, i32 40>
...
```

This change adds InsertElementInst support in the LVI analysis pass to
fix the motivating example.
---
 llvm/lib/Analysis/LazyValueInfo.cpp           | 23 +++++++++++++++++
 .../CorrelatedValuePropagation/vectors.ll     | 25 +++++++++++++++++++
 2 files changed, 48 insertions(+)

diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 92389f2896b8e..4c023ed5ed8f0 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -428,6 +428,8 @@ class LazyValueInfoImpl {
   std::optional<ValueLatticeElement> solveBlockValueIntrinsic(IntrinsicInst *II,
                                                               BasicBlock *BB);
   std::optional<ValueLatticeElement>
+  solveBlockValueInsertElement(InsertElementInst *IEI, BasicBlock *BB);
+  std::optional<ValueLatticeElement>
   solveBlockValueExtractValue(ExtractValueInst *EVI, BasicBlock *BB);
   bool isNonNullAtEndOfBlock(Value *Val, BasicBlock *BB);
   void intersectAssumeOrGuardBlockValueConstantRange(Value *Val,
@@ -657,6 +659,9 @@ LazyValueInfoImpl::solveBlockValueImpl(Value *Val, BasicBlock *BB) {
     if (BinaryOperator *BO = dyn_cast<BinaryOperator>(BBI))
       return solveBlockValueBinaryOp(BO, BB);
 
+    if (auto *IEI = dyn_cast<InsertElementInst>(BBI))
+      return solveBlockValueInsertElement(IEI, BB);
+
     if (auto *EVI = dyn_cast<ExtractValueInst>(BBI))
       return solveBlockValueExtractValue(EVI, BB);
 
@@ -1038,6 +1043,24 @@ LazyValueInfoImpl::solveBlockValueIntrinsic(IntrinsicInst *II, BasicBlock *BB) {
                    MetadataVal);
 }
 
+std::optional<ValueLatticeElement>
+LazyValueInfoImpl::solveBlockValueInsertElement(InsertElementInst *IEI,
+                                                BasicBlock *BB) {
+  std::optional<ValueLatticeElement> OptEltVal =
+      getBlockValue(IEI->getOperand(1), BB, IEI);
+  if (!OptEltVal)
+    return std::nullopt;
+  ValueLatticeElement &Res = *OptEltVal;
+
+  std::optional<ValueLatticeElement> OptVecVal =
+      getBlockValue(IEI->getOperand(0), BB, IEI);
+  if (!OptVecVal)
+    return std::nullopt;
+
+  Res.mergeIn(*OptVecVal);
+  return Res;
+}
+
 std::optional<ValueLatticeElement>
 LazyValueInfoImpl::solveBlockValueExtractValue(ExtractValueInst *EVI,
                                                BasicBlock *BB) {
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll b/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll
index 6254b54d42554..351a2c79cdff4 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll
@@ -327,3 +327,28 @@ join:
   %add = add <2 x i16> %phi, <i16 2, i16 3>
   ret <2 x i16> %add
 }
+
+;; Check if ICMP instruction is constant folded or not.
+define <2 x i1> @insertelement_fold1() {
+; CHECK-LABEL: define <2 x i1> @insertelement_fold1() {
+; CHECK-NEXT:    [[IE1:%.*]] = insertelement <2 x i32> poison, i32 10, i64 0
+; CHECK-NEXT:    [[IE2:%.*]] = insertelement <2 x i32> [[IE1]], i32 20, i64 1
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %ie1 = insertelement <2 x i32> poison, i32 10, i64 0
+  %ie2 = insertelement <2 x i32> %ie1, i32 20, i64 1
+  %icmp1 = icmp slt <2 x i32> %ie2, <i32 1024, i32 1024>
+  ret <2 x i1> %icmp1
+}
+
+;; Check if LVI is able to handle constant vector operands
+;; in InsertElementInst and CVP is able to fold ICMP instruction.
+define <2 x i1> @insertelement_fold2() {
+; CHECK-LABEL: define <2 x i1> @insertelement_fold2() {
+; CHECK-NEXT:    [[IE1:%.*]] = insertelement <2 x i32> <i32 poison, i32 20>, i32 10, i64 0
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %ie1 = insertelement <2 x i32> <i32 poison, i32 20>, i32 10, i64 0
+  %icmp1 = icmp slt <2 x i32> %ie1, <i32 1024, i32 1024>
+  ret <2 x i1> %icmp1
+}

From a1d77caaabbb5279b734c061dab36b2138ec476d Mon Sep 17 00:00:00 2001
From: Daniil Kovalev <dkovalev@accesssoftek.com>
Date: Fri, 19 Jul 2024 09:47:47 +0300
Subject: [PATCH 145/486] [test][PAC][clang] Add missing tests against linux
 triples (#99482)

Enhance tests introduced in #94056, #96992, #98276 and #98847 by adding
RUN and CHECK lines against linux triples.
---
 .../ptrauth-function-lvalue-cast-disc.c       | 10 +--
 ...ptrauth-function-type-discriminator-cast.c | 32 ++++++---
 .../ptrauth-function-type-discriminator.c     | 33 ++++++---
 clang/test/CodeGen/ptrauth-ubsan-vptr.cpp     |  3 +
 ...trauth-explicit-vtable-pointer-control.cpp | 62 ++++++++++-------
 clang/test/CodeGenCXX/ptrauth-rtti-layout.cpp | 14 ++--
 .../CodeGenCXX/ptrauth-static-destructors.cpp | 15 ++--
 clang/test/CodeGenCXX/ptrauth-throw.cpp       | 15 ++--
 clang/test/CodeGenCXX/ptrauth-thunks.cpp      |  3 +-
 .../CodeGenCXX/ptrauth-virtual-function.cpp   | 69 +++++++++++--------
 ...rauth-vtable-virtual-inheritance-thunk.cpp | 20 ++++--
 clang/test/CodeGenCXX/ubsan-vtable-checks.cpp |  4 +-
 ...irtual-member-function-return-arg-type.cpp |  3 +-
 ...table_pointer_authentication_attribute.cpp |  3 +-
 14 files changed, 185 insertions(+), 101 deletions(-)

diff --git a/clang/test/CodeGen/ptrauth-function-lvalue-cast-disc.c b/clang/test/CodeGen/ptrauth-function-lvalue-cast-disc.c
index 7d76649e2e49c..40bba99478192 100644
--- a/clang/test/CodeGen/ptrauth-function-lvalue-cast-disc.c
+++ b/clang/test/CodeGen/ptrauth-function-lvalue-cast-disc.c
@@ -1,12 +1,14 @@
 // RUN: %clang_cc1 %s -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -emit-llvm -o- -fptrauth-function-pointer-type-discrimination | FileCheck -check-prefixes CHECK,TYPE %s
+// RUN: %clang_cc1 %s -triple aarch64-linux-gnu  -fptrauth-calls -fptrauth-intrinsics -emit-llvm -o- -fptrauth-function-pointer-type-discrimination | FileCheck -check-prefixes CHECK,TYPE %s
 // RUN: %clang_cc1 %s -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -emit-llvm -o- | FileCheck -check-prefixes CHECK,ZERO %s
+// RUN: %clang_cc1 %s -triple aarch64-linux-gnu  -fptrauth-calls -fptrauth-intrinsics -emit-llvm -o- | FileCheck -check-prefixes CHECK,ZERO %s
 
 typedef void (*fptr_t)(void);
 
 char *cptr;
 void (*fptr)(void);
 
-// CHECK-LABEL: define void @test1
+// CHECK-LABEL: define{{.*}} void @test1
 void test1() {
   // TYPE: [[LOAD:%.*]] = load ptr, ptr @cptr
   // TYPE: [[TOINT:%.*]] = ptrtoint ptr [[LOAD]] to i64
@@ -17,7 +19,7 @@ void test1() {
   (*(fptr_t)cptr)();
 }
 
-// CHECK-LABEL: define i8 @test2
+// CHECK-LABEL: define{{.*}} i8 @test2
 char test2() {
   return *(char *)fptr;
 
@@ -35,7 +37,7 @@ char test2() {
   // ZERO-NOT: @llvm.ptrauth.resign
 }
 
-// CHECK-LABEL: define void @test4
+// CHECK-LABEL: define{{.*}} void @test4
 void test4() {
   (*((fptr_t)(&*((char *)(&*(fptr_t)cptr)))))();
 
@@ -49,7 +51,7 @@ void test4() {
 }
 
 void *vptr;
-// CHECK-LABEL: define void @test5
+// CHECK-LABEL: define{{.*}} void @test5
 void test5() {
   vptr = &*(char *)fptr;
 
diff --git a/clang/test/CodeGen/ptrauth-function-type-discriminator-cast.c b/clang/test/CodeGen/ptrauth-function-type-discriminator-cast.c
index cdf9ee4907525..1a1dce6f4a66e 100644
--- a/clang/test/CodeGen/ptrauth-function-type-discriminator-cast.c
+++ b/clang/test/CodeGen/ptrauth-function-type-discriminator-cast.c
@@ -1,6 +1,20 @@
-// RUN: %clang_cc1 %s -fptrauth-function-pointer-type-discrimination -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck %s --check-prefixes=CHECK,TYPE
-// RUN: %clang_cc1 %s -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck %s --check-prefixes=CHECK,ZERO
-// RUN: %clang_cc1 -xc++ %s -fptrauth-function-pointer-type-discrimination -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck %s --check-prefixes=CHECK,CHECKCXX,TYPE,TYPECXX
+// RUN: %clang_cc1 %s -fptrauth-function-pointer-type-discrimination -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -disable-llvm-passes -emit-llvm -o-       | FileCheck %s --check-prefixes=CHECK,TYPE
+
+// RUN: %clang_cc1 %s -fptrauth-function-pointer-type-discrimination -triple aarch64-linux-gnu  -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -disable-llvm-passes -emit-llvm -o-       | FileCheck %s --check-prefixes=CHECK,TYPE
+
+// RUN: %clang_cc1 %s                                                -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -disable-llvm-passes -emit-llvm -o-       | FileCheck %s --check-prefixes=CHECK,ZERO
+
+// RUN: %clang_cc1 %s                                                -triple aarch64-linux-gnu  -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -disable-llvm-passes -emit-llvm -o-       | FileCheck %s --check-prefixes=CHECK,ZERO
+
+// RUN: %clang_cc1 %s -fptrauth-function-pointer-type-discrimination -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -disable-llvm-passes -emit-llvm -xc++ -o- | FileCheck %s --check-prefixes=CHECK,CHECKCXX,TYPE,TYPECXX
+
+// RUN: %clang_cc1 %s -fptrauth-function-pointer-type-discrimination -triple aarch64-linux-gnu  -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -disable-llvm-passes -emit-llvm -xc++ -o- | FileCheck %s --check-prefixes=CHECK,CHECKCXX,TYPE,TYPECXX
 
 #ifdef __cplusplus
 extern "C" {
@@ -19,12 +33,12 @@ struct ptr_member {
 ptr_member pm;
 void (*test_member)() = (void (*)())pm.fptr_;
 
-// CHECKCXX-LABEL: define internal void @__cxx_global_var_init
+// CHECKCXX-LABEL: define{{.*}} internal void @__cxx_global_var_init
 // TYPECXX: call i64 @llvm.ptrauth.resign(i64 {{.*}}, i32 0, i64 2712, i32 0, i64 18983)
 #endif
 
 
-// CHECK-LABEL: define void @test_cast_to_opaque
+// CHECK-LABEL: define{{.*}} void @test_cast_to_opaque
 void test_cast_to_opaque() {
   opaque = (void *)f;
 
@@ -33,7 +47,7 @@ void test_cast_to_opaque() {
   // ZERO-NOT: @llvm.ptrauth.resign
 }
 
-// CHECK-LABEL: define void @test_cast_from_opaque
+// CHECK-LABEL: define{{.*}} void @test_cast_from_opaque
 void test_cast_from_opaque() {
   fptr = (void (*)(void))opaque;
 
@@ -48,7 +62,7 @@ void test_cast_from_opaque() {
   // ZERO-NOT: @llvm.ptrauth.resign
 }
 
-// CHECK-LABEL: define void @test_cast_to_intptr
+// CHECK-LABEL: define{{.*}} void @test_cast_to_intptr
 void test_cast_to_intptr() {
   uintptr = (unsigned long)fptr;
 
@@ -69,14 +83,14 @@ void test_cast_to_intptr() {
   // ZERO-NOT: @llvm.ptrauth.resign
 }
 
-// CHECK-LABEL: define void @test_function_to_function_cast
+// CHECK-LABEL: define{{.*}} void @test_function_to_function_cast
 void test_function_to_function_cast() {
   void (*fptr2)(int) = (void (*)(int))fptr;
   // TYPE: call i64 @llvm.ptrauth.resign(i64 {{.*}}, i32 0, i64 18983, i32 0, i64 2712)
   // ZERO-NOT: @llvm.ptrauth.resign
 }
 
-// CHECK-LABEL: define void @test_call_lvalue_cast
+// CHECK-LABEL: define{{.*}} void @test_call_lvalue_cast
 void test_call_lvalue_cast() {
   (*(void (*)(int))f)(42);
 
diff --git a/clang/test/CodeGen/ptrauth-function-type-discriminator.c b/clang/test/CodeGen/ptrauth-function-type-discriminator.c
index 5dea48fe5915b..58717015adb6c 100644
--- a/clang/test/CodeGen/ptrauth-function-type-discriminator.c
+++ b/clang/test/CodeGen/ptrauth-function-type-discriminator.c
@@ -1,7 +1,18 @@
-// RUN: %clang_cc1 %s       -fptrauth-function-pointer-type-discrimination -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKC
-// RUN: %clang_cc1 -xc++ %s -fptrauth-function-pointer-type-discrimination -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck %s --check-prefix=CHECK
-// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple arm64-apple-ios -fptrauth-calls -fptrauth-intrinsics -emit-pch %s -o %t.ast
-// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple arm64-apple-ios -fptrauth-calls -fptrauth-intrinsics -emit-llvm -x ast -o - %t.ast | FileCheck -check-prefix=CHECK --check-prefix=CHECKC %s
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -disable-llvm-passes -emit-llvm %s       -o- | FileCheck --check-prefixes=CHECK,CHECKC %s
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple arm64e-apple-ios13 -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -disable-llvm-passes -emit-llvm -xc++ %s -o- | FileCheck --check-prefix=CHECK %s
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple arm64-apple-ios    -fptrauth-calls -fptrauth-intrinsics -emit-pch %s -o %t.ast
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple arm64-apple-ios    -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -emit-llvm -x ast -o - %t.ast | FileCheck --check-prefixes=CHECK,CHECKC %s
+
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple aarch64-linux-gnu  -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -disable-llvm-passes -emit-llvm %s       -o- | FileCheck --check-prefixes=CHECK,CHECKC %s
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple aarch64-linux-gnu  -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -disable-llvm-passes -emit-llvm -xc++ %s -o- | FileCheck --check-prefix=CHECK %s
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple aarch64-linux-gnu  -fptrauth-calls -fptrauth-intrinsics -emit-pch %s -o %t.ast
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple aarch64-linux-gnu  -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -emit-llvm -x ast -o - %t.ast | FileCheck --check-prefixes=CHECK,CHECKC %s
 
 #ifdef __cplusplus
 extern "C" {
@@ -47,14 +58,14 @@ void (*fptr3)(void) = __builtin_ptrauth_sign_constant(&external_function, 2, 26)
 // CHECK: @fptr4 = global ptr ptrauth (ptr @external_function, i32 2, i64 26, ptr @fptr4)
 void (*fptr4)(void) = __builtin_ptrauth_sign_constant(&external_function, 2, __builtin_ptrauth_blend_discriminator(&fptr4, 26));
 
-// CHECK-LABEL: define void @test_call()
+// CHECK-LABEL: define{{.*}} void @test_call()
 void test_call() {
   // CHECK:      [[T0:%.*]] = load ptr, ptr @fnptr,
   // CHECK-NEXT: call void [[T0]]() [ "ptrauth"(i32 0, i64 18983) ]
   fnptr();
 }
 
-// CHECK-LABEL: define ptr @test_function_pointer()
+// CHECK-LABEL: define{{.*}} ptr @test_function_pointer()
 // CHECK:  ret ptr ptrauth (ptr @external_function, i32 0, i64 18983)
 void (*test_function_pointer())(void) {
   return external_function;
@@ -62,14 +73,14 @@ void (*test_function_pointer())(void) {
 
 struct InitiallyIncomplete;
 extern struct InitiallyIncomplete returns_initially_incomplete(void);
-// CHECK-LABEL: define void @use_while_incomplete()
+// CHECK-LABEL: define{{.*}} void @use_while_incomplete()
 void use_while_incomplete() {
   // CHECK:      [[VAR:%.*]] = alloca ptr,
   // CHECK-NEXT: store ptr ptrauth (ptr @returns_initially_incomplete, i32 0, i64 25106), ptr [[VAR]]
   struct InitiallyIncomplete (*fnptr)(void) = &returns_initially_incomplete;
 }
 struct InitiallyIncomplete { int x; };
-// CHECK-LABEL: define void @use_while_complete()
+// CHECK-LABEL: define{{.*}} void @use_while_complete()
 void use_while_complete() {
   // CHECK:      [[VAR:%.*]] = alloca ptr,
   // CHECK-NEXT: store ptr ptrauth (ptr @returns_initially_incomplete, i32 0, i64 25106), ptr [[VAR]]
@@ -83,7 +94,7 @@ void knr(param)
   int param;
 {}
 
-// CHECKC-LABEL: define void @test_knr
+// CHECKC-LABEL: define{{.*}} void @test_knr
 void test_knr() {
   void (*p)() = knr;
   p(0);
@@ -94,7 +105,7 @@ void test_knr() {
   // CHECKC: call void [[LOAD]](i32 noundef 0) [ "ptrauth"(i32 0, i64 18983) ]
 }
 
-// CHECKC-LABEL: define void @test_redeclaration
+// CHECKC-LABEL: define{{.*}} void @test_redeclaration
 void test_redeclaration() {
   void redecl();
   void (*ptr)() = redecl;
@@ -113,7 +124,7 @@ void knr2(param)
      int param;
 {}
 
-// CHECKC-LABEL: define void @test_redecl_knr
+// CHECKC-LABEL: define{{.*}} void @test_redecl_knr
 void test_redecl_knr() {
   void (*p)() = knr2;
   p();
diff --git a/clang/test/CodeGen/ptrauth-ubsan-vptr.cpp b/clang/test/CodeGen/ptrauth-ubsan-vptr.cpp
index 6c36004641477..8045e7bdc7460 100644
--- a/clang/test/CodeGen/ptrauth-ubsan-vptr.cpp
+++ b/clang/test/CodeGen/ptrauth-ubsan-vptr.cpp
@@ -1,6 +1,9 @@
 // RUN: %clang_cc1 -triple arm64e-apple-ios15 -fsanitize=vptr -O0 -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple arm64e-apple-ios15 -fsanitize=vptr -O2 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s
 
+// RUN: %clang_cc1 -triple aarch64-linux-gnu  -fsanitize=vptr -O0 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu  -fsanitize=vptr -O2 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s
+
 struct S {
   S() {}
   ~S() {}
diff --git a/clang/test/CodeGenCXX/ptrauth-explicit-vtable-pointer-control.cpp b/clang/test/CodeGenCXX/ptrauth-explicit-vtable-pointer-control.cpp
index e6497b3f152aa..1b103719fbe46 100644
--- a/clang/test/CodeGenCXX/ptrauth-explicit-vtable-pointer-control.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-explicit-vtable-pointer-control.cpp
@@ -1,15 +1,31 @@
 // RUN: %clang_cc1 %s -x c++ -std=c++11 -triple arm64-apple-ios -fptrauth-calls -fptrauth-intrinsics \
 // RUN:   -emit-llvm -o - | FileCheck --check-prefixes=CHECK,NODISC %s
 
-// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple arm64-apple-ios -fptrauth-calls -fptrauth-intrinsics \
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple arm64-apple-ios   -fptrauth-calls -fptrauth-intrinsics \
 // RUN:   -fptrauth-vtable-pointer-type-discrimination \
 // RUN:   -emit-llvm -o - | FileCheck --check-prefixes=CHECK,TYPE %s
 
-// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple arm64-apple-ios -fptrauth-calls -fptrauth-intrinsics \
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple arm64-apple-ios   -fptrauth-calls -fptrauth-intrinsics \
 // RUN:   -fptrauth-vtable-pointer-address-discrimination \
 // RUN:   -emit-llvm -o - | FileCheck --check-prefixes=CHECK,ADDR %s
 
-// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple arm64-apple-ios -fptrauth-calls -fptrauth-intrinsics \
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple arm64-apple-ios   -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fptrauth-vtable-pointer-address-discrimination \
+// RUN:   -emit-llvm -o - | FileCheck --check-prefixes=CHECK,BOTH %s
+
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -emit-llvm -o - | FileCheck --check-prefixes=CHECK,NODISC %s
+
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination \
+// RUN:   -emit-llvm -o - | FileCheck --check-prefixes=CHECK,TYPE %s
+
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics \
+// RUN:   -fptrauth-vtable-pointer-address-discrimination \
+// RUN:   -emit-llvm -o - | FileCheck --check-prefixes=CHECK,ADDR %s
+
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics \
 // RUN:   -fptrauth-vtable-pointer-type-discrimination \
 // RUN:   -fptrauth-vtable-pointer-address-discrimination \
 // RUN:   -emit-llvm -o - | FileCheck --check-prefixes=CHECK,BOTH %s
@@ -111,7 +127,7 @@ int TVDisc_ExplicitNoExtraDiscrimination = ptrauth_string_discriminator("_ZTVN5t
 int TVDisc_ExplicitTypeDiscrimination = ptrauth_string_discriminator("_ZTVN5test126ExplicitTypeDiscriminationE");
 
 
-// CHECK-LABEL: define void @test_default(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_default(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = load ptr, ptr {{%.*}}, align 8
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -133,7 +149,7 @@ void test_default(NoExplicitAuth *a) {
   a->f();
 }
 
-// CHECK-LABEL: define void @test_disabled(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_disabled(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = load ptr, ptr {{%.*}}, align 8
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 // CHECK-NOT:     call i64 @llvm.ptrauth.auth
@@ -141,7 +157,7 @@ void test_disabled(ExplicitlyDisableAuth *a) {
   a->f();
 }
 
-// CHECK-LABEL: define void @test_addr_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_addr_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = load ptr, ptr {{%.*}}, align 8
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -166,7 +182,7 @@ void test_addr_disc(ExplicitAddressDiscrimination *a) {
   a->f();
 }
 
-// CHECK-LABEL: define void @test_no_addr_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_no_addr_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = load ptr, ptr {{%.*}}, align 8
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -185,7 +201,7 @@ void test_no_addr_disc(ExplicitNoAddressDiscrimination *a) {
   a->f();
 }
 
-// CHECK-LABEL: define void @test_no_extra_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_no_extra_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = load ptr, ptr {{%.*}}, align 8
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -206,7 +222,7 @@ void test_no_extra_disc(ExplicitNoExtraDiscrimination *a) {
   a->f();
 }
 
-// CHECK-LABEL: define void @test_type_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_type_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = load ptr, ptr {{%.*}}, align 8
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -229,7 +245,7 @@ void test_type_disc(ExplicitTypeDiscrimination *a) {
   a->f();
 }
 
-// CHECK-LABEL: define void @test_custom_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_custom_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = load ptr, ptr {{%.*}}, align 8
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -257,7 +273,7 @@ void test_custom_disc(ExplicitCustomDiscrimination *a) {
 // Codegen should be the same as the simple cases above once we have a vtable.
 //
 
-// CHECK-LABEL: define void @test_subclass_default(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_subclass_default(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = call noundef ptr @_ZN5test113make_subclass
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -279,7 +295,7 @@ void test_subclass_default(NoExplicitAuth *a) {
   make_subclass(a)->f();
 }
 
-// CHECK-LABEL: define void @test_subclass_disabled(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_subclass_disabled(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = call noundef ptr @_ZN5test113make_subclass
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 // CHECK-NOT:     call i64 @llvm.ptrauth.auth
@@ -287,7 +303,7 @@ void test_subclass_disabled(ExplicitlyDisableAuth *a) {
   make_subclass(a)->f();
 }
 
-// CHECK-LABEL: define void @test_subclass_addr_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_subclass_addr_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = call noundef ptr @_ZN5test113make_subclass
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -312,7 +328,7 @@ void test_subclass_addr_disc(ExplicitAddressDiscrimination *a) {
   make_subclass(a)->f();
 }
 
-// CHECK-LABEL: define void @test_subclass_no_addr_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_subclass_no_addr_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = call noundef ptr @_ZN5test113make_subclass
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -331,7 +347,7 @@ void test_subclass_no_addr_disc(ExplicitNoAddressDiscrimination *a) {
   make_subclass(a)->f();
 }
 
-// CHECK-LABEL: define void @test_subclass_no_extra_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_subclass_no_extra_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = call noundef ptr @_ZN5test113make_subclass
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -352,7 +368,7 @@ void test_subclass_no_extra_disc(ExplicitNoExtraDiscrimination *a) {
   make_subclass(a)->f();
 }
 
-// CHECK-LABEL: define void @test_subclass_type_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_subclass_type_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = call noundef ptr @_ZN5test113make_subclass
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -375,7 +391,7 @@ void test_subclass_type_disc(ExplicitTypeDiscrimination *a) {
   make_subclass(a)->f();
 }
 
-// CHECK-LABEL: define void @test_subclass_custom_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_subclass_custom_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTADDR:%.*]] = call noundef ptr @_ZN5test113make_subclass
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
 //
@@ -404,7 +420,7 @@ void test_subclass_custom_disc(ExplicitCustomDiscrimination *a) {
 // Codegen should be the same as the simple cases above once we have a vtable.
 //
 
-// CHECK-LABEL: define void @test_multiple_default(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_multiple_default(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[CALL:%.*]] = call noundef ptr @_ZN5test121make_multiple_primary
 // CHECK:         [[VTADDR:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 8
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
@@ -427,7 +443,7 @@ void test_multiple_default(NoExplicitAuth *a) {
   make_multiple_primary(a)->f();
 }
 
-// CHECK-LABEL: define void @test_multiple_disabled(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_multiple_disabled(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[CALL:%.*]] = call noundef ptr @_ZN5test121make_multiple_primary
 // CHECK:         [[VTADDR:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 8
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
@@ -436,7 +452,7 @@ void test_multiple_disabled(ExplicitlyDisableAuth *a) {
   make_multiple_primary(a)->f();
 }
 
-// CHECK-LABEL: define void @test_multiple_custom_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_multiple_custom_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[CALL:%.*]] = call noundef ptr @_ZN5test121make_multiple_primary
 // CHECK:         [[VTADDR:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 8
 // CHECK:         [[VTABLE:%.*]] = load ptr, ptr [[VTADDR]], align 8
@@ -466,7 +482,7 @@ void test_multiple_custom_disc(ExplicitCustomDiscrimination *a) {
 // but twice for vtt/vtable.  The names in the vtt version have "VTT" prefixes.
 //
 
-// CHECK-LABEL: define void @test_virtual_default(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_virtual_default(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTTADDR:%.*]] = call noundef ptr @_ZN5test120make_virtual_primary
 // CHECK:         [[VTTABLE:%.*]] = load ptr, ptr [[VTTADDR]], align 8
 //
@@ -509,13 +525,13 @@ void test_virtual_default(NoExplicitAuth *a) {
   make_virtual_primary(a)->f();
 }
 
-// CHECK-LABEL: define void @test_virtual_disabled(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_virtual_disabled(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK-NOT:     call i64 @llvm.ptrauth.auth
 void test_virtual_disabled(ExplicitlyDisableAuth *a) {
   make_virtual_primary(a)->f();
 }
 
-// CHECK-LABEL: define void @test_virtual_custom_disc(ptr noundef {{%.*}}) {{#.*}} {
+// CHECK-LABEL: define{{.*}} void @test_virtual_custom_disc(ptr noundef {{%.*}}) {{#.*}} {
 // CHECK:         [[VTTADDR:%.*]] = call noundef ptr @_ZN5test120make_virtual_primary
 // CHECK:         [[VTTABLE:%.*]] = load ptr, ptr [[VTTADDR]], align 8
 //
diff --git a/clang/test/CodeGenCXX/ptrauth-rtti-layout.cpp b/clang/test/CodeGenCXX/ptrauth-rtti-layout.cpp
index b4a8784a33d8c..2b633addd677e 100644
--- a/clang/test/CodeGenCXX/ptrauth-rtti-layout.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-rtti-layout.cpp
@@ -1,10 +1,16 @@
-// RUN: %clang_cc1 %s -I%S -triple=arm64-apple-ios -fptrauth-calls -std=c++11 -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 %s -I%S -triple=arm64-apple-ios   -fptrauth-calls -std=c++11 -emit-llvm -o - | FileCheck --check-prefix=DARWIN %s
+// RUN: %clang_cc1 %s -I%S -triple=aarch64-linux-gnu -fptrauth-calls -std=c++11 -emit-llvm -o - | FileCheck --check-prefix=ELF %s
+
 #include <typeinfo>
 
 struct A { int a; };
 
-// CHECK: @_ZTVN10__cxxabiv117__class_type_infoE = external global [0 x ptr]
-// CHECK: @_ZTS1A = linkonce_odr hidden constant [3 x i8] c"1A\00"
-// CHECK: @_ZTI1A = linkonce_odr hidden constant { ptr, ptr } { ptr ptrauth (ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv117__class_type_infoE, i64 2), i32 2), ptr inttoptr (i64 add (i64 ptrtoint (ptr @_ZTS1A to i64), i64 -9223372036854775808) to ptr) }
+// DARWIN: @_ZTVN10__cxxabiv117__class_type_infoE = external global [0 x ptr]
+// DARWIN: @_ZTS1A = linkonce_odr hidden constant [3 x i8] c"1A\00"
+// DARWIN: @_ZTI1A = linkonce_odr hidden constant { ptr, ptr } { ptr ptrauth (ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv117__class_type_infoE, i64 2), i32 2), ptr inttoptr (i64 add (i64 ptrtoint (ptr @_ZTS1A to i64), i64 -9223372036854775808) to ptr) }
+
+// ELF: @_ZTVN10__cxxabiv117__class_type_infoE = external global [0 x ptr]
+// ELF: @_ZTS1A = linkonce_odr constant [3 x i8] c"1A\00"
+// ELF: @_ZTI1A = linkonce_odr constant { ptr, ptr } { ptr ptrauth (ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv117__class_type_infoE, i64 2), i32 2), ptr @_ZTS1A }
 
 auto ATI = typeid(A);
diff --git a/clang/test/CodeGenCXX/ptrauth-static-destructors.cpp b/clang/test/CodeGenCXX/ptrauth-static-destructors.cpp
index cad43dc0746df..1240f26d329da 100644
--- a/clang/test/CodeGenCXX/ptrauth-static-destructors.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-static-destructors.cpp
@@ -2,8 +2,13 @@
 // RUN:  | FileCheck %s --check-prefix=CXAATEXIT
 
 // RUN: %clang_cc1 -triple arm64-apple-ios -fptrauth-calls -emit-llvm -std=c++11 %s -o - \
-// RUN:    -fno-use-cxa-atexit \
-// RUN:  | FileCheck %s --check-prefix=ATEXIT
+// RUN:    -fno-use-cxa-atexit | FileCheck %s --check-prefixes=ATEXIT,DARWIN
+
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fptrauth-calls -emit-llvm -std=c++11 %s -o - \
+// RUN:  | FileCheck %s --check-prefix=CXAATEXIT
+
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fptrauth-calls -emit-llvm -std=c++11 %s -o - \
+// RUN:    -fno-use-cxa-atexit | FileCheck %s --check-prefixes=ATEXIT,ELF
 
 class Foo {
  public:
@@ -20,5 +25,7 @@ Foo global;
 // ATEXIT: define internal void @__cxx_global_var_init()
 // ATEXIT:   %{{.*}} = call i32 @atexit(ptr ptrauth (ptr @__dtor_global, i32 0))
 
-// ATEXIT: define internal void @__dtor_global() {{.*}} section "__TEXT,__StaticInit,regular,pure_instructions" {
-// ATEXIT:   %{{.*}} = call ptr @_ZN3FooD1Ev(ptr @global)
+// DARWIN: define internal void @__dtor_global() {{.*}} section "__TEXT,__StaticInit,regular,pure_instructions" {
+// ELF:    define internal void @__dtor_global() {{.*}} section ".text.startup" {
+// DARWIN:   %{{.*}} = call ptr @_ZN3FooD1Ev(ptr @global)
+// ELF:      call void @_ZN3FooD1Ev(ptr @global)
diff --git a/clang/test/CodeGenCXX/ptrauth-throw.cpp b/clang/test/CodeGenCXX/ptrauth-throw.cpp
index cea7226547e5a..0e6091a370223 100644
--- a/clang/test/CodeGenCXX/ptrauth-throw.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-throw.cpp
@@ -1,5 +1,8 @@
-// RUN: %clang_cc1                                                -triple arm64-apple-ios -fptrauth-calls -fcxx-exceptions -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK
-// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple arm64-apple-ios -fptrauth-calls -fcxx-exceptions -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECKDISC
+// RUN: %clang_cc1                                                -triple arm64-apple-ios   -fptrauth-calls -fcxx-exceptions -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple arm64-apple-ios   -fptrauth-calls -fcxx-exceptions -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECKDISC
+
+// RUN: %clang_cc1                                                -triple aarch64-linux-gnu -fptrauth-calls -fcxx-exceptions -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple aarch64-linux-gnu -fptrauth-calls -fcxx-exceptions -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECKDISC
 
 class Foo {
  public:
@@ -7,10 +10,10 @@ class Foo {
   }
 };
 
-// CHECK-LABEL: define void @_Z1fv()
+// CHECK-LABEL: define{{.*}} void @_Z1fv()
 // CHECK:  call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTI3Foo, ptr ptrauth (ptr @_ZN3FooD1Ev, i32 0))
 
-// CHECKDISC-LABEL: define void @_Z1fv()
+// CHECKDISC-LABEL: define{{.*}} void @_Z1fv()
 // CHECKDISC:  call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTI3Foo, ptr ptrauth (ptr @_ZN3FooD1Ev, i32 0, i64 10942))
 
 void f() {
@@ -18,10 +21,10 @@ void f() {
 }
 
 // __cxa_throw is defined to take its destructor as "void (*)(void *)" in the ABI.
-// CHECK-LABEL: define void @__cxa_throw({{.*}})
+// CHECK-LABEL: define{{.*}} void @__cxa_throw({{.*}})
 // CHECK: call void {{%.*}}(ptr noundef {{%.*}}) [ "ptrauth"(i32 0, i64 0) ]
 
-// CHECKDISC-LABEL: define void @__cxa_throw({{.*}})
+// CHECKDISC-LABEL: define{{.*}} void @__cxa_throw({{.*}})
 // CHECKDISC: call void {{%.*}}(ptr noundef {{%.*}}) [ "ptrauth"(i32 0, i64 10942) ]
 
 extern "C" void __cxa_throw(void *exception, void *, void (*dtor)(void *)) {
diff --git a/clang/test/CodeGenCXX/ptrauth-thunks.cpp b/clang/test/CodeGenCXX/ptrauth-thunks.cpp
index a85c8c4c065c4..f4491a3aab7ab 100644
--- a/clang/test/CodeGenCXX/ptrauth-thunks.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-thunks.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple arm64-apple-ios -fptrauth-calls -emit-llvm -std=c++11 %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple arm64-apple-ios   -fptrauth-calls -emit-llvm -std=c++11 %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fptrauth-calls -emit-llvm -std=c++11 %s -o - | FileCheck %s
 
 namespace Test1 {
   struct B1 {
diff --git a/clang/test/CodeGenCXX/ptrauth-virtual-function.cpp b/clang/test/CodeGenCXX/ptrauth-virtual-function.cpp
index 563d60780769b..4aa24738d8ce3 100644
--- a/clang/test/CodeGenCXX/ptrauth-virtual-function.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-virtual-function.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple arm64-apple-ios -fptrauth-calls -emit-llvm -std=c++11 %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple arm64-apple-ios   -fptrauth-calls -emit-llvm -std=c++11 %s -o - | FileCheck --check-prefixes=CHECK,DARWIN %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fptrauth-calls -emit-llvm -std=c++11 %s -o - | FileCheck --check-prefixes=CHECK,ELF    %s
 
 // Check virtual function pointers in vtables are signed.
 
@@ -182,7 +183,8 @@ V1::~V1() {
 // Check sign/authentication of vtable pointers and authentication of virtual
 // functions.
 
-// CHECK-LABEL: define noundef ptr @_ZN2V1D2Ev(
+// DARWIN-LABEL: define noundef ptr @_ZN2V1D2Ev(
+// ELF-LABEL:    define dso_local void @_ZN2V1D2Ev(
 // CHECK: %[[THIS1:.*]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T1:[0-9]+]] = ptrtoint ptr %[[T0]] to i64
@@ -193,7 +195,7 @@ V1::~V1() {
 // CHECK: %[[SIGNED_VTADDR:[0-9]+]] = inttoptr i64 %[[T7]] to ptr
 // CHECK: store ptr %[[SIGNED_VTADDR]], ptr %[[THIS1]]
 
-// CHECK-LABEL: define void @_Z8testB0m0P2B0(
+// CHECK-LABEL: define{{.*}} void @_Z8testB0m0P2B0(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -208,7 +210,7 @@ void testB0m0(B0 *a) {
   a->m0();
 }
 
-// CHECK-LABEL: define void @_Z8testB0m1P2B0(
+// CHECK-LABEL: define{{.*}} void @_Z8testB0m1P2B0(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -223,7 +225,7 @@ void testB0m1(B0 *a) {
   a->m1();
 }
 
-// CHECK-LABEL: define void @_Z8testB0m2P2B0(
+// CHECK-LABEL: define{{.*}} void @_Z8testB0m2P2B0(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -238,7 +240,7 @@ void testB0m2(B0 *a) {
   a->m2();
 }
 
-// CHECK-LABEL: define void @_Z8testD0m0P2D0(
+// CHECK-LABEL: define{{.*}} void @_Z8testD0m0P2D0(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -253,7 +255,7 @@ void testD0m0(D0 *a) {
   a->m0();
 }
 
-// CHECK-LABEL: define void @_Z8testD0m1P2D0(
+// CHECK-LABEL: define{{.*}} void @_Z8testD0m1P2D0(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -268,7 +270,7 @@ void testD0m1(D0 *a) {
   a->m1();
 }
 
-// CHECK-LABEL: define void @_Z8testD0m2P2D0(
+// CHECK-LABEL: define{{.*}} void @_Z8testD0m2P2D0(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -283,7 +285,7 @@ void testD0m2(D0 *a) {
   a->m2();
 }
 
-// CHECK-LABEL: define void @_Z8testD0m3P2D0(
+// CHECK-LABEL: define{{.*}} void @_Z8testD0m3P2D0(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -299,7 +301,7 @@ void testD0m3(D0 *a) {
 }
 
 
-// CHECK-LABEL: define void @_Z8testD1m0P2D1(
+// CHECK-LABEL: define{{.*}} void @_Z8testD1m0P2D1(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -314,7 +316,7 @@ void testD1m0(D1 *a) {
   a->m0();
 }
 
-// CHECK-LABEL: define void @_Z8testD1m1P2D1(
+// CHECK-LABEL: define{{.*}} void @_Z8testD1m1P2D1(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -329,7 +331,7 @@ void testD1m1(D1 *a) {
   a->m1();
 }
 
-// CHECK-LABEL: define void @_Z8testD1m2P2D1(
+// CHECK-LABEL: define{{.*}} void @_Z8testD1m2P2D1(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -345,7 +347,7 @@ void testD1m2(D1 *a) {
 }
 
 
-// CHECK-LABEL: define void @_Z8testD2m0P2D2(
+// CHECK-LABEL: define{{.*}} void @_Z8testD2m0P2D2(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -360,7 +362,7 @@ void testD2m0(D2 *a) {
   a->m0();
 }
 
-// CHECK-LABEL: define void @_Z8testD2m1P2D2(
+// CHECK-LABEL: define{{.*}} void @_Z8testD2m1P2D2(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -375,21 +377,21 @@ void testD2m1(D2 *a) {
   a->m1();
 }
 
-// CHECK-LABEL: define void @_Z10testD2m2D0P2D2(
+// CHECK-LABEL: define{{.*}} void @_Z10testD2m2D0P2D2(
 // CHECK: call void @_ZN2B02m2Ev(ptr noundef nonnull align {{[0-9]+}} dereferenceable(12) %{{.*}}){{$}}
 
 void testD2m2D0(D2 *a) {
   a->D0::m2();
 }
 
-// CHECK-LABEL: define void @_Z10testD2m2D1P2D2(
+// CHECK-LABEL: define{{.*}} void @_Z10testD2m2D1P2D2(
 // CHECK: call void @_ZN2B02m2Ev(ptr noundef nonnull align {{[0-9]+}} dereferenceable(12) %{{.*}}){{$}}
 
 void testD2m2D1(D2 *a) {
   a->D1::m2();
 }
 
-// CHECK-LABEL: define void @_Z8testD2m3P2D2(
+// CHECK-LABEL: define{{.*}} void @_Z8testD2m3P2D2(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -404,7 +406,7 @@ void testD2m3(D2 *a) {
   a->m3();
 }
 
-// CHECK-LABEL: define void @_Z8testD3m0P2D3(
+// CHECK-LABEL: define{{.*}} void @_Z8testD3m0P2D3(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -419,7 +421,7 @@ void testD3m0(D3 *a) {
   a->m0();
 }
 
-// CHECK-LABEL: define void @_Z8testD3m1P2D3(
+// CHECK-LABEL: define{{.*}} void @_Z8testD3m1P2D3(
 // CHECK: %[[VTABLE:[a-z]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
 // CHECK: %[[T3:[0-9]+]] = call i64 @llvm.ptrauth.auth(i64 %[[T0]], i32 2, i64 0)
@@ -434,7 +436,7 @@ void testD3m1(D3 *a) {
   a->m1();
 }
 
-// CHECK: define void @_Z8testD3m2P2D3(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z8testD3m2P2D3(ptr noundef %[[A:.*]])
 // CHECK: %[[A_ADDR:.*]] = alloca ptr, align 8
 // CHECK: store ptr %[[A]], ptr %[[A_ADDR]], align 8
 // CHECK: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]], align 8
@@ -459,7 +461,7 @@ void testD3m2(D3 *a) {
   a->m2();
 }
 
-// CHECK-LABEL: define void @_Z17testD3Destructor0P2D3(
+// CHECK-LABEL: define{{.*}} void @_Z17testD3Destructor0P2D3(
 // CHECK: load ptr, ptr
 // CHECK: %[[VTABLE:.*]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T2:[0-9]+]] = ptrtoint ptr %[[VTABLE]] to i64
@@ -475,7 +477,7 @@ void testD3Destructor0(D3 *a) {
   delete a;
 }
 
-// CHECK-LABEL: define void @_Z17testD3Destructor1P2D3(
+// CHECK-LABEL: define{{.*}} void @_Z17testD3Destructor1P2D3(
 // CHECK: %[[T6:.*]] = load ptr, ptr %
 // CHECK: %[[VTABLE0:[a-z0-9]+]] = load ptr, ptr %
 // CHECK: %[[T2:[0-9]+]] = ptrtoint ptr %[[VTABLE0]] to i64
@@ -492,14 +494,15 @@ void testD3Destructor0(D3 *a) {
 // CHECK: %[[T12:[0-9]+]] = load ptr, ptr %[[VFN]]
 // CHECK: %[[T13:[0-9]+]] = ptrtoint ptr %[[VFN]] to i64
 // CHECK: %[[T14:[0-9]+]] = call i64 @llvm.ptrauth.blend(i64 %[[T13]], i64 57279)
-// CHECK: %call = call noundef ptr %[[T12]](ptr noundef nonnull align {{[0-9]+}} dereferenceable(32) %{{.*}}) #{{.*}} [ "ptrauth"(i32 0, i64 %[[T14]]) ]
+// DARWIN: %call = call noundef ptr %[[T12]](ptr noundef nonnull align {{[0-9]+}} dereferenceable(32) %{{.*}}) #{{.*}} [ "ptrauth"(i32 0, i64 %[[T14]]) ]
+// ELF:    call void %[[T12]](ptr noundef nonnull align {{[0-9]+}} dereferenceable(32) %{{.*}}) #{{.*}} [ "ptrauth"(i32 0, i64 %[[T14]]) ]
 // CHECK: call void @_ZdlPv(ptr noundef %[[T7]])
 
 void testD3Destructor1(D3 *a) {
   ::delete a;
 }
 
-// CHECK-LABEL: define void @_Z17testD3Destructor2P2D3(
+// CHECK-LABEL: define{{.*}} void @_Z17testD3Destructor2P2D3(
 // CHECK: load ptr, ptr
 // CHECK: %[[VTABLE:.*]] = load ptr, ptr %
 // CHECK: %[[T2:.*]] = ptrtoint ptr %[[VTABLE]] to i64
@@ -509,7 +512,8 @@ void testD3Destructor1(D3 *a) {
 // CHECK: %[[T5:.*]] = load ptr, ptr %[[VFN]]
 // CHECK: %[[T6:.*]] = ptrtoint ptr %[[VFN]] to i64
 // CHECK: %[[T7:.*]] = call i64 @llvm.ptrauth.blend(i64 %[[T6]], i64 57279)
-// CHECK: %call = call noundef ptr %[[T5]](ptr noundef nonnull align {{[0-9]+}} dereferenceable(32) %{{.*}}) #{{.*}} [ "ptrauth"(i32 0, i64 %[[T7]]) ]
+// DARWIN: %call = call noundef ptr %[[T5]](ptr noundef nonnull align {{[0-9]+}} dereferenceable(32) %{{.*}}) #{{.*}} [ "ptrauth"(i32 0, i64 %[[T7]]) ]
+// ELF:    call void %[[T5]](ptr noundef nonnull align {{[0-9]+}} dereferenceable(32) %{{.*}}) #{{.*}} [ "ptrauth"(i32 0, i64 %[[T7]]) ]
 
 void testD3Destructor2(D3 *a) {
   a->~D3();
@@ -526,23 +530,27 @@ void materializeConstructors() {
   V1 V1;
 }
 
-// CHECK-LABEL: define linkonce_odr noundef ptr @_ZN2B0C2Ev(
+// DARWIN-LABEL: define linkonce_odr noundef ptr @_ZN2B0C2Ev(
+// ELF-LABEL:    define linkonce_odr void @_ZN2B0C2Ev(
 // CHECK: %[[THIS:.*]] = load ptr, ptr %
 // CHECK: %[[T0:[0-9]+]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr getelementptr inbounds inrange(-16, 40) ({ [7 x ptr] }, ptr @_ZTV2B0, i32 0, i32 0, i32 2) to i64), i32 2, i64 0)
 // CHECK: %[[SIGNED_VTADDR:[0-9]+]] = inttoptr i64 %[[T0]] to ptr
 // CHECK: store ptr %[[SIGNED_VTADDR]], ptr %[[THIS]]
 
-// CHECK-LABEL: define linkonce_odr noundef ptr @_ZN2D0C2Ev(
+// DARWIN-LABEL: define linkonce_odr noundef ptr @_ZN2D0C2Ev(
+// ELF-LABEL:    define linkonce_odr void @_ZN2D0C2Ev(
 // CHECK: %[[T0:[0-9]+]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr getelementptr inbounds inrange(-16, 56) ({ [9 x ptr] }, ptr @_ZTV2D0, i32 0, i32 0, i32 2) to i64), i32 2, i64 0)
 // CHECK: %[[SIGNED_VTADDR:[0-9]+]] = inttoptr i64 %[[T0]] to ptr
 // CHECK: store ptr %[[SIGNED_VTADDR]], ptr %[[THIS]]
 
-// CHECK-LABEL: define linkonce_odr noundef ptr @_ZN2D1C2Ev(
+// DARWIN-LABEL: define linkonce_odr noundef ptr @_ZN2D1C2Ev(
+// ELF-LABEL:    define linkonce_odr void @_ZN2D1C2Ev(
 // CHECK: %[[T0:[0-9]+]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr getelementptr inbounds inrange(-16, 48) ({ [8 x ptr] }, ptr @_ZTV2D1, i32 0, i32 0, i32 2) to i64), i32 2, i64 0)
 // CHECK: %[[SIGNED_VTADDR:[0-9]+]] = inttoptr i64 %[[T0]] to ptr
 // CHECK: store ptr %[[SIGNED_VTADDR]], ptr %[[THIS]]
 
-// CHECK-LABEL: define linkonce_odr noundef ptr @_ZN2D2C2Ev(
+// DARWIN-LABEL: define linkonce_odr noundef ptr @_ZN2D2C2Ev(
+// ELF-LABEL:    define linkonce_odr void @_ZN2D2C2Ev(
 // CHECK: %[[SLOT0:.*]] = load ptr, ptr
 // CHECK: %[[SIGN_VTADDR0:[0-9]+]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr getelementptr inbounds inrange(-16, 56) ({ [9 x ptr], [8 x ptr] }, ptr @_ZTV2D2, i32 0, i32 0, i32 2) to i64), i32 2, i64 0)
 // CHECK: %[[T1:[0-9]+]] = inttoptr i64 %[[SIGN_VTADDR0]] to ptr
@@ -552,7 +560,8 @@ void materializeConstructors() {
 // CHECK: %[[T5:[0-9]+]] = inttoptr i64 %[[SIGN_VTADDR1]] to ptr
 // CHECK: store ptr %[[T5]], ptr %[[T3]]
 
-// CHECK-LABEL: define linkonce_odr noundef ptr @_ZN2V0C2Ev(
+// DARWIN-LABEL: define linkonce_odr noundef ptr @_ZN2V0C2Ev(
+// ELF-LABEL:    define linkonce_odr void @_ZN2V0C2Ev(
 // CHECK: %[[THIS1]] = load ptr, ptr %
 // CHECK: %[[VTT:[a-z0-9]+]] = load ptr, ptr %{{.*}}
 // CHECK: %[[T0:[0-9]+]] = load ptr, ptr %[[VTT]]
diff --git a/clang/test/CodeGenCXX/ptrauth-vtable-virtual-inheritance-thunk.cpp b/clang/test/CodeGenCXX/ptrauth-vtable-virtual-inheritance-thunk.cpp
index 00b1cbd06e0f0..031bb48608af7 100644
--- a/clang/test/CodeGenCXX/ptrauth-vtable-virtual-inheritance-thunk.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-vtable-virtual-inheritance-thunk.cpp
@@ -1,4 +1,7 @@
-// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple arm64-apple-ios -fptrauth-intrinsics -fptrauth-calls -fptrauth-vtable-pointer-type-discrimination -emit-llvm -O0 -disable-llvm-passes -o - | FileCheck --check-prefix=CHECK %s
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple arm64-apple-ios   -disable-llvm-passes -fptrauth-intrinsics -fptrauth-calls \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination -emit-llvm -O0 -o - | FileCheck --check-prefixes=CHECK,DARWIN %s
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple aarch64-linux-gnu -disable-llvm-passes -fptrauth-intrinsics -fptrauth-calls \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination -emit-llvm -O0 -o - | FileCheck --check-prefixes=CHECK,ELF %s
 
 // The actual vtable construction
 
@@ -103,9 +106,11 @@
 
 // CHECK: @_ZTVN10__cxxabiv120__si_class_type_infoE = external global [0 x ptr]
 
-// CHECK: @_ZTS1B = linkonce_odr hidden constant [3 x i8] c"1B\00", align 1
+// DARWIN: @_ZTS1B = linkonce_odr hidden constant [3 x i8] c"1B\00", align 1
+// ELF:    @_ZTS1B = linkonce_odr constant [3 x i8] c"1B\00", comdat, align 1
 
-// CHECK: @_ZTI1B = linkonce_odr hidden constant { ptr, ptr, ptr } { ptr ptrauth (ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv120__si_class_type_infoE, i64 2), i32 2), ptr inttoptr (i64 add (i64 ptrtoint (ptr @_ZTS1B to i64), i64 -9223372036854775808) to ptr), ptr @_ZTI1A }, align 8
+// DARWIN: @_ZTI1B = linkonce_odr hidden constant { ptr, ptr, ptr } { ptr ptrauth (ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv120__si_class_type_infoE, i64 2), i32 2), ptr inttoptr (i64 add (i64 ptrtoint (ptr @_ZTS1B to i64), i64 -9223372036854775808) to ptr), ptr @_ZTI1A }, align 8
+// ELF:    @_ZTI1B = linkonce_odr constant { ptr, ptr, ptr } { ptr ptrauth (ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv120__si_class_type_infoE, i64 2), i32 2), ptr @_ZTS1B, ptr @_ZTI1A }, comdat, align 8
 
 // CHECK: @_ZTI1C = constant { ptr, ptr, i32, i32, ptr, i64 } { ptr ptrauth (ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2), i32 2), ptr @_ZTS1C, i32 0, i32 1, ptr @_ZTI1B, i64 -6141 }, align 8
 
@@ -177,7 +182,8 @@
 // CHECK-SAME: ptr ptrauth (ptr @_ZN1A1gEv, i32 0, i64 19402, ptr getelementptr inbounds ({ [7 x ptr] }, ptr @_ZTV1B, i32 0, i32 0, i32 3)),
 // CHECK-SAME: ptr ptrauth (ptr @_ZN1A1hEz, i32 0, i64 31735, ptr getelementptr inbounds ({ [7 x ptr] }, ptr @_ZTV1B, i32 0, i32 0, i32 4)),
 // CHECK-SAME: ptr ptrauth (ptr @_ZN1BD1Ev, i32 0, i64 2043, ptr getelementptr inbounds ({ [7 x ptr] }, ptr @_ZTV1B, i32 0, i32 0, i32 5)),
-// CHECK-SAME: ptr ptrauth (ptr @_ZN1BD0Ev, i32 0, i64 63674, ptr getelementptr inbounds ({ [7 x ptr] }, ptr @_ZTV1B, i32 0, i32 0, i32 6))] }, align 8
+// DARWIN-SAME: ptr ptrauth (ptr @_ZN1BD0Ev, i32 0, i64 63674, ptr getelementptr inbounds ({ [7 x ptr] }, ptr @_ZTV1B, i32 0, i32 0, i32 6))] }, align 8
+// ELF-SAME:    ptr ptrauth (ptr @_ZN1BD0Ev, i32 0, i64 63674, ptr getelementptr inbounds ({ [7 x ptr] }, ptr @_ZTV1B, i32 0, i32 0, i32 6))] }, comdat, align 8
 
 extern "C" int printf(const char *format, ...);
 
@@ -284,13 +290,15 @@ int main() {
 }
 
 // And check the thunks
-// CHECK: ptr @_ZTv0_n48_N1CD1Ev(ptr noundef %this)
+// DARWIN: ptr @_ZTv0_n48_N1CD1Ev(ptr noundef %this)
+// ELF:    void @_ZTv0_n48_N1CD1Ev(ptr noundef %this)
 // CHECK: [[TEMP:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[TEMP:%.*]], i32 2, i64 62866)
 
 // CHECK: void @_ZTv0_n48_N1CD0Ev(ptr noundef %this)
 // CHECK: [[TEMP:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[TEMP:%.*]], i32 2, i64 62866)
 
-// CHECK: ptr @_ZTv0_n48_N1DD1Ev(ptr noundef %this)
+// DARWIN: ptr @_ZTv0_n48_N1DD1Ev(ptr noundef %this)
+// ELF:    void @_ZTv0_n48_N1DD1Ev(ptr noundef %this)
 // CHECK: [[TEMP:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[TEMP:%.*]], i32 2, i64 62866)
 
 // CHECK: void @_ZTv0_n48_N1DD0Ev(ptr noundef %this)
diff --git a/clang/test/CodeGenCXX/ubsan-vtable-checks.cpp b/clang/test/CodeGenCXX/ubsan-vtable-checks.cpp
index 72c59cb41c34b..c891ff0a4fa42 100644
--- a/clang/test/CodeGenCXX/ubsan-vtable-checks.cpp
+++ b/clang/test/CodeGenCXX/ubsan-vtable-checks.cpp
@@ -3,6 +3,8 @@
 // RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux -emit-llvm -fsanitize=null,vptr %s -o - | FileCheck %s --check-prefix=CHECK-VPTR --check-prefix=ITANIUM
 // RUN: %clang_cc1 -std=c++11 -triple x86_64-windows -emit-llvm -fsanitize=null,vptr %s -o - | FileCheck %s --check-prefix=CHECK-VPTR --check-prefix=MSABI  --check-prefix=CHECK-VPTR-MS
 // RUN: %clang_cc1 -std=c++11 -triple arm64e-ios-13 -emit-llvm -fptrauth-intrinsics -fptrauth-calls -fptrauth-vtable-pointer-type-discrimination -fptrauth-vtable-pointer-address-discrimination -fsanitize=null,vptr %s -o - | FileCheck %s --check-prefix=CHECK-VPTR --check-prefix=ITANIUM --check-prefix=CHECK-PTRAUTH
+// RUN: %clang_cc1 -std=c++11 -triple aarch64-unknown-linux -emit-llvm -fptrauth-intrinsics -fptrauth-calls -fptrauth-vtable-pointer-type-discrimination -fptrauth-vtable-pointer-address-discrimination -fsanitize=null,vptr %s -o - | FileCheck %s --check-prefix=CHECK-VPTR --check-prefix=ITANIUM --check-prefix=CHECK-PTRAUTH
+
 struct T {
   virtual ~T() {}
   virtual int v() { return 1; }
@@ -29,7 +31,7 @@ int get_v(T* t) {
   // CHECK-NULL: load ptr, ptr {{.*}}
 
   // CHECK-PTRAUTH: [[CAST_VTABLE:%.*]] = ptrtoint ptr %vtable to i64
-  // CHECK-PTRAUTH: [[STRIPPED_VTABLE:%.*]] = call i64 @llvm.ptrauth.strip(i64 [[CAST_VTABLE]], i32 0), !nosanitize !2
+  // CHECK-PTRAUTH: [[STRIPPED_VTABLE:%.*]] = call i64 @llvm.ptrauth.strip(i64 [[CAST_VTABLE]], i32 0), !nosanitize
   // CHECK-PTRAUTH: [[STRIPPED_PTR:%.*]] = inttoptr i64 [[STRIPPED_VTABLE]] to ptr
   // CHECK-PTRAUTH: [[STRIPPED_INT:%.*]] = ptrtoint ptr [[STRIPPED_PTR]] to i64
   // Make sure authed vtable pointer feeds into hashing
diff --git a/clang/test/SemaCXX/ptrauth-incomplete-virtual-member-function-return-arg-type.cpp b/clang/test/SemaCXX/ptrauth-incomplete-virtual-member-function-return-arg-type.cpp
index 41bbba0e832fd..9cc88750363c6 100644
--- a/clang/test/SemaCXX/ptrauth-incomplete-virtual-member-function-return-arg-type.cpp
+++ b/clang/test/SemaCXX/ptrauth-incomplete-virtual-member-function-return-arg-type.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple arm64-apple-ios -std=c++17 -Wno-vla -fsyntax-only -verify -fptrauth-intrinsics -fptrauth-calls %s
+// RUN: %clang_cc1 -triple arm64-apple-ios   -std=c++17 -Wno-vla -fsyntax-only -verify -fptrauth-intrinsics -fptrauth-calls %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -std=c++17 -Wno-vla -fsyntax-only -verify -fptrauth-intrinsics -fptrauth-calls %s
 
 struct Incomplete0; // expected-note 3 {{forward declaration of 'Incomplete0'}}
 
diff --git a/clang/test/SemaCXX/vtable_pointer_authentication_attribute.cpp b/clang/test/SemaCXX/vtable_pointer_authentication_attribute.cpp
index 3a3386196fbfa..f228c4e6ad0c0 100644
--- a/clang/test/SemaCXX/vtable_pointer_authentication_attribute.cpp
+++ b/clang/test/SemaCXX/vtable_pointer_authentication_attribute.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fsyntax-only -triple arm64-apple-ios -verify -fptrauth-calls -std=c++2a %s
+// RUN: %clang_cc1 -fsyntax-only -triple arm64-apple-ios   -verify -fptrauth-calls -std=c++2a %s
+// RUN: %clang_cc1 -fsyntax-only -triple aarch64-linux-gnu -verify -fptrauth-calls -std=c++2a %s
 
 namespace basic {
 

From 2df9fd7edb73c1c2b27bda433ae0795bc8076dd3 Mon Sep 17 00:00:00 2001
From: Haowei <haowei@google.com>
Date: Fri, 19 Jul 2024 00:03:12 -0700
Subject: [PATCH 146/486] Fix diagnostics-dsym.test on mac-arm64 (#99399)

The check ordering of diagnostics-dsym.test is wrong and it causes
test failure when running on mac-arm64 machine. This patch fixes it.
---
 clang/test/InstallAPI/diagnostics-dsym.test | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/test/InstallAPI/diagnostics-dsym.test b/clang/test/InstallAPI/diagnostics-dsym.test
index 42fa67a1f9b1e..b8433653c3f76 100644
--- a/clang/test/InstallAPI/diagnostics-dsym.test
+++ b/clang/test/InstallAPI/diagnostics-dsym.test
@@ -19,8 +19,8 @@
 ; RUN: --verify-mode=Pedantic 2>&1 | FileCheck %s
 
 ; CHECK: violations found for arm64 
-; CHECK: foo.c:5:0: error: no declaration found for exported symbol 'bar' in dynamic library
-; CHECK: foo.c:1:0: error: no declaration found for exported symbol 'foo' in dynamic library
+; CHECK-DAG: foo.c:5:0: error: no declaration found for exported symbol 'bar' in dynamic library
+; CHECK-DAG: foo.c:1:0: error: no declaration found for exported symbol 'foo' in dynamic library
 
 ;--- foo.c
 int foo(void) {

From cfe043cf99f76980ff91dc1cd4026e852556fba1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Fri, 19 Jul 2024 08:05:10 +0100
Subject: [PATCH 147/486] [mlir][linalg] Restrict scalable vectorisation
 (#98639)

Updates `vectorizeScalableVectorPrecondition` so that scalable
vectorisation is only applied in well understood and tested scenarios.

It's unlikely that we would ever want an arbitrary dimension to be
scalable. While the Linalg vectoriser should be flexible enough to
handle all possibilities:
  * in more "exotic" cases, we are likely to struggle with lowerings
    further down the compilation stack,
  * it would be impractical given the limitations of LLVM (which usually
    reflect the limitations of actual hardware) - e.g. no support for
    "scalable" arrays of scalable or fixed width vectors (*).

Ultimately, the goal of this patch is to better document what's
currently supported. While this PR adds some new restrictions, no
existing tests are affected.

(*) At MLIR vector level that would correspond to e.g.
`vector<[4]x8xf32>`.
---
 .../Linalg/Transforms/Vectorization.cpp       | 71 ++++++++++++++++---
 .../Linalg/vectorization-unsupported.mlir     | 67 ++++++++++++++++-
 2 files changed, 128 insertions(+), 10 deletions(-)

diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 68ee915cca3f4..7f7168eb86832 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -1945,7 +1945,8 @@ vectorizePadOpPrecondition(tensor::PadOp padOp,
   return success();
 }
 
-/// Preconditions for scalable vectors.
+/// Preconditions for scalable vectors. This is quite restrictive - it models
+/// the fact that in practice we would only make selected dimensions scalable.
 static LogicalResult
 vectorizeScalableVectorPrecondition(Operation *op,
                                     ArrayRef<int64_t> inputVectorSizes,
@@ -1953,18 +1954,70 @@ vectorizeScalableVectorPrecondition(Operation *op,
   assert(inputVectorSizes.size() == inputScalableVecDims.size() &&
          "Number of input vector sizes and scalable dims doesn't match");
 
-  if (inputVectorSizes.empty())
-    return success();
+  size_t numOfScalableDims =
+      llvm::count_if(inputScalableVecDims, [](bool flag) { return flag; });
 
-  bool isScalable = inputScalableVecDims.back();
-  if (!isScalable)
+  if (numOfScalableDims == 0)
     return success();
 
-  // Only element-wise and 1d depthwise conv ops supported in the presence of
-  // scalable dims.
   auto linalgOp = dyn_cast<LinalgOp>(op);
-  return success(linalgOp && (isElementwise(linalgOp) ||
-                              isa<linalg::DepthwiseConv1DNwcWcOp>(op)));
+
+  // Cond 1: There's been no need for scalable vectorisation of
+  // non-linalg Ops so far
+  if (!linalgOp)
+    return failure();
+
+  // Cond 2: There's been no need for more than 2 scalable dims so far
+  if (numOfScalableDims > 2)
+    return failure();
+
+  // Cond 3: Look at the configuration in `inputScalableVecDims` and verify that
+  // it matches one of the supported cases:
+  //  1. exactly 1 dim is scalable and that's the _last_ parallel dim
+  //  2. exactly 2 dims are scalable and those are the _last two adjacent_
+  //     parallel dims
+  // The 2nd restriction above means that only Matmul-like Ops are supported
+  // when 2 dims are scalable, e.g. :
+  //    * iterators = [parallel, parallel, reduction]
+  //    * scalable flags = [true, true, false]
+
+  // Find the first scalable flag
+  bool seenParalell = false;
+  auto iterators = linalgOp.getIteratorTypesArray();
+  SmallVector<bool> scalableFlags(inputScalableVecDims);
+  while (!scalableFlags.back()) {
+    seenParalell |= (iterators.back() == utils::IteratorType::parallel);
+
+    iterators.pop_back();
+    scalableFlags.pop_back();
+  }
+
+  // TODO: Support scalable vectorisation for reduction dims
+  if (iterators.back() == utils::IteratorType::reduction)
+    return failure();
+
+  // If this is not the _last_ parallel dim, 1. above is not met
+  if (seenParalell)
+    return failure();
+
+  // If present, check the 2nd scalable dim. ATM, only Matmul-like Ops are
+  // supported for which expect the folowing config:
+  //    * iterators = [parallel, parallel, reduction]
+  //    * scalable flags = [true, true, false]
+  if (numOfScalableDims == 2) {
+    scalableFlags.pop_back();
+    iterators.pop_back();
+
+    if (!scalableFlags.back() ||
+        (iterators.back() != utils::IteratorType::parallel))
+      return failure();
+  }
+
+  // Cond 4: Only the following ops are supported in the
+  // presence of scalable vectors
+  return success(isElementwise(linalgOp) || isa<linalg::MatmulOp>(op) ||
+                 isa<linalg::MatmulTransposeAOp>(op) ||
+                 isa<linalg::DepthwiseConv1DNwcWcOp>(op));
 }
 
 LogicalResult mlir::linalg::vectorizeOpPrecondition(
diff --git a/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir b/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir
index 5d3c07c8e23c1..c7ec39b0dbfb3 100644
--- a/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir
@@ -110,7 +110,7 @@ module attributes {transform.with_named_sequence} {
   }
 }
 
-  // -----
+// -----
 
 func.func @test_pack_no_vectorize_dynamic_shape(%arg0: tensor<?xf32>, %arg1: tensor<4x16xf32>) -> tensor<4x16xf32> {
   %pad = arith.constant 0.000000e+00 : f32
@@ -126,3 +126,68 @@ module attributes {transform.with_named_sequence} {
     transform.yield
   }
 }
+
+// -----
+
+func.func @linalg_reduce_scalable(%input: tensor<?xf32>,
+                                  %acc: tensor<f32>) -> tensor<f32> {
+
+  // expected-error @+1 {{Attempted to vectorize, but failed}}
+  %0 = linalg.reduce ins(%input : tensor<?xf32>) outs(%acc : tensor<f32>) dimensions = [0]
+  (%in: f32, %init: f32) {
+    %0 = arith.addf %in, %init : f32
+    linalg.yield %0 : f32
+  }
+  return %0 : tensor<f32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.reduce"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %0 vector_sizes [[4]] : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @linalg_generic_scalable_reduction_dim(%input: tensor<?x?xf32>,
+                                                 %acc: tensor<?xf32>) -> tensor<?xf32> {
+
+  // expected-error @+1 {{Attempted to vectorize, but failed}}
+  %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
+                                         affine_map<(d0, d1) -> (d0)>],
+                        iterator_types = ["parallel", "reduction"] }
+    ins(%input : tensor<?x?xf32>)
+    outs(%acc : tensor<?xf32>) {
+    ^bb(%in: f32, %out: f32) :
+      %0 = arith.addf %in, %out : f32
+      linalg.yield %0 : f32
+    } -> tensor<?xf32>
+  return %0 : tensor<?xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %0 vector_sizes [1, [4]] : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @linalg_matmul_scalable_leading_parallel_dim(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {
+  // expected-error @+1 {{Attempted to vectorize, but failed}}
+  linalg.matmul ins(%A, %B: memref<?x?xf32>, memref<?x?xf32>)
+            outs(%C: memref<?x?xf32>)
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %matmul vector_sizes [[8], 16, 4] : !transform.any_op
+    transform.yield
+  }
+}

From 8d0cbef0eddb6c651be5880bdeb0a7709629ac9f Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 19 Jul 2024 00:13:33 -0700
Subject: [PATCH 148/486] [ADT] Use range-based for loops (NFC) (#99605)

---
 llvm/include/llvm/ADT/SetOperations.h | 33 ++++++++++++---------------
 1 file changed, 14 insertions(+), 19 deletions(-)

diff --git a/llvm/include/llvm/ADT/SetOperations.h b/llvm/include/llvm/ADT/SetOperations.h
index 2b1a103565f7d..40e4b05fd7425 100644
--- a/llvm/include/llvm/ADT/SetOperations.h
+++ b/llvm/include/llvm/ADT/SetOperations.h
@@ -43,9 +43,8 @@ static constexpr bool HasMemberEraseIter =
 template <class S1Ty, class S2Ty> bool set_union(S1Ty &S1, const S2Ty &S2) {
   bool Changed = false;
 
-  for (typename S2Ty::const_iterator SI = S2.begin(), SE = S2.end(); SI != SE;
-       ++SI)
-    if (S1.insert(*SI).second)
+  for (const auto &E : S2)
+    if (S1.insert(E).second)
       Changed = true;
 
   return Changed;
@@ -73,10 +72,9 @@ template <class S1Ty, class S2Ty> void set_intersect(S1Ty &S1, const S2Ty &S2) {
 template <class S1Ty, class S2Ty>
 S1Ty set_intersection_impl(const S1Ty &S1, const S2Ty &S2) {
   S1Ty Result;
-  for (typename S1Ty::const_iterator SI = S1.begin(), SE = S1.end(); SI != SE;
-       ++SI)
-    if (S2.count(*SI))
-      Result.insert(*SI);
+  for (const auto &E : S1)
+    if (S2.count(E))
+      Result.insert(E);
   return Result;
 }
 
@@ -94,10 +92,9 @@ S1Ty set_intersection(const S1Ty &S1, const S2Ty &S2) {
 template <class S1Ty, class S2Ty>
 S1Ty set_difference(const S1Ty &S1, const S2Ty &S2) {
   S1Ty Result;
-  for (typename S1Ty::const_iterator SI = S1.begin(), SE = S1.end(); SI != SE;
-       ++SI)
-    if (!S2.count(*SI)) // if the element is not in set2
-      Result.insert(*SI);
+  for (const auto &E : S1)
+    if (!S2.count(E)) // if the element is not in set2
+      Result.insert(E);
   return Result;
 }
 
@@ -132,9 +129,8 @@ template <class S1Ty, class S2Ty> void set_subtract(S1Ty &S1, const S2Ty &S2) {
     }
   }
 
-  for (typename S2Ty::const_iterator SI = S2.begin(), SE = S2.end(); SI != SE;
-       ++SI)
-    S1.erase(*SI);
+  for (const auto &E : S2)
+    S1.erase(E);
 }
 
 /// set_subtract(A, B, C, D) - Compute A := A - B, set C to the elements of B
@@ -142,12 +138,11 @@ template <class S1Ty, class S2Ty> void set_subtract(S1Ty &S1, const S2Ty &S2) {
 /// from A (B - A).
 template <class S1Ty, class S2Ty>
 void set_subtract(S1Ty &S1, const S2Ty &S2, S1Ty &Removed, S1Ty &Remaining) {
-  for (typename S2Ty::const_iterator SI = S2.begin(), SE = S2.end(); SI != SE;
-       ++SI)
-    if (S1.erase(*SI))
-      Removed.insert(*SI);
+  for (const auto &E : S2)
+    if (S1.erase(E))
+      Removed.insert(E);
     else
-      Remaining.insert(*SI);
+      Remaining.insert(E);
 }
 
 /// set_is_subset(A, B) - Return true iff A in B

From e6668b1be8acbabeea0704bdc224637059aaa2af Mon Sep 17 00:00:00 2001
From: Corentin Ferry <corentin.ferry@amd.com>
Date: Fri, 19 Jul 2024 09:22:39 +0200
Subject: [PATCH 149/486] [mlir][tosa] Use roundeven in TOSA cast splat
 constant op folding (#99484)

The behavior of TOSA Cast operation for floating-point to integers is to round to the nearest even. This commit aligns the behavior of folding a TOSA Cast of a float splat to int, so it also uses roundeven.
---
 mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp |  3 ++-
 mlir/test/Dialect/Tosa/constant-op-fold.mlir       | 11 +++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 866ab0d2228f7..da9a93feac4d6 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -764,7 +764,8 @@ OpFoldResult CastOp::fold(FoldAdaptor adaptor) {
           llvm::cast<IntegerType>(outETy).getIntOrFloatBitWidth(), unsign);
       auto floatVal = operand.getSplatValue<APFloat>();
       bool exact;
-      floatVal.convertToInteger(intVal, llvm::RoundingMode::TowardZero, &exact);
+      floatVal.convertToInteger(intVal, llvm::RoundingMode::NearestTiesToEven,
+                                &exact);
       return SplatElementsAttr::get(outTy, intVal);
     }
 
diff --git a/mlir/test/Dialect/Tosa/constant-op-fold.mlir b/mlir/test/Dialect/Tosa/constant-op-fold.mlir
index ad82c9f8858e6..8e19f87dbf4aa 100644
--- a/mlir/test/Dialect/Tosa/constant-op-fold.mlir
+++ b/mlir/test/Dialect/Tosa/constant-op-fold.mlir
@@ -566,6 +566,17 @@ func.func @cast_float_to_int() -> tensor<i16> {
 
 // -----
 
+// CHECK: func.func @cast_float_to_int_round
+func.func @cast_float_to_int_round() -> tensor<i16> {
+  %splat = "tosa.const"() {value = dense<-3.5> : tensor<f32>} : () -> tensor<f32>
+  // CHECK: %[[SPLAT:.+]] = "tosa.const"() <{value = dense<-4> : tensor<i16>}
+  %cast = tosa.cast %splat : (tensor<f32>) -> tensor<i16>
+  // CHECK: return %[[SPLAT]]
+  return %cast : tensor<i16>
+}
+
+// -----
+
 // CHECK: func.func @cast_int_to_int_trunc
 func.func @cast_int_to_int_trunc() -> tensor<i16> {
   %splat = "tosa.const"() {value = dense<-1> : tensor<i32>} : () -> tensor<i32>

From 5c93a94f5adaa304922a936842162cba91785211 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project@meinersbur.de>
Date: Fri, 19 Jul 2024 09:24:40 +0200
Subject: [PATCH 150/486] [Clang][OpenMP] Add interchange directive (#93022)

Add the interchange directive which will be introduced in the upcoming
OpenMP 6.0 specification. A preview has been published in [Technical
Report 12](https://www.openmp.org/wp-content/uploads/openmp-TR12.pdf).
---
 clang/include/clang-c/Index.h                 |    4 +
 clang/include/clang/AST/RecursiveASTVisitor.h |    3 +
 clang/include/clang/AST/StmtOpenMP.h          |   76 +-
 clang/include/clang/Basic/StmtNodes.td        |    1 +
 clang/include/clang/Sema/SemaOpenMP.h         |    6 +
 .../include/clang/Serialization/ASTBitCodes.h |    1 +
 clang/lib/AST/StmtOpenMP.cpp                  |   20 +
 clang/lib/AST/StmtPrinter.cpp                 |    5 +
 clang/lib/AST/StmtProfile.cpp                 |    5 +
 clang/lib/Basic/OpenMPKinds.cpp               |    3 +-
 clang/lib/CodeGen/CGStmt.cpp                  |    3 +
 clang/lib/CodeGen/CGStmtOpenMP.cpp            |   10 +
 clang/lib/CodeGen/CodeGenFunction.h           |    1 +
 clang/lib/Parse/ParseOpenMP.cpp               |    1 +
 clang/lib/Sema/SemaExceptionSpec.cpp          |    1 +
 clang/lib/Sema/SemaOpenMP.cpp                 |  160 ++
 clang/lib/Sema/TreeTransform.h                |   11 +
 clang/lib/Serialization/ASTReaderStmt.cpp     |   11 +
 clang/lib/Serialization/ASTWriterStmt.cpp     |    5 +
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp  |    1 +
 clang/test/OpenMP/interchange_ast_print.cpp   |  135 ++
 clang/test/OpenMP/interchange_codegen.cpp     | 1990 +++++++++++++++++
 clang/test/OpenMP/interchange_messages.cpp    |   77 +
 clang/tools/libclang/CIndex.cpp               |    8 +
 clang/tools/libclang/CXCursor.cpp             |    3 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |    4 +
 .../test/transform/interchange/foreach.cpp    |  216 ++
 .../test/transform/interchange/intfor.c       |   38 +
 .../test/transform/interchange/iterfor.cpp    |  222 ++
 .../parallel-wsloop-collapse-foreach.cpp      |  340 +++
 .../parallel-wsloop-collapse-intfor.cpp       |  106 +
 31 files changed, 3465 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/OpenMP/interchange_ast_print.cpp
 create mode 100644 clang/test/OpenMP/interchange_codegen.cpp
 create mode 100644 clang/test/OpenMP/interchange_messages.cpp
 create mode 100644 openmp/runtime/test/transform/interchange/foreach.cpp
 create mode 100644 openmp/runtime/test/transform/interchange/intfor.c
 create mode 100644 openmp/runtime/test/transform/interchange/iterfor.cpp
 create mode 100644 openmp/runtime/test/transform/interchange/parallel-wsloop-collapse-foreach.cpp
 create mode 100644 openmp/runtime/test/transform/interchange/parallel-wsloop-collapse-intfor.cpp

diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index d096183ef2037..115f5ab090f96 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2150,6 +2150,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPReverseDirective = 307,
 
+  /** OpenMP interchange directive.
+   */
+  CXCursor_OMPInterchangeDirective = 308,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 36deec918c4b9..e3c0cb46799f7 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3035,6 +3035,9 @@ DEF_TRAVERSE_STMT(OMPUnrollDirective,
 DEF_TRAVERSE_STMT(OMPReverseDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPInterchangeDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPForDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index e41a9e52b7674..194eb2d10dcb3 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -1009,7 +1009,7 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
   static bool classof(const Stmt *T) {
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
-           C == OMPReverseDirectiveClass;
+           C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass;
   }
 };
 
@@ -5776,6 +5776,80 @@ class OMPReverseDirective final : public OMPLoopTransformationDirective {
   }
 };
 
+/// Represents the '#pragma omp interchange' loop transformation directive.
+///
+/// \code{c}
+///   #pragma omp interchange
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j)
+///       ..
+/// \endcode
+class OMPInterchangeDirective final : public OMPLoopTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  /// Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPInterchangeDirective(SourceLocation StartLoc,
+                                   SourceLocation EndLoc, unsigned NumLoops)
+      : OMPLoopTransformationDirective(OMPInterchangeDirectiveClass,
+                                       llvm::omp::OMPD_interchange, StartLoc,
+                                       EndLoc, NumLoops) {
+    setNumGeneratedLoops(3 * NumLoops);
+  }
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for '#pragma omp interchange'.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
+  /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
+  /// \param Clauses   The directive's clauses.
+  /// \param NumLoops  Number of affected loops
+  ///                  (number of items in the 'permutation' clause if present).
+  /// \param AssociatedStmt  The outermost associated loop.
+  /// \param TransformedStmt The loop nest after tiling, or nullptr in
+  ///                        dependent contexts.
+  /// \param PreInits  Helper preinits statements for the loop nest.
+  static OMPInterchangeDirective *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+         ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+         Stmt *TransformedStmt, Stmt *PreInits);
+
+  /// Build an empty '#pragma omp interchange' AST node for deserialization.
+  ///
+  /// \param C          Context of the AST.
+  /// \param NumClauses Number of clauses to allocate.
+  /// \param NumLoops   Number of associated loops to allocate.
+  static OMPInterchangeDirective *
+  CreateEmpty(const ASTContext &C, unsigned NumClauses, unsigned NumLoops);
+
+  /// Gets the associated loops after the transformation. This is the de-sugared
+  /// replacement or nullptr in dependent contexts.
+  Stmt *getTransformedStmt() const {
+    return Data->getChildren()[TransformedStmtOffset];
+  }
+
+  /// Return preinits statement.
+  Stmt *getPreInits() const { return Data->getChildren()[PreInitsOffset]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPInterchangeDirectiveClass;
+  }
+};
+
 /// This represents '#pragma omp scan' directive.
 ///
 /// \code
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index 426b8ec4b4496..9bf23fae50a9e 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -231,6 +231,7 @@ def OMPLoopTransformationDirective : StmtNode<OMPLoopBasedDirective, 1>;
 def OMPTileDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPUnrollDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPReverseDirective : StmtNode<OMPLoopTransformationDirective>;
+def OMPInterchangeDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPForDirective : StmtNode<OMPLoopDirective>;
 def OMPForSimdDirective : StmtNode<OMPLoopDirective>;
 def OMPSectionsDirective : StmtNode<OMPExecutableDirective>;
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 14dd3fec3e2e9..54d81f91ffebc 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -426,6 +426,12 @@ class SemaOpenMP : public SemaBase {
   /// Called on well-formed '#pragma omp reverse'.
   StmtResult ActOnOpenMPReverseDirective(Stmt *AStmt, SourceLocation StartLoc,
                                          SourceLocation EndLoc);
+  /// Called on well-formed '#pragma omp interchange' after parsing of its
+  /// clauses and the associated statement.
+  StmtResult ActOnOpenMPInterchangeDirective(ArrayRef<OMPClause *> Clauses,
+                                             Stmt *AStmt,
+                                             SourceLocation StartLoc,
+                                             SourceLocation EndLoc);
   /// Called on well-formed '\#pragma omp for' after parsing
   /// of the associated statement.
   StmtResult
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 8cd24699505df..5dd0ba33f8a9c 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1896,6 +1896,7 @@ enum StmtCode {
   STMT_OMP_TILE_DIRECTIVE,
   STMT_OMP_UNROLL_DIRECTIVE,
   STMT_OMP_REVERSE_DIRECTIVE,
+  STMT_OMP_INTERCHANGE_DIRECTIVE,
   STMT_OMP_FOR_DIRECTIVE,
   STMT_OMP_FOR_SIMD_DIRECTIVE,
   STMT_OMP_SECTIONS_DIRECTIVE,
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 8b19b4209757f..a2325b177d41e 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -467,6 +467,26 @@ OMPReverseDirective *OMPReverseDirective::CreateEmpty(const ASTContext &C) {
       TransformedStmtOffset + 1, SourceLocation(), SourceLocation());
 }
 
+OMPInterchangeDirective *OMPInterchangeDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+    Stmt *TransformedStmt, Stmt *PreInits) {
+  OMPInterchangeDirective *Dir = createDirective<OMPInterchangeDirective>(
+      C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
+  Dir->setTransformedStmt(TransformedStmt);
+  Dir->setPreInits(PreInits);
+  return Dir;
+}
+
+OMPInterchangeDirective *
+OMPInterchangeDirective::CreateEmpty(const ASTContext &C, unsigned NumClauses,
+                                     unsigned NumLoops) {
+  return createEmptyDirective<OMPInterchangeDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
+      SourceLocation(), SourceLocation(), NumLoops);
+}
+
 OMPForSimdDirective *
 OMPForSimdDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                             SourceLocation EndLoc, unsigned CollapsedNum,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 58fb222d99153..69e0b763e8ddc 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -768,6 +768,11 @@ void StmtPrinter::VisitOMPReverseDirective(OMPReverseDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPInterchangeDirective(OMPInterchangeDirective *Node) {
+  Indent() << "#pragma omp interchange";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPForDirective(OMPForDirective *Node) {
   Indent() << "#pragma omp for";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index ba04136d9896e..89d2a422509d8 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -989,6 +989,11 @@ void StmtProfiler::VisitOMPReverseDirective(const OMPReverseDirective *S) {
   VisitOMPLoopTransformationDirective(S);
 }
 
+void StmtProfiler::VisitOMPInterchangeDirective(
+    const OMPInterchangeDirective *S) {
+  VisitOMPLoopTransformationDirective(S);
+}
+
 void StmtProfiler::VisitOMPForDirective(const OMPForDirective *S) {
   VisitOMPLoopDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index a442b02ad3b55..b141e48e77e3c 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -684,7 +684,8 @@ bool clang::isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind) {
 }
 
 bool clang::isOpenMPLoopTransformationDirective(OpenMPDirectiveKind DKind) {
-  return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse;
+  return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
+         DKind == OMPD_interchange;
 }
 
 bool clang::isOpenMPCombinedParallelADirective(OpenMPDirectiveKind DKind) {
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 47246494fc782..aa97f685ac7a9 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -225,6 +225,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPReverseDirectiveClass:
     EmitOMPReverseDirective(cast<OMPReverseDirective>(*S));
     break;
+  case Stmt::OMPInterchangeDirectiveClass:
+    EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));
+    break;
   case Stmt::OMPForDirectiveClass:
     EmitOMPForDirective(cast<OMPForDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index ec8eeabf96210..adf74ea16c895 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -189,6 +189,9 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
       PreInits = Unroll->getPreInits();
     } else if (const auto *Reverse = dyn_cast<OMPReverseDirective>(&S)) {
       PreInits = Reverse->getPreInits();
+    } else if (const auto *Interchange =
+                   dyn_cast<OMPInterchangeDirective>(&S)) {
+      PreInits = Interchange->getPreInits();
     } else {
       llvm_unreachable("Unknown loop-based directive kind.");
     }
@@ -2770,6 +2773,13 @@ void CodeGenFunction::EmitOMPReverseDirective(const OMPReverseDirective &S) {
   EmitStmt(S.getTransformedStmt());
 }
 
+void CodeGenFunction::EmitOMPInterchangeDirective(
+    const OMPInterchangeDirective &S) {
+  // Emit the de-sugared statement.
+  OMPTransformDirectiveScopeRAII InterchangeScope(*this, &S);
+  EmitStmt(S.getTransformedStmt());
+}
+
 void CodeGenFunction::EmitOMPUnrollDirective(const OMPUnrollDirective &S) {
   bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index d83e38cab8e2d..eebb865e8f42c 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3821,6 +3821,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPTileDirective(const OMPTileDirective &S);
   void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
   void EmitOMPReverseDirective(const OMPReverseDirective &S);
+  void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
   void EmitOMPForDirective(const OMPForDirective &S);
   void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
   void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 5b2fd4f4d5397..f5b44d210680c 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -2886,6 +2886,7 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
     break;
   }
   case OMPD_reverse:
+  case OMPD_interchange:
   case OMPD_declare_target: {
     SourceLocation DTLoc = ConsumeAnyToken();
     bool HasClauses = Tok.isNot(tok::annot_pragma_openmp_end);
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 4192777d3d4bd..427ffd9061ef3 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1467,6 +1467,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPTileDirectiveClass:
   case Stmt::OMPUnrollDirectiveClass:
   case Stmt::OMPReverseDirectiveClass:
+  case Stmt::OMPInterchangeDirectiveClass:
   case Stmt::OMPSingleDirectiveClass:
   case Stmt::OMPTargetDataDirectiveClass:
   case Stmt::OMPTargetDirectiveClass:
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 0f2e8bc1513a7..3bd981cb442aa 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -40,6 +40,7 @@
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/PointerEmbeddedInt.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Frontend/OpenMP/OMPAssume.h"
@@ -4406,6 +4407,7 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
   case OMPD_tile:
   case OMPD_unroll:
   case OMPD_reverse:
+  case OMPD_interchange:
     break;
   default:
     processCapturedRegions(SemaRef, DKind, CurScope,
@@ -6290,6 +6292,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
            "reverse directive does not support any clauses");
     Res = ActOnOpenMPReverseDirective(AStmt, StartLoc, EndLoc);
     break;
+  case OMPD_interchange:
+    Res = ActOnOpenMPInterchangeDirective(ClausesWithImplicit, AStmt, StartLoc,
+                                          EndLoc);
+    break;
   case OMPD_for:
     Res = ActOnOpenMPForDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc,
                                   VarsWithInheritedDSA);
@@ -14048,6 +14054,8 @@ bool SemaOpenMP::checkTransformableLoopNest(
           DependentPreInits = Dir->getPreInits();
         else if (auto *Dir = dyn_cast<OMPReverseDirective>(Transform))
           DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPInterchangeDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
         else
           llvm_unreachable("Unhandled loop transformation");
 
@@ -14853,6 +14861,158 @@ StmtResult SemaOpenMP::ActOnOpenMPReverseDirective(Stmt *AStmt,
                                      buildPreInits(Context, PreInits));
 }
 
+StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
+    ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+    SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // interchange without permutation clause swaps two loops.
+  constexpr size_t NumLoops = 2;
+
+  // Verify and diagnose loop nest.
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
+  Stmt *Body = nullptr;
+  SmallVector<SmallVector<Stmt *, 0>, 2> OriginalInits;
+  if (!checkTransformableLoopNest(OMPD_interchange, AStmt, NumLoops,
+                                  LoopHelpers, Body, OriginalInits))
+    return StmtError();
+
+  // Delay interchange to when template is completely instantiated.
+  if (CurContext->isDependentContext())
+    return OMPInterchangeDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                           NumLoops, AStmt, nullptr, nullptr);
+
+  assert(LoopHelpers.size() == NumLoops &&
+         "Expecting loop iteration space dimensionaly to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == NumLoops &&
+         "Expecting loop iteration space dimensionaly to match number of "
+         "affected loops");
+
+  // Decode the permutation clause.
+  constexpr uint64_t Permutation[] = {1, 0};
+
+  // Find the affected loops.
+  SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
+  collectLoopStmts(AStmt, LoopStmts);
+
+  // Collect pre-init statements on the order before the permuation.
+  SmallVector<Stmt *> PreInits;
+  for (auto I : llvm::seq<int>(NumLoops)) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+
+    assert(LoopHelper.Counters.size() == 1 &&
+           "Single-dimensional loop iteration space expected");
+    auto *OrigCntVar = cast<DeclRefExpr>(LoopHelper.Counters.front());
+
+    std::string OrigVarName = OrigCntVar->getNameInfo().getAsString();
+    addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
+                    PreInits);
+  }
+
+  SmallVector<VarDecl *> PermutedIndVars(NumLoops);
+  CaptureVars CopyTransformer(SemaRef);
+
+  // Create the permuted loops from the inside to the outside of the
+  // interchanged loop nest. Body of the innermost new loop is the original
+  // innermost body.
+  Stmt *Inner = Body;
+  for (auto TargetIdx : llvm::reverse(llvm::seq<int>(NumLoops))) {
+    // Get the original loop that belongs to this new position.
+    uint64_t SourceIdx = Permutation[TargetIdx];
+    OMPLoopBasedDirective::HelperExprs &SourceHelper = LoopHelpers[SourceIdx];
+    Stmt *SourceLoopStmt = LoopStmts[SourceIdx];
+    assert(SourceHelper.Counters.size() == 1 &&
+           "Single-dimensional loop iteration space expected");
+    auto *OrigCntVar = cast<DeclRefExpr>(SourceHelper.Counters.front());
+
+    // Normalized loop counter variable: From 0 to n-1, always an integer type.
+    DeclRefExpr *IterVarRef = cast<DeclRefExpr>(SourceHelper.IterationVarRef);
+    QualType IVTy = IterVarRef->getType();
+    assert(IVTy->isIntegerType() &&
+           "Expected the logical iteration counter to be an integer");
+
+    std::string OrigVarName = OrigCntVar->getNameInfo().getAsString();
+    SourceLocation OrigVarLoc = IterVarRef->getExprLoc();
+
+    // Make a copy of the NumIterations expression for each use: By the AST
+    // constraints, every expression object in a DeclContext must be unique.
+    auto MakeNumIterations = [&CopyTransformer, &SourceHelper]() -> Expr * {
+      return AssertSuccess(
+          CopyTransformer.TransformExpr(SourceHelper.NumIterations));
+    };
+
+    // Iteration variable for the permuted loop. Reuse the one from
+    // checkOpenMPLoop which will also be used to update the original loop
+    // variable.
+    SmallString<64> PermutedCntName(".permuted_");
+    PermutedCntName.append({llvm::utostr(TargetIdx), ".iv.", OrigVarName});
+    auto *PermutedCntDecl = cast<VarDecl>(IterVarRef->getDecl());
+    PermutedCntDecl->setDeclName(
+        &SemaRef.PP.getIdentifierTable().get(PermutedCntName));
+    PermutedIndVars[TargetIdx] = PermutedCntDecl;
+    auto MakePermutedRef = [this, PermutedCntDecl, IVTy, OrigVarLoc]() {
+      return buildDeclRefExpr(SemaRef, PermutedCntDecl, IVTy, OrigVarLoc);
+    };
+
+    // For init-statement:
+    // \code
+    //   auto .permuted_{target}.iv = 0
+    // \endcode
+    ExprResult Zero = SemaRef.ActOnIntegerConstant(OrigVarLoc, 0);
+    if (!Zero.isUsable())
+      return StmtError();
+    SemaRef.AddInitializerToDecl(PermutedCntDecl, Zero.get(),
+                                 /*DirectInit=*/false);
+    StmtResult InitStmt = new (Context)
+        DeclStmt(DeclGroupRef(PermutedCntDecl), OrigCntVar->getBeginLoc(),
+                 OrigCntVar->getEndLoc());
+    if (!InitStmt.isUsable())
+      return StmtError();
+
+    // For cond-expression:
+    // \code
+    //   .permuted_{target}.iv < MakeNumIterations()
+    // \endcode
+    ExprResult CondExpr =
+        SemaRef.BuildBinOp(CurScope, SourceHelper.Cond->getExprLoc(), BO_LT,
+                           MakePermutedRef(), MakeNumIterations());
+    if (!CondExpr.isUsable())
+      return StmtError();
+
+    // For incr-statement:
+    // \code
+    //   ++.tile.iv
+    // \endcode
+    ExprResult IncrStmt = SemaRef.BuildUnaryOp(
+        CurScope, SourceHelper.Inc->getExprLoc(), UO_PreInc, MakePermutedRef());
+    if (!IncrStmt.isUsable())
+      return StmtError();
+
+    SmallVector<Stmt *, 4> BodyParts(SourceHelper.Updates.begin(),
+                                     SourceHelper.Updates.end());
+    if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(SourceLoopStmt))
+      BodyParts.push_back(SourceCXXFor->getLoopVarStmt());
+    BodyParts.push_back(Inner);
+    Inner = CompoundStmt::Create(Context, BodyParts, FPOptionsOverride(),
+                                 Inner->getBeginLoc(), Inner->getEndLoc());
+    Inner = new (Context) ForStmt(
+        Context, InitStmt.get(), CondExpr.get(), nullptr, IncrStmt.get(), Inner,
+        SourceHelper.Init->getBeginLoc(), SourceHelper.Init->getBeginLoc(),
+        SourceHelper.Inc->getEndLoc());
+  }
+
+  return OMPInterchangeDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                         NumLoops, AStmt, Inner,
+                                         buildPreInits(Context, PreInits));
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
                                                    Expr *Expr,
                                                    SourceLocation StartLoc,
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 0f266171cb67a..84e846356e437 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -9250,6 +9250,17 @@ TreeTransform<Derived>::TransformOMPReverseDirective(OMPReverseDirective *D) {
   return Res;
 }
 
+template <typename Derived>
+StmtResult TreeTransform<Derived>::TransformOMPInterchangeDirective(
+    OMPInterchangeDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPForDirective(OMPForDirective *D) {
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 9d9329c9a58cc..a7fe20bd0a466 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2449,6 +2449,10 @@ void ASTStmtReader::VisitOMPReverseDirective(OMPReverseDirective *D) {
   VisitOMPLoopTransformationDirective(D);
 }
 
+void ASTStmtReader::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
+  VisitOMPLoopTransformationDirective(D);
+}
+
 void ASTStmtReader::VisitOMPForDirective(OMPForDirective *D) {
   VisitOMPLoopDirective(D);
   D->setHasCancel(Record.readBool());
@@ -3477,6 +3481,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       break;
     }
 
+    case STMT_OMP_INTERCHANGE_DIRECTIVE: {
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
+      unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
+      S = OMPInterchangeDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      break;
+    }
+
     case STMT_OMP_FOR_DIRECTIVE: {
       unsigned CollapsedNum = Record[ASTStmtReader::NumStmtFields];
       unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index ba5824e585972..caa222277f062 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2442,6 +2442,11 @@ void ASTStmtWriter::VisitOMPReverseDirective(OMPReverseDirective *D) {
   Code = serialization::STMT_OMP_REVERSE_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
+  VisitOMPLoopTransformationDirective(D);
+  Code = serialization::STMT_OMP_INTERCHANGE_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPForDirective(OMPForDirective *D) {
   VisitOMPLoopDirective(D);
   Record.writeBool(D->hasCancel());
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index e56f75be8ebc2..62a240ecbc600 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1813,6 +1813,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
     case Stmt::OMPReverseDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
+    case Stmt::OMPInterchangeDirectiveClass:
     case Stmt::OMPInteropDirectiveClass:
     case Stmt::OMPDispatchDirectiveClass:
     case Stmt::OMPMaskedDirectiveClass:
diff --git a/clang/test/OpenMP/interchange_ast_print.cpp b/clang/test/OpenMP/interchange_ast_print.cpp
new file mode 100644
index 0000000000000..f8bf075cd300f
--- /dev/null
+++ b/clang/test/OpenMP/interchange_ast_print.cpp
@@ -0,0 +1,135 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// PRINT-LABEL: void foo1(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo1
+void foo1() {
+  // PRINT: #pragma omp interchange
+  // DUMP:  OMPInterchangeDirective
+  #pragma omp interchange
+  // PRINT: for (int i = 7; i < 17; i += 3)
+  // DUMP-NEXT: ForStmt
+  for (int i = 7; i < 17; i += 3)
+    // PRINT: for (int j = 7; j < 17; j += 3)
+    // DUMP:  ForStmt
+    for (int j = 7; j < 17; j += 3)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+
+
+
+// PRINT-LABEL: void foo3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo3
+void foo3() {
+  // PRINT: #pragma omp for collapse(3)
+  // DUMP:      OMPForDirective
+  // DUMP-NEXT:   OMPCollapseClause
+  // DUMP-NEXT:     ConstantExpr
+  // DUMP-NEXT:       value: Int 3
+  // DUMP-NEXT:     IntegerLiteral {{.*}} 3
+  // DUMP-NEXT:     CapturedStmt
+  // DUMP-NEXT:       CapturedDecl
+  #pragma omp for collapse(3)
+  // PRINT: #pragma omp interchange
+  // DUMP:  OMPInterchangeDirective
+  #pragma omp interchange
+  // PRINT: for (int i = 7; i < 17; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 7; i < 17; i += 1)
+    // PRINT: for (int j = 7; j < 17; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 7; j < 17; j += 1)
+      // PRINT: for (int k = 7; k < 17; k += 1)
+      // DUMP:  ForStmt
+      for (int k = 7; k < 17; k += 1)
+        // PRINT: body(i, j, k);
+        // DUMP:  CallExpr
+        body(i, j, k);
+}
+
+
+// PRINT-LABEL: void foo6(
+// DUMP-LABEL: FunctionTemplateDecl {{.*}} foo6
+template<int Tile>
+void foo6() {
+    // PRINT:     #pragma omp interchange
+    // DUMP:      OMPInterchangeDirective
+    #pragma omp interchange
+      // PRINT-NEXT: for (int i = 0; i < 11; i += 2)
+      // DUMP-NEXT:  ForStmt
+      for (int i = 0; i < 11; i += 2)
+        // PRINT-NEXT: #pragma omp tile sizes(Tile)
+        // DUMP:       OMPTileDirective
+        #pragma omp tile sizes(Tile)
+        // PRINT-NEXT: for (int j = 0; j < 13; j += 2)
+        // DUMP:       ForStmt
+        for (int j = 0; j < 13; j += 2)
+          // PRINT-NEXT: body(i, j);
+          // DUMP:       CallExpr
+          body(i, j);
+}
+
+// Also test instantiating the template.
+void tfoo6() {
+  foo6<32>();
+}
+
+
+// PRINT-LABEL: void foo7(
+// DUMP-LABEL: FunctionDecl {{.*}} foo7
+void foo7() {
+  double arr[128];
+  // PRINT: #pragma omp interchange
+  // DUMP:  OMPInterchangeDirective
+  #pragma omp interchange
+  // PRINT-NEXT: for (double c = 42; auto &&v : arr)
+  // DUMP-NEXT:  CXXForRangeStmt
+  for (double c = 42; auto &&v : arr)
+    // PRINT-NEXT: for (int i = 0; i < 42; i += 2)
+    // DUMP:       ForStmt
+    for (int i = 0; i < 42; i += 2)
+      // PRINT-NEXT: body(c, v, i);
+      // DUMP:       CallExpr
+      body(c, v, i);
+}
+
+
+// PRINT-LABEL: void foo8(
+// DUMP-LABEL: FunctionDecl {{.*}} foo8
+void foo8() {
+  double arr[128];
+  // PRINT: #pragma omp interchange
+  // DUMP:  OMPInterchangeDirective
+  #pragma omp interchange
+  // PRINT-NEXT: for (int i = 0; i < 42; i += 2)
+  // DUMP-NEXT:  ForStmt
+  for (int i = 0; i < 42; i += 2)
+    // PRINT-NEXT: for (double c = 42; auto &&v : arr)
+    // DUMP:       CXXForRangeStmt
+    for (double c = 42; auto &&v : arr)
+      // PRINT-NEXT: body(i, c, v);
+      // DUMP:       CallExpr
+      body(i, c, v);
+}
+
+#endif
+
diff --git a/clang/test/OpenMP/interchange_codegen.cpp b/clang/test/OpenMP/interchange_codegen.cpp
new file mode 100644
index 0000000000000..9c1782183cf98
--- /dev/null
+++ b/clang/test/OpenMP/interchange_codegen.cpp
@@ -0,0 +1,1990 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
+
+// expected-no-diagnostics
+
+// Check code generation
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fclang-abi-compat=latest -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+
+// Check same results after serialization round-trip
+// FIXME: They should be exactly the same but currently differ in function order
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fclang-abi-compat=latest -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fclang-abi-compat=latest -fopenmp -fopenmp-version=60 -include-pch %t -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK2
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...) {}
+
+
+
+
+extern "C" void foo2(int start1, int start2, int end1, int end2, int step1, int step2) {
+#pragma omp interchange
+  for (int i = start1; i < end1; i += step1)
+    for (int j = start2; j < end2; j += step2)
+      body(i, j);
+}
+
+
+extern "C" void foo3() {
+#pragma omp for
+#pragma omp interchange
+  for (int i = 7; i < 17; i += 3)
+    for (int j = 7; j < 17; j += 3)
+      body(i, j);
+}
+
+
+extern "C" void foo4() {
+#pragma omp for collapse(2)
+  for (int k = 7; k < 17; k += 3)
+#pragma omp interchange
+    for (int i = 7; i < 17; i += 3)
+      for (int j = 7; j < 17; j += 3)
+        body(i, j);
+}
+
+
+extern "C" void foo6() {
+#pragma omp for collapse(4)
+  for (int i = 7; i < 17; i += 3)
+#pragma omp interchange
+    for (int j = 7; j < 17; j += 3)
+      for (int k = 7; k < 17; k += 3)
+        for (int l = 7; l < 17; l += 3)
+          body(i, j, k, l);
+}
+
+
+extern "C" void foo9() {
+  double arr[128];
+  #pragma omp interchange
+  for (double c = 42; auto && v : arr)
+    for (int i = 0; i < 42; i += 2)
+      body(c, v, i);
+}
+
+
+extern "C" void foo10() {
+  double A[128], B[16];
+  #pragma omp for collapse(4)
+  for (int i = 0; i < 128; ++i)
+    #pragma omp interchange
+    for (double c = 42; auto aa : A)
+      for (double d = 42; auto &bb : B)
+        for (int j = 0; j < 128; ++j)
+          body(i, c, aa, d, bb, j);
+}
+
+#endif /* HEADER */
+
+// CHECK1-LABEL: define {{[^@]+}}@body
+// CHECK1-SAME: (...) #[[ATTR0:[0-9]+]] {
+// CHECK1-NEXT:  entry:
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define {{[^@]+}}@foo2
+// CHECK1-SAME: (i32 noundef [[START1:%.*]], i32 noundef [[START2:%.*]], i32 noundef [[END1:%.*]], i32 noundef [[END2:%.*]], i32 noundef [[STEP1:%.*]], i32 noundef [[STEP2:%.*]]) #[[ATTR0]] {
+// CHECK1-NEXT:  entry:
+// CHECK1-NEXT:    [[START1_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[START2_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[END1_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[END2_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[STEP1_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[STEP2_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP7:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_8:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_0_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_1_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    store i32 [[START1]], ptr [[START1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[START2]], ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[END1]], ptr [[END1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[END2]], ptr [[END2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[STEP1]], ptr [[STEP1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[STEP2]], ptr [[STEP2_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[END1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[STEP1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP6]]
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
+// CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP9]], ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[END2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB9:%.*]] = sub i32 [[TMP12]], [[TMP13]]
+// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[SUB9]], 1
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[SUB10]], [[TMP14]]
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV12:%.*]] = udiv i32 [[ADD11]], [[TMP15]]
+// CHECK1-NEXT:    [[SUB13:%.*]] = sub i32 [[DIV12]], 1
+// CHECK1-NEXT:    store i32 [[SUB13]], ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK1:       for.cond:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP17]], 1
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP16]], [[ADD14]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP19]], [[TMP20]]
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP18]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND16:%.*]]
+// CHECK1:       for.cond16:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD17:%.*]] = add i32 [[TMP22]], 1
+// CHECK1-NEXT:    [[CMP18:%.*]] = icmp ult i32 [[TMP21]], [[ADD17]]
+// CHECK1-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body19:
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL20:%.*]] = mul i32 [[TMP24]], [[TMP25]]
+// CHECK1-NEXT:    [[ADD21:%.*]] = add i32 [[TMP23]], [[MUL20]]
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP26]], i32 noundef [[TMP27]])
+// CHECK1-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK1:       for.inc:
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP28]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND16]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK1:       for.end:
+// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK1:       for.inc22:
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK1-NEXT:    [[INC23:%.*]] = add i32 [[TMP29]], 1
+// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK1:       for.end24:
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define {{[^@]+}}@foo3
+// CHECK1-SAME: () #[[ATTR0]] {
+// CHECK1-NEXT:  entry:
+// CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_0_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_1_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
+// CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 3, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK1-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 3
+// CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 3, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK1:       omp.inner.for.cond:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP1:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    br i1 [[CMP1]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1:       omp.inner.for.body:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL2:%.*]] = mul nsw i32 [[TMP7]], 3
+// CHECK1-NEXT:    [[ADD3:%.*]] = add nsw i32 7, [[MUL2]]
+// CHECK1-NEXT:    store i32 [[ADD3]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK1:       for.cond:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 4
+// CHECK1-NEXT:    br i1 [[CMP4]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 7, [[MUL5]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP10]], i32 noundef [[TMP11]])
+// CHECK1-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK1:       for.inc:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP12]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK1:       for.end:
+// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1:       omp.body.continue:
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK1:       omp.inner.for.inc:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP13]], 1
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK1:       omp.inner.for.end:
+// CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK1:       omp.loop.exit:
+// CHECK1-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define {{[^@]+}}@foo4
+// CHECK1-SAME: () #[[ATTR0]] {
+// CHECK1-NEXT:  entry:
+// CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_0_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_1_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+// CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 15, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK1-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 15
+// CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 15, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK1:       omp.inner.for.cond:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1:       omp.inner.for.body:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 3
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV3:%.*]] = sdiv i32 [[TMP8]], 4
+// CHECK1-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[DIV3]], 4
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL4]]
+// CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[SUB]], 1
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL5]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 7, [[MUL7]]
+// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK1:       for.cond:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 4
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 7, [[MUL10]]
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
+// CHECK1-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK1:       for.inc:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK1:       for.end:
+// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1:       omp.body.continue:
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK1:       omp.inner.for.inc:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK1:       omp.inner.for.end:
+// CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK1:       omp.loop.exit:
+// CHECK1-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define {{[^@]+}}@foo6
+// CHECK1-SAME: () #[[ATTR0]] {
+// CHECK1-NEXT:  entry:
+// CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP3:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[L:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+// CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 7, ptr [[K]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 255, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK1-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 255
+// CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 255, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK1:       omp.inner.for.cond:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP4:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    br i1 [[CMP4]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1:       omp.inner.for.body:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 64
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 3
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV5:%.*]] = sdiv i32 [[TMP8]], 64
+// CHECK1-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[DIV5]], 64
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL6]]
+// CHECK1-NEXT:    [[DIV7:%.*]] = sdiv i32 [[SUB]], 16
+// CHECK1-NEXT:    [[MUL8:%.*]] = mul nsw i32 [[DIV7]], 1
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 0, [[MUL8]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[DOTPERMUTED_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV10:%.*]] = sdiv i32 [[TMP10]], 64
+// CHECK1-NEXT:    [[MUL11:%.*]] = mul nsw i32 [[DIV10]], 64
+// CHECK1-NEXT:    [[SUB12:%.*]] = sub nsw i32 [[TMP9]], [[MUL11]]
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV13:%.*]] = sdiv i32 [[TMP12]], 64
+// CHECK1-NEXT:    [[MUL14:%.*]] = mul nsw i32 [[DIV13]], 64
+// CHECK1-NEXT:    [[SUB15:%.*]] = sub nsw i32 [[TMP11]], [[MUL14]]
+// CHECK1-NEXT:    [[DIV16:%.*]] = sdiv i32 [[SUB15]], 16
+// CHECK1-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[DIV16]], 16
+// CHECK1-NEXT:    [[SUB18:%.*]] = sub nsw i32 [[SUB12]], [[MUL17]]
+// CHECK1-NEXT:    [[DIV19:%.*]] = sdiv i32 [[SUB18]], 4
+// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[DIV19]], 1
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 0, [[MUL20]]
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[DOTPERMUTED_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV22:%.*]] = sdiv i32 [[TMP14]], 64
+// CHECK1-NEXT:    [[MUL23:%.*]] = mul nsw i32 [[DIV22]], 64
+// CHECK1-NEXT:    [[SUB24:%.*]] = sub nsw i32 [[TMP13]], [[MUL23]]
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV25:%.*]] = sdiv i32 [[TMP16]], 64
+// CHECK1-NEXT:    [[MUL26:%.*]] = mul nsw i32 [[DIV25]], 64
+// CHECK1-NEXT:    [[SUB27:%.*]] = sub nsw i32 [[TMP15]], [[MUL26]]
+// CHECK1-NEXT:    [[DIV28:%.*]] = sdiv i32 [[SUB27]], 16
+// CHECK1-NEXT:    [[MUL29:%.*]] = mul nsw i32 [[DIV28]], 16
+// CHECK1-NEXT:    [[SUB30:%.*]] = sub nsw i32 [[SUB24]], [[MUL29]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV31:%.*]] = sdiv i32 [[TMP18]], 64
+// CHECK1-NEXT:    [[MUL32:%.*]] = mul nsw i32 [[DIV31]], 64
+// CHECK1-NEXT:    [[SUB33:%.*]] = sub nsw i32 [[TMP17]], [[MUL32]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV34:%.*]] = sdiv i32 [[TMP20]], 64
+// CHECK1-NEXT:    [[MUL35:%.*]] = mul nsw i32 [[DIV34]], 64
+// CHECK1-NEXT:    [[SUB36:%.*]] = sub nsw i32 [[TMP19]], [[MUL35]]
+// CHECK1-NEXT:    [[DIV37:%.*]] = sdiv i32 [[SUB36]], 16
+// CHECK1-NEXT:    [[MUL38:%.*]] = mul nsw i32 [[DIV37]], 16
+// CHECK1-NEXT:    [[SUB39:%.*]] = sub nsw i32 [[SUB33]], [[MUL38]]
+// CHECK1-NEXT:    [[DIV40:%.*]] = sdiv i32 [[SUB39]], 4
+// CHECK1-NEXT:    [[MUL41:%.*]] = mul nsw i32 [[DIV40]], 4
+// CHECK1-NEXT:    [[SUB42:%.*]] = sub nsw i32 [[SUB30]], [[MUL41]]
+// CHECK1-NEXT:    [[MUL43:%.*]] = mul nsw i32 [[SUB42]], 3
+// CHECK1-NEXT:    [[ADD44:%.*]] = add nsw i32 7, [[MUL43]]
+// CHECK1-NEXT:    store i32 [[ADD44]], ptr [[L]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_K]], align 4
+// CHECK1-NEXT:    [[MUL45:%.*]] = mul nsw i32 [[TMP21]], 3
+// CHECK1-NEXT:    [[ADD46:%.*]] = add nsw i32 7, [[MUL45]]
+// CHECK1-NEXT:    store i32 [[ADD46]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL47:%.*]] = mul nsw i32 [[TMP22]], 3
+// CHECK1-NEXT:    [[ADD48:%.*]] = add nsw i32 7, [[MUL47]]
+// CHECK1-NEXT:    store i32 [[ADD48]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[L]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP23]], i32 noundef [[TMP24]], i32 noundef [[TMP25]], i32 noundef [[TMP26]])
+// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1:       omp.body.continue:
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK1:       omp.inner.for.inc:
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD49:%.*]] = add nsw i32 [[TMP27]], 1
+// CHECK1-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK1:       omp.inner.for.end:
+// CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK1:       omp.loop.exit:
+// CHECK1-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define {{[^@]+}}@foo9
+// CHECK1-SAME: () #[[ATTR0]] {
+// CHECK1-NEXT:  entry:
+// CHECK1-NEXT:    [[ARR:%.*]] = alloca [128 x double], align 16
+// CHECK1-NEXT:    [[C:%.*]] = alloca double, align 8
+// CHECK1-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_1_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP0]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
+// CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP3]], ptr [[DOTCAPTURE_EXPR_3]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_3]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i64 [[SUB]], 1
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i64 [[ADD]], 1
+// CHECK1-NEXT:    [[SUB5:%.*]] = sub nsw i64 [[DIV]], 1
+// CHECK1-NEXT:    store i64 [[SUB5]], ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTPERMUTED_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK1:       for.cond:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP6]], 21
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 2
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// CHECK1-NEXT:    store i64 0, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
+// CHECK1:       for.cond7:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP9]], 1
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i64 [[TMP8]], [[ADD8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body10:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[MUL11:%.*]] = mul nsw i64 [[TMP11]], 1
+// CHECK1-NEXT:    [[ADD_PTR12:%.*]] = getelementptr inbounds double, ptr [[TMP10]], i64 [[MUL11]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR12]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP12]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load double, ptr [[C]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load double, ptr [[TMP14]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP13]], double noundef [[TMP15]], i32 noundef [[TMP16]])
+// CHECK1-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK1:       for.inc:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP17]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK1:       for.end:
+// CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
+// CHECK1:       for.inc13:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC14:%.*]] = add nsw i32 [[TMP18]], 1
+// CHECK1-NEXT:    store i32 [[INC14]], ptr [[DOTPERMUTED_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1:       for.end15:
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define {{[^@]+}}@foo10
+// CHECK1-SAME: () #[[ATTR0]] {
+// CHECK1-NEXT:  entry:
+// CHECK1-NEXT:    [[A:%.*]] = alloca [128 x double], align 16
+// CHECK1-NEXT:    [[B:%.*]] = alloca [16 x double], align 16
+// CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[_TMP2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[_TMP3:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[C:%.*]] = alloca double, align 8
+// CHECK1-NEXT:    [[__RANGE3:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END3:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN3:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[D:%.*]] = alloca double, align 8
+// CHECK1-NEXT:    [[__RANGE4:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END4:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN4:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_12:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_15:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_24:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_26:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_28:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_0_IV___BEGIN4:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTPERMUTED_1_IV___BEGIN3:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I37:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTPERMUTED_0_IV___BEGIN438:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTPERMUTED_1_IV___BEGIN339:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[J40:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[BB:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[AA:%.*]] = alloca double, align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+// CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
+// CHECK1-NEXT:    store ptr [[A]], ptr [[__RANGE3]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
+// CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END3]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY4:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY4]], ptr [[__BEGIN3]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY5:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP3]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY5]], ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[__END3]], align 8
+// CHECK1-NEXT:    store ptr [[TMP4]], ptr [[DOTCAPTURE_EXPR_6]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_6]], align 8
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP6]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i64 [[SUB]], 1
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i64 [[ADD]], 1
+// CHECK1-NEXT:    [[SUB8:%.*]] = sub nsw i64 [[DIV]], 1
+// CHECK1-NEXT:    store i64 [[SUB8]], ptr [[DOTCAPTURE_EXPR_7]], align 8
+// CHECK1-NEXT:    store double 4.200000e+01, ptr [[D]], align 8
+// CHECK1-NEXT:    store ptr [[B]], ptr [[__RANGE4]], align 8
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP7]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR10:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY9]], i64 16
+// CHECK1-NEXT:    store ptr [[ADD_PTR10]], ptr [[__END4]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY11:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP8]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY11]], ptr [[__BEGIN4]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY13:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY13]], ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__END4]], align 8
+// CHECK1-NEXT:    store ptr [[TMP10]], ptr [[DOTCAPTURE_EXPR_14]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_14]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST16:%.*]] = ptrtoint ptr [[TMP11]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST17:%.*]] = ptrtoint ptr [[TMP12]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB18:%.*]] = sub i64 [[SUB_PTR_LHS_CAST16]], [[SUB_PTR_RHS_CAST17]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV19:%.*]] = sdiv exact i64 [[SUB_PTR_SUB18]], 8
+// CHECK1-NEXT:    [[SUB20:%.*]] = sub nsw i64 [[SUB_PTR_DIV19]], 1
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i64 [[SUB20]], 1
+// CHECK1-NEXT:    [[DIV22:%.*]] = sdiv i64 [[ADD21]], 1
+// CHECK1-NEXT:    [[SUB23:%.*]] = sub nsw i64 [[DIV22]], 1
+// CHECK1-NEXT:    store i64 [[SUB23]], ptr [[DOTCAPTURE_EXPR_15]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_15]], align 8
+// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i64 [[TMP13]], 1
+// CHECK1-NEXT:    store i64 [[ADD25]], ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_7]], align 8
+// CHECK1-NEXT:    [[ADD27:%.*]] = add nsw i64 [[TMP14]], 1
+// CHECK1-NEXT:    store i64 [[ADD27]], ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB29:%.*]] = sub nsw i64 [[TMP15]], 0
+// CHECK1-NEXT:    [[DIV30:%.*]] = sdiv i64 [[SUB29]], 1
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 128, [[DIV30]]
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[TMP16]], 0
+// CHECK1-NEXT:    [[DIV32:%.*]] = sdiv i64 [[SUB31]], 1
+// CHECK1-NEXT:    [[MUL33:%.*]] = mul nsw i64 [[MUL]], [[DIV32]]
+// CHECK1-NEXT:    [[MUL34:%.*]] = mul nsw i64 [[MUL33]], 128
+// CHECK1-NEXT:    [[SUB35:%.*]] = sub nsw i64 [[MUL34]], 1
+// CHECK1-NEXT:    store i64 [[SUB35]], ptr [[DOTCAPTURE_EXPR_28]], align 8
+// CHECK1-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK1-NEXT:    store i64 0, ptr [[DOTPERMUTED_0_IV___BEGIN4]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTPERMUTED_1_IV___BEGIN3]], align 8
+// CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 0, [[TMP17]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK1:       land.lhs.true:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[CMP36:%.*]] = icmp slt i64 0, [[TMP18]]
+// CHECK1-NEXT:    br i1 [[CMP36]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1:       omp.precond.then:
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_28]], align 8
+// CHECK1-NEXT:    store i64 [[TMP19]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_28]], align 8
+// CHECK1-NEXT:    [[CMP41:%.*]] = icmp sgt i64 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    br i1 [[CMP41]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_28]], align 8
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP22]], [[COND_TRUE]] ], [ [[TMP23]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK1-NEXT:    store i64 [[TMP24]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK1:       omp.inner.for.cond:
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[CMP42:%.*]] = icmp sle i64 [[TMP25]], [[TMP26]]
+// CHECK1-NEXT:    br i1 [[CMP42]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1:       omp.inner.for.body:
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB43:%.*]] = sub nsw i64 [[TMP28]], 0
+// CHECK1-NEXT:    [[DIV44:%.*]] = sdiv i64 [[SUB43]], 1
+// CHECK1-NEXT:    [[MUL45:%.*]] = mul nsw i64 1, [[DIV44]]
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB46:%.*]] = sub nsw i64 [[TMP29]], 0
+// CHECK1-NEXT:    [[DIV47:%.*]] = sdiv i64 [[SUB46]], 1
+// CHECK1-NEXT:    [[MUL48:%.*]] = mul nsw i64 [[MUL45]], [[DIV47]]
+// CHECK1-NEXT:    [[MUL49:%.*]] = mul nsw i64 [[MUL48]], 128
+// CHECK1-NEXT:    [[DIV50:%.*]] = sdiv i64 [[TMP27]], [[MUL49]]
+// CHECK1-NEXT:    [[MUL51:%.*]] = mul nsw i64 [[DIV50]], 1
+// CHECK1-NEXT:    [[ADD52:%.*]] = add nsw i64 0, [[MUL51]]
+// CHECK1-NEXT:    [[CONV:%.*]] = trunc i64 [[ADD52]] to i32
+// CHECK1-NEXT:    store i32 [[CONV]], ptr [[I37]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB53:%.*]] = sub nsw i64 [[TMP32]], 0
+// CHECK1-NEXT:    [[DIV54:%.*]] = sdiv i64 [[SUB53]], 1
+// CHECK1-NEXT:    [[MUL55:%.*]] = mul nsw i64 1, [[DIV54]]
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB56:%.*]] = sub nsw i64 [[TMP33]], 0
+// CHECK1-NEXT:    [[DIV57:%.*]] = sdiv i64 [[SUB56]], 1
+// CHECK1-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[MUL55]], [[DIV57]]
+// CHECK1-NEXT:    [[MUL59:%.*]] = mul nsw i64 [[MUL58]], 128
+// CHECK1-NEXT:    [[DIV60:%.*]] = sdiv i64 [[TMP31]], [[MUL59]]
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB61:%.*]] = sub nsw i64 [[TMP34]], 0
+// CHECK1-NEXT:    [[DIV62:%.*]] = sdiv i64 [[SUB61]], 1
+// CHECK1-NEXT:    [[MUL63:%.*]] = mul nsw i64 1, [[DIV62]]
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB64:%.*]] = sub nsw i64 [[TMP35]], 0
+// CHECK1-NEXT:    [[DIV65:%.*]] = sdiv i64 [[SUB64]], 1
+// CHECK1-NEXT:    [[MUL66:%.*]] = mul nsw i64 [[MUL63]], [[DIV65]]
+// CHECK1-NEXT:    [[MUL67:%.*]] = mul nsw i64 [[MUL66]], 128
+// CHECK1-NEXT:    [[MUL68:%.*]] = mul nsw i64 [[DIV60]], [[MUL67]]
+// CHECK1-NEXT:    [[SUB69:%.*]] = sub nsw i64 [[TMP30]], [[MUL68]]
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB70:%.*]] = sub nsw i64 [[TMP36]], 0
+// CHECK1-NEXT:    [[DIV71:%.*]] = sdiv i64 [[SUB70]], 1
+// CHECK1-NEXT:    [[MUL72:%.*]] = mul nsw i64 1, [[DIV71]]
+// CHECK1-NEXT:    [[MUL73:%.*]] = mul nsw i64 [[MUL72]], 128
+// CHECK1-NEXT:    [[DIV74:%.*]] = sdiv i64 [[SUB69]], [[MUL73]]
+// CHECK1-NEXT:    [[MUL75:%.*]] = mul nsw i64 [[DIV74]], 1
+// CHECK1-NEXT:    [[ADD76:%.*]] = add nsw i64 0, [[MUL75]]
+// CHECK1-NEXT:    store i64 [[ADD76]], ptr [[DOTPERMUTED_0_IV___BEGIN438]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB77:%.*]] = sub nsw i64 [[TMP39]], 0
+// CHECK1-NEXT:    [[DIV78:%.*]] = sdiv i64 [[SUB77]], 1
+// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 1, [[DIV78]]
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP40]], 0
+// CHECK1-NEXT:    [[DIV81:%.*]] = sdiv i64 [[SUB80]], 1
+// CHECK1-NEXT:    [[MUL82:%.*]] = mul nsw i64 [[MUL79]], [[DIV81]]
+// CHECK1-NEXT:    [[MUL83:%.*]] = mul nsw i64 [[MUL82]], 128
+// CHECK1-NEXT:    [[DIV84:%.*]] = sdiv i64 [[TMP38]], [[MUL83]]
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB85:%.*]] = sub nsw i64 [[TMP41]], 0
+// CHECK1-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB85]], 1
+// CHECK1-NEXT:    [[MUL87:%.*]] = mul nsw i64 1, [[DIV86]]
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB88:%.*]] = sub nsw i64 [[TMP42]], 0
+// CHECK1-NEXT:    [[DIV89:%.*]] = sdiv i64 [[SUB88]], 1
+// CHECK1-NEXT:    [[MUL90:%.*]] = mul nsw i64 [[MUL87]], [[DIV89]]
+// CHECK1-NEXT:    [[MUL91:%.*]] = mul nsw i64 [[MUL90]], 128
+// CHECK1-NEXT:    [[MUL92:%.*]] = mul nsw i64 [[DIV84]], [[MUL91]]
+// CHECK1-NEXT:    [[SUB93:%.*]] = sub nsw i64 [[TMP37]], [[MUL92]]
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB94:%.*]] = sub nsw i64 [[TMP45]], 0
+// CHECK1-NEXT:    [[DIV95:%.*]] = sdiv i64 [[SUB94]], 1
+// CHECK1-NEXT:    [[MUL96:%.*]] = mul nsw i64 1, [[DIV95]]
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB97:%.*]] = sub nsw i64 [[TMP46]], 0
+// CHECK1-NEXT:    [[DIV98:%.*]] = sdiv i64 [[SUB97]], 1
+// CHECK1-NEXT:    [[MUL99:%.*]] = mul nsw i64 [[MUL96]], [[DIV98]]
+// CHECK1-NEXT:    [[MUL100:%.*]] = mul nsw i64 [[MUL99]], 128
+// CHECK1-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP44]], [[MUL100]]
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB102:%.*]] = sub nsw i64 [[TMP47]], 0
+// CHECK1-NEXT:    [[DIV103:%.*]] = sdiv i64 [[SUB102]], 1
+// CHECK1-NEXT:    [[MUL104:%.*]] = mul nsw i64 1, [[DIV103]]
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB105:%.*]] = sub nsw i64 [[TMP48]], 0
+// CHECK1-NEXT:    [[DIV106:%.*]] = sdiv i64 [[SUB105]], 1
+// CHECK1-NEXT:    [[MUL107:%.*]] = mul nsw i64 [[MUL104]], [[DIV106]]
+// CHECK1-NEXT:    [[MUL108:%.*]] = mul nsw i64 [[MUL107]], 128
+// CHECK1-NEXT:    [[MUL109:%.*]] = mul nsw i64 [[DIV101]], [[MUL108]]
+// CHECK1-NEXT:    [[SUB110:%.*]] = sub nsw i64 [[TMP43]], [[MUL109]]
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB111:%.*]] = sub nsw i64 [[TMP49]], 0
+// CHECK1-NEXT:    [[DIV112:%.*]] = sdiv i64 [[SUB111]], 1
+// CHECK1-NEXT:    [[MUL113:%.*]] = mul nsw i64 1, [[DIV112]]
+// CHECK1-NEXT:    [[MUL114:%.*]] = mul nsw i64 [[MUL113]], 128
+// CHECK1-NEXT:    [[DIV115:%.*]] = sdiv i64 [[SUB110]], [[MUL114]]
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB116:%.*]] = sub nsw i64 [[TMP50]], 0
+// CHECK1-NEXT:    [[DIV117:%.*]] = sdiv i64 [[SUB116]], 1
+// CHECK1-NEXT:    [[MUL118:%.*]] = mul nsw i64 1, [[DIV117]]
+// CHECK1-NEXT:    [[MUL119:%.*]] = mul nsw i64 [[MUL118]], 128
+// CHECK1-NEXT:    [[MUL120:%.*]] = mul nsw i64 [[DIV115]], [[MUL119]]
+// CHECK1-NEXT:    [[SUB121:%.*]] = sub nsw i64 [[SUB93]], [[MUL120]]
+// CHECK1-NEXT:    [[DIV122:%.*]] = sdiv i64 [[SUB121]], 128
+// CHECK1-NEXT:    [[MUL123:%.*]] = mul nsw i64 [[DIV122]], 1
+// CHECK1-NEXT:    [[ADD124:%.*]] = add nsw i64 0, [[MUL123]]
+// CHECK1-NEXT:    store i64 [[ADD124]], ptr [[DOTPERMUTED_1_IV___BEGIN339]], align 8
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB125:%.*]] = sub nsw i64 [[TMP53]], 0
+// CHECK1-NEXT:    [[DIV126:%.*]] = sdiv i64 [[SUB125]], 1
+// CHECK1-NEXT:    [[MUL127:%.*]] = mul nsw i64 1, [[DIV126]]
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB128:%.*]] = sub nsw i64 [[TMP54]], 0
+// CHECK1-NEXT:    [[DIV129:%.*]] = sdiv i64 [[SUB128]], 1
+// CHECK1-NEXT:    [[MUL130:%.*]] = mul nsw i64 [[MUL127]], [[DIV129]]
+// CHECK1-NEXT:    [[MUL131:%.*]] = mul nsw i64 [[MUL130]], 128
+// CHECK1-NEXT:    [[DIV132:%.*]] = sdiv i64 [[TMP52]], [[MUL131]]
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB133:%.*]] = sub nsw i64 [[TMP55]], 0
+// CHECK1-NEXT:    [[DIV134:%.*]] = sdiv i64 [[SUB133]], 1
+// CHECK1-NEXT:    [[MUL135:%.*]] = mul nsw i64 1, [[DIV134]]
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP56]], 0
+// CHECK1-NEXT:    [[DIV137:%.*]] = sdiv i64 [[SUB136]], 1
+// CHECK1-NEXT:    [[MUL138:%.*]] = mul nsw i64 [[MUL135]], [[DIV137]]
+// CHECK1-NEXT:    [[MUL139:%.*]] = mul nsw i64 [[MUL138]], 128
+// CHECK1-NEXT:    [[MUL140:%.*]] = mul nsw i64 [[DIV132]], [[MUL139]]
+// CHECK1-NEXT:    [[SUB141:%.*]] = sub nsw i64 [[TMP51]], [[MUL140]]
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB142:%.*]] = sub nsw i64 [[TMP59]], 0
+// CHECK1-NEXT:    [[DIV143:%.*]] = sdiv i64 [[SUB142]], 1
+// CHECK1-NEXT:    [[MUL144:%.*]] = mul nsw i64 1, [[DIV143]]
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB145:%.*]] = sub nsw i64 [[TMP60]], 0
+// CHECK1-NEXT:    [[DIV146:%.*]] = sdiv i64 [[SUB145]], 1
+// CHECK1-NEXT:    [[MUL147:%.*]] = mul nsw i64 [[MUL144]], [[DIV146]]
+// CHECK1-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[MUL147]], 128
+// CHECK1-NEXT:    [[DIV149:%.*]] = sdiv i64 [[TMP58]], [[MUL148]]
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB150:%.*]] = sub nsw i64 [[TMP61]], 0
+// CHECK1-NEXT:    [[DIV151:%.*]] = sdiv i64 [[SUB150]], 1
+// CHECK1-NEXT:    [[MUL152:%.*]] = mul nsw i64 1, [[DIV151]]
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB153:%.*]] = sub nsw i64 [[TMP62]], 0
+// CHECK1-NEXT:    [[DIV154:%.*]] = sdiv i64 [[SUB153]], 1
+// CHECK1-NEXT:    [[MUL155:%.*]] = mul nsw i64 [[MUL152]], [[DIV154]]
+// CHECK1-NEXT:    [[MUL156:%.*]] = mul nsw i64 [[MUL155]], 128
+// CHECK1-NEXT:    [[MUL157:%.*]] = mul nsw i64 [[DIV149]], [[MUL156]]
+// CHECK1-NEXT:    [[SUB158:%.*]] = sub nsw i64 [[TMP57]], [[MUL157]]
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB159:%.*]] = sub nsw i64 [[TMP63]], 0
+// CHECK1-NEXT:    [[DIV160:%.*]] = sdiv i64 [[SUB159]], 1
+// CHECK1-NEXT:    [[MUL161:%.*]] = mul nsw i64 1, [[DIV160]]
+// CHECK1-NEXT:    [[MUL162:%.*]] = mul nsw i64 [[MUL161]], 128
+// CHECK1-NEXT:    [[DIV163:%.*]] = sdiv i64 [[SUB158]], [[MUL162]]
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB164:%.*]] = sub nsw i64 [[TMP64]], 0
+// CHECK1-NEXT:    [[DIV165:%.*]] = sdiv i64 [[SUB164]], 1
+// CHECK1-NEXT:    [[MUL166:%.*]] = mul nsw i64 1, [[DIV165]]
+// CHECK1-NEXT:    [[MUL167:%.*]] = mul nsw i64 [[MUL166]], 128
+// CHECK1-NEXT:    [[MUL168:%.*]] = mul nsw i64 [[DIV163]], [[MUL167]]
+// CHECK1-NEXT:    [[SUB169:%.*]] = sub nsw i64 [[SUB141]], [[MUL168]]
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB170:%.*]] = sub nsw i64 [[TMP67]], 0
+// CHECK1-NEXT:    [[DIV171:%.*]] = sdiv i64 [[SUB170]], 1
+// CHECK1-NEXT:    [[MUL172:%.*]] = mul nsw i64 1, [[DIV171]]
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB173:%.*]] = sub nsw i64 [[TMP68]], 0
+// CHECK1-NEXT:    [[DIV174:%.*]] = sdiv i64 [[SUB173]], 1
+// CHECK1-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[MUL172]], [[DIV174]]
+// CHECK1-NEXT:    [[MUL176:%.*]] = mul nsw i64 [[MUL175]], 128
+// CHECK1-NEXT:    [[DIV177:%.*]] = sdiv i64 [[TMP66]], [[MUL176]]
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB178:%.*]] = sub nsw i64 [[TMP69]], 0
+// CHECK1-NEXT:    [[DIV179:%.*]] = sdiv i64 [[SUB178]], 1
+// CHECK1-NEXT:    [[MUL180:%.*]] = mul nsw i64 1, [[DIV179]]
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB181:%.*]] = sub nsw i64 [[TMP70]], 0
+// CHECK1-NEXT:    [[DIV182:%.*]] = sdiv i64 [[SUB181]], 1
+// CHECK1-NEXT:    [[MUL183:%.*]] = mul nsw i64 [[MUL180]], [[DIV182]]
+// CHECK1-NEXT:    [[MUL184:%.*]] = mul nsw i64 [[MUL183]], 128
+// CHECK1-NEXT:    [[MUL185:%.*]] = mul nsw i64 [[DIV177]], [[MUL184]]
+// CHECK1-NEXT:    [[SUB186:%.*]] = sub nsw i64 [[TMP65]], [[MUL185]]
+// CHECK1-NEXT:    [[TMP71:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP72:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP73:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB187:%.*]] = sub nsw i64 [[TMP73]], 0
+// CHECK1-NEXT:    [[DIV188:%.*]] = sdiv i64 [[SUB187]], 1
+// CHECK1-NEXT:    [[MUL189:%.*]] = mul nsw i64 1, [[DIV188]]
+// CHECK1-NEXT:    [[TMP74:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB190:%.*]] = sub nsw i64 [[TMP74]], 0
+// CHECK1-NEXT:    [[DIV191:%.*]] = sdiv i64 [[SUB190]], 1
+// CHECK1-NEXT:    [[MUL192:%.*]] = mul nsw i64 [[MUL189]], [[DIV191]]
+// CHECK1-NEXT:    [[MUL193:%.*]] = mul nsw i64 [[MUL192]], 128
+// CHECK1-NEXT:    [[DIV194:%.*]] = sdiv i64 [[TMP72]], [[MUL193]]
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK1-NEXT:    [[SUB195:%.*]] = sub nsw i64 [[TMP75]], 0
+// CHECK1-NEXT:    [[DIV196:%.*]] = sdiv i64 [[SUB195]], 1
+// CHECK1-NEXT:    [[MUL197:%.*]] = mul nsw i64 1, [[DIV196]]
+// CHECK1-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB198:%.*]] = sub nsw i64 [[TMP76]], 0
+// CHECK1-NEXT:    [[DIV199:%.*]] = sdiv i64 [[SUB198]], 1
+// CHECK1-NEXT:    [[MUL200:%.*]] = mul nsw i64 [[MUL197]], [[DIV199]]
+// CHECK1-NEXT:    [[MUL201:%.*]] = mul nsw i64 [[MUL200]], 128
+// CHECK1-NEXT:    [[MUL202:%.*]] = mul nsw i64 [[DIV194]], [[MUL201]]
+// CHECK1-NEXT:    [[SUB203:%.*]] = sub nsw i64 [[TMP71]], [[MUL202]]
+// CHECK1-NEXT:    [[TMP77:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB204:%.*]] = sub nsw i64 [[TMP77]], 0
+// CHECK1-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB204]], 1
+// CHECK1-NEXT:    [[MUL206:%.*]] = mul nsw i64 1, [[DIV205]]
+// CHECK1-NEXT:    [[MUL207:%.*]] = mul nsw i64 [[MUL206]], 128
+// CHECK1-NEXT:    [[DIV208:%.*]] = sdiv i64 [[SUB203]], [[MUL207]]
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK1-NEXT:    [[SUB209:%.*]] = sub nsw i64 [[TMP78]], 0
+// CHECK1-NEXT:    [[DIV210:%.*]] = sdiv i64 [[SUB209]], 1
+// CHECK1-NEXT:    [[MUL211:%.*]] = mul nsw i64 1, [[DIV210]]
+// CHECK1-NEXT:    [[MUL212:%.*]] = mul nsw i64 [[MUL211]], 128
+// CHECK1-NEXT:    [[MUL213:%.*]] = mul nsw i64 [[DIV208]], [[MUL212]]
+// CHECK1-NEXT:    [[SUB214:%.*]] = sub nsw i64 [[SUB186]], [[MUL213]]
+// CHECK1-NEXT:    [[DIV215:%.*]] = sdiv i64 [[SUB214]], 128
+// CHECK1-NEXT:    [[MUL216:%.*]] = mul nsw i64 [[DIV215]], 128
+// CHECK1-NEXT:    [[SUB217:%.*]] = sub nsw i64 [[SUB169]], [[MUL216]]
+// CHECK1-NEXT:    [[MUL218:%.*]] = mul nsw i64 [[SUB217]], 1
+// CHECK1-NEXT:    [[ADD219:%.*]] = add nsw i64 0, [[MUL218]]
+// CHECK1-NEXT:    [[CONV220:%.*]] = trunc i64 [[ADD219]] to i32
+// CHECK1-NEXT:    store i32 [[CONV220]], ptr [[J40]], align 4
+// CHECK1-NEXT:    [[TMP79:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK1-NEXT:    [[TMP80:%.*]] = load i64, ptr [[DOTPERMUTED_0_IV___BEGIN438]], align 8
+// CHECK1-NEXT:    [[MUL221:%.*]] = mul nsw i64 [[TMP80]], 1
+// CHECK1-NEXT:    [[ADD_PTR222:%.*]] = getelementptr inbounds double, ptr [[TMP79]], i64 [[MUL221]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR222]], ptr [[__BEGIN4]], align 8
+// CHECK1-NEXT:    [[TMP81:%.*]] = load ptr, ptr [[__BEGIN4]], align 8
+// CHECK1-NEXT:    store ptr [[TMP81]], ptr [[BB]], align 8
+// CHECK1-NEXT:    [[TMP82:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP83:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN339]], align 8
+// CHECK1-NEXT:    [[MUL223:%.*]] = mul nsw i64 [[TMP83]], 1
+// CHECK1-NEXT:    [[ADD_PTR224:%.*]] = getelementptr inbounds double, ptr [[TMP82]], i64 [[MUL223]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR224]], ptr [[__BEGIN3]], align 8
+// CHECK1-NEXT:    [[TMP84:%.*]] = load ptr, ptr [[__BEGIN3]], align 8
+// CHECK1-NEXT:    [[TMP85:%.*]] = load double, ptr [[TMP84]], align 8
+// CHECK1-NEXT:    store double [[TMP85]], ptr [[AA]], align 8
+// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[I37]], align 4
+// CHECK1-NEXT:    [[TMP87:%.*]] = load double, ptr [[C]], align 8
+// CHECK1-NEXT:    [[TMP88:%.*]] = load double, ptr [[AA]], align 8
+// CHECK1-NEXT:    [[TMP89:%.*]] = load double, ptr [[D]], align 8
+// CHECK1-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[BB]], align 8
+// CHECK1-NEXT:    [[TMP91:%.*]] = load double, ptr [[TMP90]], align 8
+// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[J40]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP86]], double noundef [[TMP87]], double noundef [[TMP88]], double noundef [[TMP89]], double noundef [[TMP91]], i32 noundef [[TMP92]])
+// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1:       omp.body.continue:
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK1:       omp.inner.for.inc:
+// CHECK1-NEXT:    [[TMP93:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD225:%.*]] = add nsw i64 [[TMP93]], 1
+// CHECK1-NEXT:    store i64 [[ADD225]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK1:       omp.inner.for.end:
+// CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK1:       omp.loop.exit:
+// CHECK1-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// CHECK1-NEXT:    br label [[OMP_PRECOND_END]]
+// CHECK1:       omp.precond.end:
+// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@body
+// CHECK2-SAME: (...) #[[ATTR0:[0-9]+]] {
+// CHECK2-NEXT:  entry:
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@foo10
+// CHECK2-SAME: () #[[ATTR0]] {
+// CHECK2-NEXT:  entry:
+// CHECK2-NEXT:    [[A:%.*]] = alloca [128 x double], align 16
+// CHECK2-NEXT:    [[B:%.*]] = alloca [16 x double], align 16
+// CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[_TMP2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[_TMP3:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[C:%.*]] = alloca double, align 8
+// CHECK2-NEXT:    [[__RANGE3:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END3:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN3:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[D:%.*]] = alloca double, align 8
+// CHECK2-NEXT:    [[__RANGE4:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END4:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN4:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_12:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_15:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_24:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_26:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_28:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_0_IV___BEGIN4:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTPERMUTED_1_IV___BEGIN3:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I37:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_0_IV___BEGIN438:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTPERMUTED_1_IV___BEGIN339:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[J40:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[BB:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[AA:%.*]] = alloca double, align 8
+// CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
+// CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
+// CHECK2-NEXT:    store ptr [[A]], ptr [[__RANGE3]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
+// CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END3]], align 8
+// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY4:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY4]], ptr [[__BEGIN3]], align 8
+// CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY5:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP3]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY5]], ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[__END3]], align 8
+// CHECK2-NEXT:    store ptr [[TMP4]], ptr [[DOTCAPTURE_EXPR_6]], align 8
+// CHECK2-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_6]], align 8
+// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP6]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i64 [[SUB]], 1
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i64 [[ADD]], 1
+// CHECK2-NEXT:    [[SUB8:%.*]] = sub nsw i64 [[DIV]], 1
+// CHECK2-NEXT:    store i64 [[SUB8]], ptr [[DOTCAPTURE_EXPR_7]], align 8
+// CHECK2-NEXT:    store double 4.200000e+01, ptr [[D]], align 8
+// CHECK2-NEXT:    store ptr [[B]], ptr [[__RANGE4]], align 8
+// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP7]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR10:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY9]], i64 16
+// CHECK2-NEXT:    store ptr [[ADD_PTR10]], ptr [[__END4]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY11:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP8]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY11]], ptr [[__BEGIN4]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY13:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY13]], ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__END4]], align 8
+// CHECK2-NEXT:    store ptr [[TMP10]], ptr [[DOTCAPTURE_EXPR_14]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_14]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST16:%.*]] = ptrtoint ptr [[TMP11]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST17:%.*]] = ptrtoint ptr [[TMP12]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB18:%.*]] = sub i64 [[SUB_PTR_LHS_CAST16]], [[SUB_PTR_RHS_CAST17]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV19:%.*]] = sdiv exact i64 [[SUB_PTR_SUB18]], 8
+// CHECK2-NEXT:    [[SUB20:%.*]] = sub nsw i64 [[SUB_PTR_DIV19]], 1
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i64 [[SUB20]], 1
+// CHECK2-NEXT:    [[DIV22:%.*]] = sdiv i64 [[ADD21]], 1
+// CHECK2-NEXT:    [[SUB23:%.*]] = sub nsw i64 [[DIV22]], 1
+// CHECK2-NEXT:    store i64 [[SUB23]], ptr [[DOTCAPTURE_EXPR_15]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_15]], align 8
+// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i64 [[TMP13]], 1
+// CHECK2-NEXT:    store i64 [[ADD25]], ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_7]], align 8
+// CHECK2-NEXT:    [[ADD27:%.*]] = add nsw i64 [[TMP14]], 1
+// CHECK2-NEXT:    store i64 [[ADD27]], ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB29:%.*]] = sub nsw i64 [[TMP15]], 0
+// CHECK2-NEXT:    [[DIV30:%.*]] = sdiv i64 [[SUB29]], 1
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 128, [[DIV30]]
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[TMP16]], 0
+// CHECK2-NEXT:    [[DIV32:%.*]] = sdiv i64 [[SUB31]], 1
+// CHECK2-NEXT:    [[MUL33:%.*]] = mul nsw i64 [[MUL]], [[DIV32]]
+// CHECK2-NEXT:    [[MUL34:%.*]] = mul nsw i64 [[MUL33]], 128
+// CHECK2-NEXT:    [[SUB35:%.*]] = sub nsw i64 [[MUL34]], 1
+// CHECK2-NEXT:    store i64 [[SUB35]], ptr [[DOTCAPTURE_EXPR_28]], align 8
+// CHECK2-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK2-NEXT:    store i64 0, ptr [[DOTPERMUTED_0_IV___BEGIN4]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTPERMUTED_1_IV___BEGIN3]], align 8
+// CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 0, [[TMP17]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK2:       land.lhs.true:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[CMP36:%.*]] = icmp slt i64 0, [[TMP18]]
+// CHECK2-NEXT:    br i1 [[CMP36]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2:       omp.precond.then:
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_28]], align 8
+// CHECK2-NEXT:    store i64 [[TMP19]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_28]], align 8
+// CHECK2-NEXT:    [[CMP41:%.*]] = icmp sgt i64 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    br i1 [[CMP41]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_28]], align 8
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP22]], [[COND_TRUE]] ], [ [[TMP23]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK2-NEXT:    store i64 [[TMP24]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK2:       omp.inner.for.cond:
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[CMP42:%.*]] = icmp sle i64 [[TMP25]], [[TMP26]]
+// CHECK2-NEXT:    br i1 [[CMP42]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2:       omp.inner.for.body:
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB43:%.*]] = sub nsw i64 [[TMP28]], 0
+// CHECK2-NEXT:    [[DIV44:%.*]] = sdiv i64 [[SUB43]], 1
+// CHECK2-NEXT:    [[MUL45:%.*]] = mul nsw i64 1, [[DIV44]]
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB46:%.*]] = sub nsw i64 [[TMP29]], 0
+// CHECK2-NEXT:    [[DIV47:%.*]] = sdiv i64 [[SUB46]], 1
+// CHECK2-NEXT:    [[MUL48:%.*]] = mul nsw i64 [[MUL45]], [[DIV47]]
+// CHECK2-NEXT:    [[MUL49:%.*]] = mul nsw i64 [[MUL48]], 128
+// CHECK2-NEXT:    [[DIV50:%.*]] = sdiv i64 [[TMP27]], [[MUL49]]
+// CHECK2-NEXT:    [[MUL51:%.*]] = mul nsw i64 [[DIV50]], 1
+// CHECK2-NEXT:    [[ADD52:%.*]] = add nsw i64 0, [[MUL51]]
+// CHECK2-NEXT:    [[CONV:%.*]] = trunc i64 [[ADD52]] to i32
+// CHECK2-NEXT:    store i32 [[CONV]], ptr [[I37]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB53:%.*]] = sub nsw i64 [[TMP32]], 0
+// CHECK2-NEXT:    [[DIV54:%.*]] = sdiv i64 [[SUB53]], 1
+// CHECK2-NEXT:    [[MUL55:%.*]] = mul nsw i64 1, [[DIV54]]
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB56:%.*]] = sub nsw i64 [[TMP33]], 0
+// CHECK2-NEXT:    [[DIV57:%.*]] = sdiv i64 [[SUB56]], 1
+// CHECK2-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[MUL55]], [[DIV57]]
+// CHECK2-NEXT:    [[MUL59:%.*]] = mul nsw i64 [[MUL58]], 128
+// CHECK2-NEXT:    [[DIV60:%.*]] = sdiv i64 [[TMP31]], [[MUL59]]
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB61:%.*]] = sub nsw i64 [[TMP34]], 0
+// CHECK2-NEXT:    [[DIV62:%.*]] = sdiv i64 [[SUB61]], 1
+// CHECK2-NEXT:    [[MUL63:%.*]] = mul nsw i64 1, [[DIV62]]
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB64:%.*]] = sub nsw i64 [[TMP35]], 0
+// CHECK2-NEXT:    [[DIV65:%.*]] = sdiv i64 [[SUB64]], 1
+// CHECK2-NEXT:    [[MUL66:%.*]] = mul nsw i64 [[MUL63]], [[DIV65]]
+// CHECK2-NEXT:    [[MUL67:%.*]] = mul nsw i64 [[MUL66]], 128
+// CHECK2-NEXT:    [[MUL68:%.*]] = mul nsw i64 [[DIV60]], [[MUL67]]
+// CHECK2-NEXT:    [[SUB69:%.*]] = sub nsw i64 [[TMP30]], [[MUL68]]
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB70:%.*]] = sub nsw i64 [[TMP36]], 0
+// CHECK2-NEXT:    [[DIV71:%.*]] = sdiv i64 [[SUB70]], 1
+// CHECK2-NEXT:    [[MUL72:%.*]] = mul nsw i64 1, [[DIV71]]
+// CHECK2-NEXT:    [[MUL73:%.*]] = mul nsw i64 [[MUL72]], 128
+// CHECK2-NEXT:    [[DIV74:%.*]] = sdiv i64 [[SUB69]], [[MUL73]]
+// CHECK2-NEXT:    [[MUL75:%.*]] = mul nsw i64 [[DIV74]], 1
+// CHECK2-NEXT:    [[ADD76:%.*]] = add nsw i64 0, [[MUL75]]
+// CHECK2-NEXT:    store i64 [[ADD76]], ptr [[DOTPERMUTED_0_IV___BEGIN438]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB77:%.*]] = sub nsw i64 [[TMP39]], 0
+// CHECK2-NEXT:    [[DIV78:%.*]] = sdiv i64 [[SUB77]], 1
+// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 1, [[DIV78]]
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP40]], 0
+// CHECK2-NEXT:    [[DIV81:%.*]] = sdiv i64 [[SUB80]], 1
+// CHECK2-NEXT:    [[MUL82:%.*]] = mul nsw i64 [[MUL79]], [[DIV81]]
+// CHECK2-NEXT:    [[MUL83:%.*]] = mul nsw i64 [[MUL82]], 128
+// CHECK2-NEXT:    [[DIV84:%.*]] = sdiv i64 [[TMP38]], [[MUL83]]
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB85:%.*]] = sub nsw i64 [[TMP41]], 0
+// CHECK2-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB85]], 1
+// CHECK2-NEXT:    [[MUL87:%.*]] = mul nsw i64 1, [[DIV86]]
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB88:%.*]] = sub nsw i64 [[TMP42]], 0
+// CHECK2-NEXT:    [[DIV89:%.*]] = sdiv i64 [[SUB88]], 1
+// CHECK2-NEXT:    [[MUL90:%.*]] = mul nsw i64 [[MUL87]], [[DIV89]]
+// CHECK2-NEXT:    [[MUL91:%.*]] = mul nsw i64 [[MUL90]], 128
+// CHECK2-NEXT:    [[MUL92:%.*]] = mul nsw i64 [[DIV84]], [[MUL91]]
+// CHECK2-NEXT:    [[SUB93:%.*]] = sub nsw i64 [[TMP37]], [[MUL92]]
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB94:%.*]] = sub nsw i64 [[TMP45]], 0
+// CHECK2-NEXT:    [[DIV95:%.*]] = sdiv i64 [[SUB94]], 1
+// CHECK2-NEXT:    [[MUL96:%.*]] = mul nsw i64 1, [[DIV95]]
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB97:%.*]] = sub nsw i64 [[TMP46]], 0
+// CHECK2-NEXT:    [[DIV98:%.*]] = sdiv i64 [[SUB97]], 1
+// CHECK2-NEXT:    [[MUL99:%.*]] = mul nsw i64 [[MUL96]], [[DIV98]]
+// CHECK2-NEXT:    [[MUL100:%.*]] = mul nsw i64 [[MUL99]], 128
+// CHECK2-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP44]], [[MUL100]]
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB102:%.*]] = sub nsw i64 [[TMP47]], 0
+// CHECK2-NEXT:    [[DIV103:%.*]] = sdiv i64 [[SUB102]], 1
+// CHECK2-NEXT:    [[MUL104:%.*]] = mul nsw i64 1, [[DIV103]]
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB105:%.*]] = sub nsw i64 [[TMP48]], 0
+// CHECK2-NEXT:    [[DIV106:%.*]] = sdiv i64 [[SUB105]], 1
+// CHECK2-NEXT:    [[MUL107:%.*]] = mul nsw i64 [[MUL104]], [[DIV106]]
+// CHECK2-NEXT:    [[MUL108:%.*]] = mul nsw i64 [[MUL107]], 128
+// CHECK2-NEXT:    [[MUL109:%.*]] = mul nsw i64 [[DIV101]], [[MUL108]]
+// CHECK2-NEXT:    [[SUB110:%.*]] = sub nsw i64 [[TMP43]], [[MUL109]]
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB111:%.*]] = sub nsw i64 [[TMP49]], 0
+// CHECK2-NEXT:    [[DIV112:%.*]] = sdiv i64 [[SUB111]], 1
+// CHECK2-NEXT:    [[MUL113:%.*]] = mul nsw i64 1, [[DIV112]]
+// CHECK2-NEXT:    [[MUL114:%.*]] = mul nsw i64 [[MUL113]], 128
+// CHECK2-NEXT:    [[DIV115:%.*]] = sdiv i64 [[SUB110]], [[MUL114]]
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB116:%.*]] = sub nsw i64 [[TMP50]], 0
+// CHECK2-NEXT:    [[DIV117:%.*]] = sdiv i64 [[SUB116]], 1
+// CHECK2-NEXT:    [[MUL118:%.*]] = mul nsw i64 1, [[DIV117]]
+// CHECK2-NEXT:    [[MUL119:%.*]] = mul nsw i64 [[MUL118]], 128
+// CHECK2-NEXT:    [[MUL120:%.*]] = mul nsw i64 [[DIV115]], [[MUL119]]
+// CHECK2-NEXT:    [[SUB121:%.*]] = sub nsw i64 [[SUB93]], [[MUL120]]
+// CHECK2-NEXT:    [[DIV122:%.*]] = sdiv i64 [[SUB121]], 128
+// CHECK2-NEXT:    [[MUL123:%.*]] = mul nsw i64 [[DIV122]], 1
+// CHECK2-NEXT:    [[ADD124:%.*]] = add nsw i64 0, [[MUL123]]
+// CHECK2-NEXT:    store i64 [[ADD124]], ptr [[DOTPERMUTED_1_IV___BEGIN339]], align 8
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB125:%.*]] = sub nsw i64 [[TMP53]], 0
+// CHECK2-NEXT:    [[DIV126:%.*]] = sdiv i64 [[SUB125]], 1
+// CHECK2-NEXT:    [[MUL127:%.*]] = mul nsw i64 1, [[DIV126]]
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB128:%.*]] = sub nsw i64 [[TMP54]], 0
+// CHECK2-NEXT:    [[DIV129:%.*]] = sdiv i64 [[SUB128]], 1
+// CHECK2-NEXT:    [[MUL130:%.*]] = mul nsw i64 [[MUL127]], [[DIV129]]
+// CHECK2-NEXT:    [[MUL131:%.*]] = mul nsw i64 [[MUL130]], 128
+// CHECK2-NEXT:    [[DIV132:%.*]] = sdiv i64 [[TMP52]], [[MUL131]]
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB133:%.*]] = sub nsw i64 [[TMP55]], 0
+// CHECK2-NEXT:    [[DIV134:%.*]] = sdiv i64 [[SUB133]], 1
+// CHECK2-NEXT:    [[MUL135:%.*]] = mul nsw i64 1, [[DIV134]]
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP56]], 0
+// CHECK2-NEXT:    [[DIV137:%.*]] = sdiv i64 [[SUB136]], 1
+// CHECK2-NEXT:    [[MUL138:%.*]] = mul nsw i64 [[MUL135]], [[DIV137]]
+// CHECK2-NEXT:    [[MUL139:%.*]] = mul nsw i64 [[MUL138]], 128
+// CHECK2-NEXT:    [[MUL140:%.*]] = mul nsw i64 [[DIV132]], [[MUL139]]
+// CHECK2-NEXT:    [[SUB141:%.*]] = sub nsw i64 [[TMP51]], [[MUL140]]
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB142:%.*]] = sub nsw i64 [[TMP59]], 0
+// CHECK2-NEXT:    [[DIV143:%.*]] = sdiv i64 [[SUB142]], 1
+// CHECK2-NEXT:    [[MUL144:%.*]] = mul nsw i64 1, [[DIV143]]
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB145:%.*]] = sub nsw i64 [[TMP60]], 0
+// CHECK2-NEXT:    [[DIV146:%.*]] = sdiv i64 [[SUB145]], 1
+// CHECK2-NEXT:    [[MUL147:%.*]] = mul nsw i64 [[MUL144]], [[DIV146]]
+// CHECK2-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[MUL147]], 128
+// CHECK2-NEXT:    [[DIV149:%.*]] = sdiv i64 [[TMP58]], [[MUL148]]
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB150:%.*]] = sub nsw i64 [[TMP61]], 0
+// CHECK2-NEXT:    [[DIV151:%.*]] = sdiv i64 [[SUB150]], 1
+// CHECK2-NEXT:    [[MUL152:%.*]] = mul nsw i64 1, [[DIV151]]
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB153:%.*]] = sub nsw i64 [[TMP62]], 0
+// CHECK2-NEXT:    [[DIV154:%.*]] = sdiv i64 [[SUB153]], 1
+// CHECK2-NEXT:    [[MUL155:%.*]] = mul nsw i64 [[MUL152]], [[DIV154]]
+// CHECK2-NEXT:    [[MUL156:%.*]] = mul nsw i64 [[MUL155]], 128
+// CHECK2-NEXT:    [[MUL157:%.*]] = mul nsw i64 [[DIV149]], [[MUL156]]
+// CHECK2-NEXT:    [[SUB158:%.*]] = sub nsw i64 [[TMP57]], [[MUL157]]
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB159:%.*]] = sub nsw i64 [[TMP63]], 0
+// CHECK2-NEXT:    [[DIV160:%.*]] = sdiv i64 [[SUB159]], 1
+// CHECK2-NEXT:    [[MUL161:%.*]] = mul nsw i64 1, [[DIV160]]
+// CHECK2-NEXT:    [[MUL162:%.*]] = mul nsw i64 [[MUL161]], 128
+// CHECK2-NEXT:    [[DIV163:%.*]] = sdiv i64 [[SUB158]], [[MUL162]]
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB164:%.*]] = sub nsw i64 [[TMP64]], 0
+// CHECK2-NEXT:    [[DIV165:%.*]] = sdiv i64 [[SUB164]], 1
+// CHECK2-NEXT:    [[MUL166:%.*]] = mul nsw i64 1, [[DIV165]]
+// CHECK2-NEXT:    [[MUL167:%.*]] = mul nsw i64 [[MUL166]], 128
+// CHECK2-NEXT:    [[MUL168:%.*]] = mul nsw i64 [[DIV163]], [[MUL167]]
+// CHECK2-NEXT:    [[SUB169:%.*]] = sub nsw i64 [[SUB141]], [[MUL168]]
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB170:%.*]] = sub nsw i64 [[TMP67]], 0
+// CHECK2-NEXT:    [[DIV171:%.*]] = sdiv i64 [[SUB170]], 1
+// CHECK2-NEXT:    [[MUL172:%.*]] = mul nsw i64 1, [[DIV171]]
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB173:%.*]] = sub nsw i64 [[TMP68]], 0
+// CHECK2-NEXT:    [[DIV174:%.*]] = sdiv i64 [[SUB173]], 1
+// CHECK2-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[MUL172]], [[DIV174]]
+// CHECK2-NEXT:    [[MUL176:%.*]] = mul nsw i64 [[MUL175]], 128
+// CHECK2-NEXT:    [[DIV177:%.*]] = sdiv i64 [[TMP66]], [[MUL176]]
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB178:%.*]] = sub nsw i64 [[TMP69]], 0
+// CHECK2-NEXT:    [[DIV179:%.*]] = sdiv i64 [[SUB178]], 1
+// CHECK2-NEXT:    [[MUL180:%.*]] = mul nsw i64 1, [[DIV179]]
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB181:%.*]] = sub nsw i64 [[TMP70]], 0
+// CHECK2-NEXT:    [[DIV182:%.*]] = sdiv i64 [[SUB181]], 1
+// CHECK2-NEXT:    [[MUL183:%.*]] = mul nsw i64 [[MUL180]], [[DIV182]]
+// CHECK2-NEXT:    [[MUL184:%.*]] = mul nsw i64 [[MUL183]], 128
+// CHECK2-NEXT:    [[MUL185:%.*]] = mul nsw i64 [[DIV177]], [[MUL184]]
+// CHECK2-NEXT:    [[SUB186:%.*]] = sub nsw i64 [[TMP65]], [[MUL185]]
+// CHECK2-NEXT:    [[TMP71:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP72:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP73:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB187:%.*]] = sub nsw i64 [[TMP73]], 0
+// CHECK2-NEXT:    [[DIV188:%.*]] = sdiv i64 [[SUB187]], 1
+// CHECK2-NEXT:    [[MUL189:%.*]] = mul nsw i64 1, [[DIV188]]
+// CHECK2-NEXT:    [[TMP74:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB190:%.*]] = sub nsw i64 [[TMP74]], 0
+// CHECK2-NEXT:    [[DIV191:%.*]] = sdiv i64 [[SUB190]], 1
+// CHECK2-NEXT:    [[MUL192:%.*]] = mul nsw i64 [[MUL189]], [[DIV191]]
+// CHECK2-NEXT:    [[MUL193:%.*]] = mul nsw i64 [[MUL192]], 128
+// CHECK2-NEXT:    [[DIV194:%.*]] = sdiv i64 [[TMP72]], [[MUL193]]
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_24]], align 8
+// CHECK2-NEXT:    [[SUB195:%.*]] = sub nsw i64 [[TMP75]], 0
+// CHECK2-NEXT:    [[DIV196:%.*]] = sdiv i64 [[SUB195]], 1
+// CHECK2-NEXT:    [[MUL197:%.*]] = mul nsw i64 1, [[DIV196]]
+// CHECK2-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB198:%.*]] = sub nsw i64 [[TMP76]], 0
+// CHECK2-NEXT:    [[DIV199:%.*]] = sdiv i64 [[SUB198]], 1
+// CHECK2-NEXT:    [[MUL200:%.*]] = mul nsw i64 [[MUL197]], [[DIV199]]
+// CHECK2-NEXT:    [[MUL201:%.*]] = mul nsw i64 [[MUL200]], 128
+// CHECK2-NEXT:    [[MUL202:%.*]] = mul nsw i64 [[DIV194]], [[MUL201]]
+// CHECK2-NEXT:    [[SUB203:%.*]] = sub nsw i64 [[TMP71]], [[MUL202]]
+// CHECK2-NEXT:    [[TMP77:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB204:%.*]] = sub nsw i64 [[TMP77]], 0
+// CHECK2-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB204]], 1
+// CHECK2-NEXT:    [[MUL206:%.*]] = mul nsw i64 1, [[DIV205]]
+// CHECK2-NEXT:    [[MUL207:%.*]] = mul nsw i64 [[MUL206]], 128
+// CHECK2-NEXT:    [[DIV208:%.*]] = sdiv i64 [[SUB203]], [[MUL207]]
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_26]], align 8
+// CHECK2-NEXT:    [[SUB209:%.*]] = sub nsw i64 [[TMP78]], 0
+// CHECK2-NEXT:    [[DIV210:%.*]] = sdiv i64 [[SUB209]], 1
+// CHECK2-NEXT:    [[MUL211:%.*]] = mul nsw i64 1, [[DIV210]]
+// CHECK2-NEXT:    [[MUL212:%.*]] = mul nsw i64 [[MUL211]], 128
+// CHECK2-NEXT:    [[MUL213:%.*]] = mul nsw i64 [[DIV208]], [[MUL212]]
+// CHECK2-NEXT:    [[SUB214:%.*]] = sub nsw i64 [[SUB186]], [[MUL213]]
+// CHECK2-NEXT:    [[DIV215:%.*]] = sdiv i64 [[SUB214]], 128
+// CHECK2-NEXT:    [[MUL216:%.*]] = mul nsw i64 [[DIV215]], 128
+// CHECK2-NEXT:    [[SUB217:%.*]] = sub nsw i64 [[SUB169]], [[MUL216]]
+// CHECK2-NEXT:    [[MUL218:%.*]] = mul nsw i64 [[SUB217]], 1
+// CHECK2-NEXT:    [[ADD219:%.*]] = add nsw i64 0, [[MUL218]]
+// CHECK2-NEXT:    [[CONV220:%.*]] = trunc i64 [[ADD219]] to i32
+// CHECK2-NEXT:    store i32 [[CONV220]], ptr [[J40]], align 4
+// CHECK2-NEXT:    [[TMP79:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK2-NEXT:    [[TMP80:%.*]] = load i64, ptr [[DOTPERMUTED_0_IV___BEGIN438]], align 8
+// CHECK2-NEXT:    [[MUL221:%.*]] = mul nsw i64 [[TMP80]], 1
+// CHECK2-NEXT:    [[ADD_PTR222:%.*]] = getelementptr inbounds double, ptr [[TMP79]], i64 [[MUL221]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR222]], ptr [[__BEGIN4]], align 8
+// CHECK2-NEXT:    [[TMP81:%.*]] = load ptr, ptr [[__BEGIN4]], align 8
+// CHECK2-NEXT:    store ptr [[TMP81]], ptr [[BB]], align 8
+// CHECK2-NEXT:    [[TMP82:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP83:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN339]], align 8
+// CHECK2-NEXT:    [[MUL223:%.*]] = mul nsw i64 [[TMP83]], 1
+// CHECK2-NEXT:    [[ADD_PTR224:%.*]] = getelementptr inbounds double, ptr [[TMP82]], i64 [[MUL223]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR224]], ptr [[__BEGIN3]], align 8
+// CHECK2-NEXT:    [[TMP84:%.*]] = load ptr, ptr [[__BEGIN3]], align 8
+// CHECK2-NEXT:    [[TMP85:%.*]] = load double, ptr [[TMP84]], align 8
+// CHECK2-NEXT:    store double [[TMP85]], ptr [[AA]], align 8
+// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[I37]], align 4
+// CHECK2-NEXT:    [[TMP87:%.*]] = load double, ptr [[C]], align 8
+// CHECK2-NEXT:    [[TMP88:%.*]] = load double, ptr [[AA]], align 8
+// CHECK2-NEXT:    [[TMP89:%.*]] = load double, ptr [[D]], align 8
+// CHECK2-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[BB]], align 8
+// CHECK2-NEXT:    [[TMP91:%.*]] = load double, ptr [[TMP90]], align 8
+// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[J40]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP86]], double noundef [[TMP87]], double noundef [[TMP88]], double noundef [[TMP89]], double noundef [[TMP91]], i32 noundef [[TMP92]])
+// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2:       omp.body.continue:
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK2:       omp.inner.for.inc:
+// CHECK2-NEXT:    [[TMP93:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD225:%.*]] = add nsw i64 [[TMP93]], 1
+// CHECK2-NEXT:    store i64 [[ADD225]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK2:       omp.inner.for.end:
+// CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK2:       omp.loop.exit:
+// CHECK2-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// CHECK2-NEXT:    br label [[OMP_PRECOND_END]]
+// CHECK2:       omp.precond.end:
+// CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@foo2
+// CHECK2-SAME: (i32 noundef [[START1:%.*]], i32 noundef [[START2:%.*]], i32 noundef [[END1:%.*]], i32 noundef [[END2:%.*]], i32 noundef [[STEP1:%.*]], i32 noundef [[STEP2:%.*]]) #[[ATTR0]] {
+// CHECK2-NEXT:  entry:
+// CHECK2-NEXT:    [[START1_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[START2_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[END1_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[END2_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[STEP1_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[STEP2_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP7:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_8:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_0_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_1_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    store i32 [[START1]], ptr [[START1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[START2]], ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[END1]], ptr [[END1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[END2]], ptr [[END2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[STEP1]], ptr [[STEP1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[STEP2]], ptr [[STEP2_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[END1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[STEP1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP6]]
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
+// CHECK2-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// CHECK2-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP9]], ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[END2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB9:%.*]] = sub i32 [[TMP12]], [[TMP13]]
+// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[SUB9]], 1
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[SUB10]], [[TMP14]]
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV12:%.*]] = udiv i32 [[ADD11]], [[TMP15]]
+// CHECK2-NEXT:    [[SUB13:%.*]] = sub i32 [[DIV12]], 1
+// CHECK2-NEXT:    store i32 [[SUB13]], ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK2:       for.cond:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP17]], 1
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP16]], [[ADD14]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP19]], [[TMP20]]
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP18]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND16:%.*]]
+// CHECK2:       for.cond16:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD17:%.*]] = add i32 [[TMP22]], 1
+// CHECK2-NEXT:    [[CMP18:%.*]] = icmp ult i32 [[TMP21]], [[ADD17]]
+// CHECK2-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body19:
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL20:%.*]] = mul i32 [[TMP24]], [[TMP25]]
+// CHECK2-NEXT:    [[ADD21:%.*]] = add i32 [[TMP23]], [[MUL20]]
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP26]], i32 noundef [[TMP27]])
+// CHECK2-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK2:       for.inc:
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP28]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND16]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK2:       for.end:
+// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK2:       for.inc22:
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK2-NEXT:    [[INC23:%.*]] = add i32 [[TMP29]], 1
+// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK2:       for.end24:
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@foo3
+// CHECK2-SAME: () #[[ATTR0]] {
+// CHECK2-NEXT:  entry:
+// CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_0_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_1_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+// CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 3, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK2-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 3
+// CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 3, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK2:       omp.inner.for.cond:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP1:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    br i1 [[CMP1]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2:       omp.inner.for.body:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL2:%.*]] = mul nsw i32 [[TMP7]], 3
+// CHECK2-NEXT:    [[ADD3:%.*]] = add nsw i32 7, [[MUL2]]
+// CHECK2-NEXT:    store i32 [[ADD3]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK2:       for.cond:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 4
+// CHECK2-NEXT:    br i1 [[CMP4]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 7, [[MUL5]]
+// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP10]], i32 noundef [[TMP11]])
+// CHECK2-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK2:       for.inc:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP12]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2:       for.end:
+// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2:       omp.body.continue:
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK2:       omp.inner.for.inc:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP13]], 1
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK2:       omp.inner.for.end:
+// CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK2:       omp.loop.exit:
+// CHECK2-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@foo4
+// CHECK2-SAME: () #[[ATTR0]] {
+// CHECK2-NEXT:  entry:
+// CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_0_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_1_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+// CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 15, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK2-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 15
+// CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 15, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK2:       omp.inner.for.cond:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2:       omp.inner.for.body:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 3
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV3:%.*]] = sdiv i32 [[TMP8]], 4
+// CHECK2-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[DIV3]], 4
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL4]]
+// CHECK2-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[SUB]], 1
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL5]]
+// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 7, [[MUL7]]
+// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK2:       for.cond:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 4
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 7, [[MUL10]]
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
+// CHECK2-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK2:       for.inc:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK2:       for.end:
+// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2:       omp.body.continue:
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK2:       omp.inner.for.inc:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK2:       omp.inner.for.end:
+// CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK2:       omp.loop.exit:
+// CHECK2-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@foo6
+// CHECK2-SAME: () #[[ATTR0]] {
+// CHECK2-NEXT:  entry:
+// CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP3:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[L:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+// CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 7, ptr [[K]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 255, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK2-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 255
+// CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 255, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK2:       omp.inner.for.cond:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP4:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    br i1 [[CMP4]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2:       omp.inner.for.body:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 64
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 3
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV5:%.*]] = sdiv i32 [[TMP8]], 64
+// CHECK2-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[DIV5]], 64
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL6]]
+// CHECK2-NEXT:    [[DIV7:%.*]] = sdiv i32 [[SUB]], 16
+// CHECK2-NEXT:    [[MUL8:%.*]] = mul nsw i32 [[DIV7]], 1
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 0, [[MUL8]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[DOTPERMUTED_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV10:%.*]] = sdiv i32 [[TMP10]], 64
+// CHECK2-NEXT:    [[MUL11:%.*]] = mul nsw i32 [[DIV10]], 64
+// CHECK2-NEXT:    [[SUB12:%.*]] = sub nsw i32 [[TMP9]], [[MUL11]]
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV13:%.*]] = sdiv i32 [[TMP12]], 64
+// CHECK2-NEXT:    [[MUL14:%.*]] = mul nsw i32 [[DIV13]], 64
+// CHECK2-NEXT:    [[SUB15:%.*]] = sub nsw i32 [[TMP11]], [[MUL14]]
+// CHECK2-NEXT:    [[DIV16:%.*]] = sdiv i32 [[SUB15]], 16
+// CHECK2-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[DIV16]], 16
+// CHECK2-NEXT:    [[SUB18:%.*]] = sub nsw i32 [[SUB12]], [[MUL17]]
+// CHECK2-NEXT:    [[DIV19:%.*]] = sdiv i32 [[SUB18]], 4
+// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[DIV19]], 1
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 0, [[MUL20]]
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[DOTPERMUTED_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV22:%.*]] = sdiv i32 [[TMP14]], 64
+// CHECK2-NEXT:    [[MUL23:%.*]] = mul nsw i32 [[DIV22]], 64
+// CHECK2-NEXT:    [[SUB24:%.*]] = sub nsw i32 [[TMP13]], [[MUL23]]
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV25:%.*]] = sdiv i32 [[TMP16]], 64
+// CHECK2-NEXT:    [[MUL26:%.*]] = mul nsw i32 [[DIV25]], 64
+// CHECK2-NEXT:    [[SUB27:%.*]] = sub nsw i32 [[TMP15]], [[MUL26]]
+// CHECK2-NEXT:    [[DIV28:%.*]] = sdiv i32 [[SUB27]], 16
+// CHECK2-NEXT:    [[MUL29:%.*]] = mul nsw i32 [[DIV28]], 16
+// CHECK2-NEXT:    [[SUB30:%.*]] = sub nsw i32 [[SUB24]], [[MUL29]]
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV31:%.*]] = sdiv i32 [[TMP18]], 64
+// CHECK2-NEXT:    [[MUL32:%.*]] = mul nsw i32 [[DIV31]], 64
+// CHECK2-NEXT:    [[SUB33:%.*]] = sub nsw i32 [[TMP17]], [[MUL32]]
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV34:%.*]] = sdiv i32 [[TMP20]], 64
+// CHECK2-NEXT:    [[MUL35:%.*]] = mul nsw i32 [[DIV34]], 64
+// CHECK2-NEXT:    [[SUB36:%.*]] = sub nsw i32 [[TMP19]], [[MUL35]]
+// CHECK2-NEXT:    [[DIV37:%.*]] = sdiv i32 [[SUB36]], 16
+// CHECK2-NEXT:    [[MUL38:%.*]] = mul nsw i32 [[DIV37]], 16
+// CHECK2-NEXT:    [[SUB39:%.*]] = sub nsw i32 [[SUB33]], [[MUL38]]
+// CHECK2-NEXT:    [[DIV40:%.*]] = sdiv i32 [[SUB39]], 4
+// CHECK2-NEXT:    [[MUL41:%.*]] = mul nsw i32 [[DIV40]], 4
+// CHECK2-NEXT:    [[SUB42:%.*]] = sub nsw i32 [[SUB30]], [[MUL41]]
+// CHECK2-NEXT:    [[MUL43:%.*]] = mul nsw i32 [[SUB42]], 3
+// CHECK2-NEXT:    [[ADD44:%.*]] = add nsw i32 7, [[MUL43]]
+// CHECK2-NEXT:    store i32 [[ADD44]], ptr [[L]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_K]], align 4
+// CHECK2-NEXT:    [[MUL45:%.*]] = mul nsw i32 [[TMP21]], 3
+// CHECK2-NEXT:    [[ADD46:%.*]] = add nsw i32 7, [[MUL45]]
+// CHECK2-NEXT:    store i32 [[ADD46]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL47:%.*]] = mul nsw i32 [[TMP22]], 3
+// CHECK2-NEXT:    [[ADD48:%.*]] = add nsw i32 7, [[MUL47]]
+// CHECK2-NEXT:    store i32 [[ADD48]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[L]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP23]], i32 noundef [[TMP24]], i32 noundef [[TMP25]], i32 noundef [[TMP26]])
+// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2:       omp.body.continue:
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK2:       omp.inner.for.inc:
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD49:%.*]] = add nsw i32 [[TMP27]], 1
+// CHECK2-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK2:       omp.inner.for.end:
+// CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK2:       omp.loop.exit:
+// CHECK2-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@foo9
+// CHECK2-SAME: () #[[ATTR0]] {
+// CHECK2-NEXT:  entry:
+// CHECK2-NEXT:    [[ARR:%.*]] = alloca [128 x double], align 16
+// CHECK2-NEXT:    [[C:%.*]] = alloca double, align 8
+// CHECK2-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTPERMUTED_1_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP0]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
+// CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP3]], ptr [[DOTCAPTURE_EXPR_3]], align 8
+// CHECK2-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_3]], align 8
+// CHECK2-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i64 [[SUB]], 1
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i64 [[ADD]], 1
+// CHECK2-NEXT:    [[SUB5:%.*]] = sub nsw i64 [[DIV]], 1
+// CHECK2-NEXT:    store i64 [[SUB5]], ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTPERMUTED_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK2:       for.cond:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP6]], 21
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 2
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// CHECK2-NEXT:    store i64 0, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
+// CHECK2:       for.cond7:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP9]], 1
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i64 [[TMP8]], [[ADD8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body10:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[MUL11:%.*]] = mul nsw i64 [[TMP11]], 1
+// CHECK2-NEXT:    [[ADD_PTR12:%.*]] = getelementptr inbounds double, ptr [[TMP10]], i64 [[MUL11]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR12]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP12]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load double, ptr [[C]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP15:%.*]] = load double, ptr [[TMP14]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP13]], double noundef [[TMP15]], i32 noundef [[TMP16]])
+// CHECK2-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK2:       for.inc:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP17]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2:       for.end:
+// CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
+// CHECK2:       for.inc13:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC14:%.*]] = add nsw i32 [[TMP18]], 1
+// CHECK2-NEXT:    store i32 [[INC14]], ptr [[DOTPERMUTED_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK2:       for.end15:
+// CHECK2-NEXT:    ret void
+//
diff --git a/clang/test/OpenMP/interchange_messages.cpp b/clang/test/OpenMP/interchange_messages.cpp
new file mode 100644
index 0000000000000..175c2f1efa744
--- /dev/null
+++ b/clang/test/OpenMP/interchange_messages.cpp
@@ -0,0 +1,77 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 -fopenmp -fopenmp-version=60 -fsyntax-only -Wuninitialized -verify %s
+
+void func() {
+
+  // expected-warning@+1 {{extra tokens at the end of '#pragma omp interchange' are ignored}}
+  #pragma omp interchange foo
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 13; ++j)
+      ;
+
+  // expected-error@+1 {{unexpected OpenMP clause 'collapse' in directive '#pragma omp interchange'}}
+  #pragma omp interchange collapse(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 13; ++j)
+      ;
+
+  {
+    // expected-error@+2 {{expected statement}}
+    #pragma omp interchange
+  }
+
+  // expected-error@+2 {{statement after '#pragma omp interchange' must be a for loop}}
+  #pragma omp interchange
+  int b = 0;
+
+  // expected-error@+3 {{statement after '#pragma omp interchange' must be a for loop}}
+  #pragma omp interchange
+  for (int i = 0; i < 7; ++i)
+    ;
+
+  // expected-error@+2 {{statement after '#pragma omp interchange' must be a for loop}}
+  #pragma omp interchange
+  for (int i = 0; i < 7; ++i) {
+    int k = 3;
+    for (int j = 0; j < 7; ++j)
+      ;
+  }
+
+  // expected-error@+3 {{expected loop invariant expression}}
+  #pragma omp interchange
+  for (int i = 0; i < 7; ++i)
+    for (int j = i; j < 7; ++j)
+      ;
+
+  // expected-error@+3 {{expected loop invariant expression}}
+  #pragma omp interchange
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < i; ++j)
+      ;
+
+  // expected-error@+3 {{expected loop invariant expression}}
+  #pragma omp interchange
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < i; ++j)
+      ;
+
+  // expected-error@+6 {{expected 3 for loops after '#pragma omp for', but found only 2}}
+  // expected-note@+1 {{as specified in 'collapse' clause}}
+  #pragma omp for collapse(3)
+  #pragma omp interchange 
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 13; ++j)
+      ;
+
+  // expected-error@+2 {{statement after '#pragma omp interchange' must be a for loop}}
+  #pragma omp interchange
+  #pragma omp for
+  for (int i = 0; i < 7; ++i)
+    ;
+
+  // expected-error@+3 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'j'}}
+  #pragma omp interchange 
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j/3<7; ++j)
+      ;
+}
+
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 52d469c31715d..d8af5abce7cee 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2183,6 +2183,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOMPTileDirective(const OMPTileDirective *D);
   void VisitOMPUnrollDirective(const OMPUnrollDirective *D);
   void VisitOMPReverseDirective(const OMPReverseDirective *D);
+  void VisitOMPInterchangeDirective(const OMPInterchangeDirective *D);
   void VisitOMPForDirective(const OMPForDirective *D);
   void VisitOMPForSimdDirective(const OMPForSimdDirective *D);
   void VisitOMPSectionsDirective(const OMPSectionsDirective *D);
@@ -3233,6 +3234,11 @@ void EnqueueVisitor::VisitOMPReverseDirective(const OMPReverseDirective *D) {
   VisitOMPLoopTransformationDirective(D);
 }
 
+void EnqueueVisitor::VisitOMPInterchangeDirective(
+    const OMPInterchangeDirective *D) {
+  VisitOMPLoopTransformationDirective(D);
+}
+
 void EnqueueVisitor::VisitOMPForDirective(const OMPForDirective *D) {
   VisitOMPLoopDirective(D);
 }
@@ -6104,6 +6110,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPUnrollDirective");
   case CXCursor_OMPReverseDirective:
     return cxstring::createRef("OMPReverseDirective");
+  case CXCursor_OMPInterchangeDirective:
+    return cxstring::createRef("OMPInterchangeDirective");
   case CXCursor_OMPForDirective:
     return cxstring::createRef("OMPForDirective");
   case CXCursor_OMPForSimdDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index d59151f756dc3..782c0c243ef1f 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -676,6 +676,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPReverseDirectiveClass:
     K = CXCursor_OMPReverseDirective;
     break;
+  case Stmt::OMPInterchangeDirectiveClass:
+    K = CXCursor_OMPTileDirective;
+    break;
   case Stmt::OMPForDirectiveClass:
     K = CXCursor_OMPForDirective;
     break;
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 4bd398128b673..99cef340f40df 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -735,6 +735,10 @@ def OMP_For : Directive<"for"> {
   let association = AS_Loop;
   let category = CA_Executable;
 }
+def OMP_Interchange : Directive<"interchange"> {
+  let association = AS_Loop;
+  let category = CA_Executable;
+}
 def OMP_interop : Directive<"interop"> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend>,
diff --git a/openmp/runtime/test/transform/interchange/foreach.cpp b/openmp/runtime/test/transform/interchange/foreach.cpp
new file mode 100644
index 0000000000000..17eaed6029d01
--- /dev/null
+++ b/openmp/runtime/test/transform/interchange/foreach.cpp
@@ -0,0 +1,216 @@
+// RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdarg>
+#include <cstdio>
+#include <vector>
+
+struct Reporter {
+  const char *name;
+
+  Reporter(const char *name) : name(name) { print("ctor"); }
+
+  Reporter() : name("<anon>") { print("ctor"); }
+
+  Reporter(const Reporter &that) : name(that.name) { print("copy ctor"); }
+
+  Reporter(Reporter &&that) : name(that.name) { print("move ctor"); }
+
+  ~Reporter() { print("dtor"); }
+
+  const Reporter &operator=(const Reporter &that) {
+    print("copy assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  const Reporter &operator=(Reporter &&that) {
+    print("move assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  struct Iterator {
+    const Reporter *owner;
+    int pos;
+
+    Iterator(const Reporter *owner, int pos) : owner(owner), pos(pos) {}
+
+    Iterator(const Iterator &that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator copy ctor");
+    }
+
+    Iterator(Iterator &&that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator move ctor");
+    }
+
+    ~Iterator() { owner->print("iterator dtor"); }
+
+    const Iterator &operator=(const Iterator &that) {
+      owner->print("iterator copy assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    const Iterator &operator=(Iterator &&that) {
+      owner->print("iterator move assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    bool operator==(const Iterator &that) const {
+      owner->print("iterator %d == %d", 2 - this->pos, 2 - that.pos);
+      return this->pos == that.pos;
+    }
+
+    Iterator &operator++() {
+      owner->print("iterator prefix ++");
+      pos -= 1;
+      return *this;
+    }
+
+    Iterator operator++(int) {
+      owner->print("iterator postfix ++");
+      auto result = *this;
+      pos -= 1;
+      return result;
+    }
+
+    int operator*() const {
+      int result = 2 - pos;
+      owner->print("iterator deref: %i", result);
+      return result;
+    }
+
+    size_t operator-(const Iterator &that) const {
+      int result = (2 - this->pos) - (2 - that.pos);
+      owner->print("iterator distance: %d", result);
+      return result;
+    }
+
+    Iterator operator+(int steps) const {
+      owner->print("iterator advance: %i += %i", 2 - this->pos, steps);
+      return Iterator(owner, pos - steps);
+    }
+
+    void print(const char *msg) const { owner->print(msg); }
+  };
+
+  Iterator begin() const {
+    print("begin()");
+    return Iterator(this, 2);
+  }
+
+  Iterator end() const {
+    print("end()");
+    return Iterator(this, -1);
+  }
+
+  void print(const char *msg, ...) const {
+    va_list args;
+    va_start(args, msg);
+    printf("[%s] ", name);
+    vprintf(msg, args);
+    printf("\n");
+    va_end(args);
+  }
+};
+
+int main() {
+  printf("do\n");
+#pragma omp interchange
+  for (Reporter c{"C"}; auto &&v : Reporter("A"))
+    for (Reporter d{"D"}; auto &&w : Reporter("B"))
+      printf("v=%d w=%d\n", v, w);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: [C] ctor
+// CHECK-NEXT: [A] ctor
+// CHECK-NEXT: [A] end()
+// CHECK-NEXT: [A] begin()
+// CHECK-NEXT: [A] begin()
+// CHECK-NEXT: [A] iterator distance: 3
+// CHECK-NEXT: [D] ctor
+// CHECK-NEXT: [B] ctor
+// CHECK-NEXT: [B] end()
+// CHECK-NEXT: [B] begin()
+// CHECK-NEXT: [B] begin()
+// CHECK-NEXT: [B] iterator distance: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: v=0 w=0
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: v=1 w=0
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: v=2 w=0
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: v=0 w=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: v=1 w=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: v=2 w=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: v=0 w=2
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: v=1 w=2
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: v=2 w=2
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] dtor
+// CHECK-NEXT: [D] dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] dtor
+// CHECK-NEXT: [C] dtor
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/interchange/intfor.c b/openmp/runtime/test/transform/interchange/intfor.c
new file mode 100644
index 0000000000000..b4842f0f82913
--- /dev/null
+++ b/openmp/runtime/test/transform/interchange/intfor.c
@@ -0,0 +1,38 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp interchange
+  for (int i = 7; i < 17; i += 3)
+    for (int j = 8; j < 18; j += 3)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=7 j=8
+// CHECK-NEXT: i=10 j=8
+// CHECK-NEXT: i=13 j=8
+// CHECK-NEXT: i=16 j=8
+// CHECK-NEXT: i=7 j=11
+// CHECK-NEXT: i=10 j=11
+// CHECK-NEXT: i=13 j=11
+// CHECK-NEXT: i=16 j=11
+// CHECK-NEXT: i=7 j=14
+// CHECK-NEXT: i=10 j=14
+// CHECK-NEXT: i=13 j=14
+// CHECK-NEXT: i=16 j=14
+// CHECK-NEXT: i=7 j=17
+// CHECK-NEXT: i=10 j=17
+// CHECK-NEXT: i=13 j=17
+// CHECK-NEXT: i=16 j=17
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/interchange/iterfor.cpp b/openmp/runtime/test/transform/interchange/iterfor.cpp
new file mode 100644
index 0000000000000..51219a07402e3
--- /dev/null
+++ b/openmp/runtime/test/transform/interchange/iterfor.cpp
@@ -0,0 +1,222 @@
+// RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdarg>
+#include <cstdio>
+#include <vector>
+
+struct Reporter {
+  const char *name;
+
+  Reporter(const char *name) : name(name) { print("ctor"); }
+
+  Reporter() : name("<anon>") { print("ctor"); }
+
+  Reporter(const Reporter &that) : name(that.name) { print("copy ctor"); }
+
+  Reporter(Reporter &&that) : name(that.name) { print("move ctor"); }
+
+  ~Reporter() { print("dtor"); }
+
+  const Reporter &operator=(const Reporter &that) {
+    print("copy assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  const Reporter &operator=(Reporter &&that) {
+    print("move assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  struct Iterator {
+    const Reporter *owner;
+    int pos;
+
+    Iterator(const Reporter *owner, int pos) : owner(owner), pos(pos) {}
+
+    Iterator(const Iterator &that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator copy ctor");
+    }
+
+    Iterator(Iterator &&that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator move ctor");
+    }
+
+    ~Iterator() { owner->print("iterator dtor"); }
+
+    const Iterator &operator=(const Iterator &that) {
+      owner->print("iterator copy assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    const Iterator &operator=(Iterator &&that) {
+      owner->print("iterator move assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    bool operator==(const Iterator &that) const {
+      owner->print("iterator %d == %d", 2 - this->pos, 2 - that.pos);
+      return this->pos == that.pos;
+    }
+
+    bool operator!=(const Iterator &that) const {
+      owner->print("iterator %d != %d", 2 - this->pos, 2 - that.pos);
+      return this->pos == that.pos;
+    }
+
+    Iterator &operator++() {
+      owner->print("iterator prefix ++");
+      pos -= 1;
+      return *this;
+    }
+
+    Iterator operator++(int) {
+      owner->print("iterator postfix ++");
+      auto result = *this;
+      pos -= 1;
+      return result;
+    }
+
+    int operator*() const {
+      int result = 2 - pos;
+      owner->print("iterator deref: %i", result);
+      return result;
+    }
+
+    size_t operator-(const Iterator &that) const {
+      int result = (2 - this->pos) - (2 - that.pos);
+      owner->print("iterator distance: %d", result);
+      return result;
+    }
+
+    Iterator operator+(int steps) const {
+      owner->print("iterator advance: %i += %i", 2 - this->pos, steps);
+      return Iterator(owner, pos - steps);
+    }
+  };
+
+  Iterator begin() const {
+    print("begin()");
+    return Iterator(this, 2);
+  }
+
+  Iterator end() const {
+    print("end()");
+    return Iterator(this, -1);
+  }
+
+  void print(const char *msg, ...) const {
+    va_list args;
+    va_start(args, msg);
+    printf("[%s] ", name);
+    vprintf(msg, args);
+    printf("\n");
+    va_end(args);
+  }
+};
+
+int main() {
+  printf("do\n");
+  Reporter A("A"), B("B");
+#pragma omp interchange
+  for (auto it = A.begin(); it != A.end(); ++it)
+    for (auto jt = B.begin(); jt != B.end(); ++jt)
+      printf("i=%d j=%d\n", *it, *jt);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: [A] ctor
+// CHECK-NEXT: [B] ctor
+// CHECK-NEXT: [A] begin()
+// CHECK-NEXT: [A] begin()
+// CHECK-NEXT: [A] end()
+// CHECK-NEXT: [A] iterator distance: 3
+// CHECK-NEXT: [B] begin()
+// CHECK-NEXT: [B] begin()
+// CHECK-NEXT: [B] end()
+// CHECK-NEXT: [B] iterator distance: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: i=2 j=2
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: done
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] dtor
+// CHECK-NEXT: [A] dtor
diff --git a/openmp/runtime/test/transform/interchange/parallel-wsloop-collapse-foreach.cpp b/openmp/runtime/test/transform/interchange/parallel-wsloop-collapse-foreach.cpp
new file mode 100644
index 0000000000000..54399d25a3f15
--- /dev/null
+++ b/openmp/runtime/test/transform/interchange/parallel-wsloop-collapse-foreach.cpp
@@ -0,0 +1,340 @@
+// RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdarg>
+#include <cstdio>
+#include <vector>
+
+struct Reporter {
+  const char *name;
+
+  Reporter(const char *name) : name(name) { print("ctor"); }
+
+  Reporter() : name("<anon>") { print("ctor"); }
+
+  Reporter(const Reporter &that) : name(that.name) { print("copy ctor"); }
+
+  Reporter(Reporter &&that) : name(that.name) { print("move ctor"); }
+
+  ~Reporter() { print("dtor"); }
+
+  const Reporter &operator=(const Reporter &that) {
+    print("copy assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  const Reporter &operator=(Reporter &&that) {
+    print("move assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  struct Iterator {
+    const Reporter *owner;
+    int pos;
+
+    Iterator(const Reporter *owner, int pos) : owner(owner), pos(pos) {}
+
+    Iterator(const Iterator &that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator copy ctor");
+    }
+
+    Iterator(Iterator &&that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator move ctor");
+    }
+
+    ~Iterator() { owner->print("iterator dtor"); }
+
+    const Iterator &operator=(const Iterator &that) {
+      owner->print("iterator copy assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    const Iterator &operator=(Iterator &&that) {
+      owner->print("iterator move assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    bool operator==(const Iterator &that) const {
+      owner->print("iterator %d == %d", 2 - this->pos, 2 - that.pos);
+      return this->pos == that.pos;
+    }
+
+    Iterator &operator++() {
+      owner->print("iterator prefix ++");
+      pos -= 1;
+      return *this;
+    }
+
+    Iterator operator++(int) {
+      owner->print("iterator postfix ++");
+      auto result = *this;
+      pos -= 1;
+      return result;
+    }
+
+    int operator*() const {
+      int result = 2 - pos;
+      owner->print("iterator deref: %i", result);
+      return result;
+    }
+
+    size_t operator-(const Iterator &that) const {
+      int result = (2 - this->pos) - (2 - that.pos);
+      owner->print("iterator distance: %d", result);
+      return result;
+    }
+
+    Iterator operator+(int steps) const {
+      owner->print("iterator advance: %i += %i", 2 - this->pos, steps);
+      return Iterator(owner, pos - steps);
+    }
+  };
+
+  Iterator begin() const {
+    print("begin()");
+    return Iterator(this, 2);
+  }
+
+  Iterator end() const {
+    print("end()");
+    return Iterator(this, -1);
+  }
+
+  void print(const char *msg, ...) const {
+    va_list args;
+    va_start(args, msg);
+    printf("[%s] ", name);
+    vprintf(msg, args);
+    printf("\n");
+    va_end(args);
+  }
+};
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+  for (int i = 0; i < 2; ++i)
+#pragma omp interchange
+    for (Reporter c{"C"}; auto &&v : Reporter("A"))
+      for (Reporter d{"D"}; auto &&w : Reporter("B"))
+        for (int k = 0; k < 2; ++k)
+          printf("i=%d v=%d w=%d k=%d\n", i, v, w, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: [C] ctor
+// CHECK-NEXT: [A] ctor
+// CHECK-NEXT: [A] end()
+// CHECK-NEXT: [A] begin()
+// CHECK-NEXT: [A] begin()
+// CHECK-NEXT: [A] iterator distance: 3
+// CHECK-NEXT: [D] ctor
+// CHECK-NEXT: [B] ctor
+// CHECK-NEXT: [B] end()
+// CHECK-NEXT: [B] begin()
+// CHECK-NEXT: [B] begin()
+// CHECK-NEXT: [B] iterator distance: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: i=0 v=0 w=0 k=0
+// CHECK-NEXT: i=0 v=0 w=0 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: i=0 v=1 w=0 k=0
+// CHECK-NEXT: i=0 v=1 w=0 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: i=0 v=2 w=0 k=0
+// CHECK-NEXT: i=0 v=2 w=0 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: i=0 v=0 w=1 k=0
+// CHECK-NEXT: i=0 v=0 w=1 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: i=0 v=1 w=1 k=0
+// CHECK-NEXT: i=0 v=1 w=1 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: i=0 v=2 w=1 k=0
+// CHECK-NEXT: i=0 v=2 w=1 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: i=0 v=0 w=2 k=0
+// CHECK-NEXT: i=0 v=0 w=2 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: i=0 v=1 w=2 k=0
+// CHECK-NEXT: i=0 v=1 w=2 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: i=0 v=2 w=2 k=0
+// CHECK-NEXT: i=0 v=2 w=2 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: i=1 v=0 w=0 k=0
+// CHECK-NEXT: i=1 v=0 w=0 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: i=1 v=1 w=0 k=0
+// CHECK-NEXT: i=1 v=1 w=0 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: i=1 v=2 w=0 k=0
+// CHECK-NEXT: i=1 v=2 w=0 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: i=1 v=0 w=1 k=0
+// CHECK-NEXT: i=1 v=0 w=1 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: i=1 v=1 w=1 k=0
+// CHECK-NEXT: i=1 v=1 w=1 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: i=1 v=2 w=1 k=0
+// CHECK-NEXT: i=1 v=2 w=1 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: i=1 v=0 w=2 k=0
+// CHECK-NEXT: i=1 v=0 w=2 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: i=1 v=1 w=2 k=0
+// CHECK-NEXT: i=1 v=1 w=2 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: i=1 v=2 w=2 k=0
+// CHECK-NEXT: i=1 v=2 w=2 k=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] dtor
+// CHECK-NEXT: [D] dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] dtor
+// CHECK-NEXT: [C] dtor
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/interchange/parallel-wsloop-collapse-intfor.cpp b/openmp/runtime/test/transform/interchange/parallel-wsloop-collapse-intfor.cpp
new file mode 100644
index 0000000000000..a4520afdb0f9c
--- /dev/null
+++ b/openmp/runtime/test/transform/interchange/parallel-wsloop-collapse-intfor.cpp
@@ -0,0 +1,106 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(4) num_threads(1)
+  for (int i = 0; i < 3; ++i)
+#pragma omp interchange
+    for (int j = 0; j < 3; ++j)
+      for (int k = 0; k < 3; ++k)
+        for (int l = 0; l < 3; ++l)
+          printf("i=%d j=%d k=%d l=%d\n", i, j, k, l);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0 j=0 k=0 l=0
+// CHECK-NEXT: i=0 j=0 k=0 l=1
+// CHECK-NEXT: i=0 j=0 k=0 l=2
+// CHECK-NEXT: i=0 j=1 k=0 l=0
+// CHECK-NEXT: i=0 j=1 k=0 l=1
+// CHECK-NEXT: i=0 j=1 k=0 l=2
+// CHECK-NEXT: i=0 j=2 k=0 l=0
+// CHECK-NEXT: i=0 j=2 k=0 l=1
+// CHECK-NEXT: i=0 j=2 k=0 l=2
+// CHECK-NEXT: i=0 j=0 k=1 l=0
+// CHECK-NEXT: i=0 j=0 k=1 l=1
+// CHECK-NEXT: i=0 j=0 k=1 l=2
+// CHECK-NEXT: i=0 j=1 k=1 l=0
+// CHECK-NEXT: i=0 j=1 k=1 l=1
+// CHECK-NEXT: i=0 j=1 k=1 l=2
+// CHECK-NEXT: i=0 j=2 k=1 l=0
+// CHECK-NEXT: i=0 j=2 k=1 l=1
+// CHECK-NEXT: i=0 j=2 k=1 l=2
+// CHECK-NEXT: i=0 j=0 k=2 l=0
+// CHECK-NEXT: i=0 j=0 k=2 l=1
+// CHECK-NEXT: i=0 j=0 k=2 l=2
+// CHECK-NEXT: i=0 j=1 k=2 l=0
+// CHECK-NEXT: i=0 j=1 k=2 l=1
+// CHECK-NEXT: i=0 j=1 k=2 l=2
+// CHECK-NEXT: i=0 j=2 k=2 l=0
+// CHECK-NEXT: i=0 j=2 k=2 l=1
+// CHECK-NEXT: i=0 j=2 k=2 l=2
+// CHECK-NEXT: i=1 j=0 k=0 l=0
+// CHECK-NEXT: i=1 j=0 k=0 l=1
+// CHECK-NEXT: i=1 j=0 k=0 l=2
+// CHECK-NEXT: i=1 j=1 k=0 l=0
+// CHECK-NEXT: i=1 j=1 k=0 l=1
+// CHECK-NEXT: i=1 j=1 k=0 l=2
+// CHECK-NEXT: i=1 j=2 k=0 l=0
+// CHECK-NEXT: i=1 j=2 k=0 l=1
+// CHECK-NEXT: i=1 j=2 k=0 l=2
+// CHECK-NEXT: i=1 j=0 k=1 l=0
+// CHECK-NEXT: i=1 j=0 k=1 l=1
+// CHECK-NEXT: i=1 j=0 k=1 l=2
+// CHECK-NEXT: i=1 j=1 k=1 l=0
+// CHECK-NEXT: i=1 j=1 k=1 l=1
+// CHECK-NEXT: i=1 j=1 k=1 l=2
+// CHECK-NEXT: i=1 j=2 k=1 l=0
+// CHECK-NEXT: i=1 j=2 k=1 l=1
+// CHECK-NEXT: i=1 j=2 k=1 l=2
+// CHECK-NEXT: i=1 j=0 k=2 l=0
+// CHECK-NEXT: i=1 j=0 k=2 l=1
+// CHECK-NEXT: i=1 j=0 k=2 l=2
+// CHECK-NEXT: i=1 j=1 k=2 l=0
+// CHECK-NEXT: i=1 j=1 k=2 l=1
+// CHECK-NEXT: i=1 j=1 k=2 l=2
+// CHECK-NEXT: i=1 j=2 k=2 l=0
+// CHECK-NEXT: i=1 j=2 k=2 l=1
+// CHECK-NEXT: i=1 j=2 k=2 l=2
+// CHECK-NEXT: i=2 j=0 k=0 l=0
+// CHECK-NEXT: i=2 j=0 k=0 l=1
+// CHECK-NEXT: i=2 j=0 k=0 l=2
+// CHECK-NEXT: i=2 j=1 k=0 l=0
+// CHECK-NEXT: i=2 j=1 k=0 l=1
+// CHECK-NEXT: i=2 j=1 k=0 l=2
+// CHECK-NEXT: i=2 j=2 k=0 l=0
+// CHECK-NEXT: i=2 j=2 k=0 l=1
+// CHECK-NEXT: i=2 j=2 k=0 l=2
+// CHECK-NEXT: i=2 j=0 k=1 l=0
+// CHECK-NEXT: i=2 j=0 k=1 l=1
+// CHECK-NEXT: i=2 j=0 k=1 l=2
+// CHECK-NEXT: i=2 j=1 k=1 l=0
+// CHECK-NEXT: i=2 j=1 k=1 l=1
+// CHECK-NEXT: i=2 j=1 k=1 l=2
+// CHECK-NEXT: i=2 j=2 k=1 l=0
+// CHECK-NEXT: i=2 j=2 k=1 l=1
+// CHECK-NEXT: i=2 j=2 k=1 l=2
+// CHECK-NEXT: i=2 j=0 k=2 l=0
+// CHECK-NEXT: i=2 j=0 k=2 l=1
+// CHECK-NEXT: i=2 j=0 k=2 l=2
+// CHECK-NEXT: i=2 j=1 k=2 l=0
+// CHECK-NEXT: i=2 j=1 k=2 l=1
+// CHECK-NEXT: i=2 j=1 k=2 l=2
+// CHECK-NEXT: i=2 j=2 k=2 l=0
+// CHECK-NEXT: i=2 j=2 k=2 l=1
+// CHECK-NEXT: i=2 j=2 k=2 l=2
+// CHECK-NEXT: done

From c248d05c6807baba34ff4fb254176e300922ea72 Mon Sep 17 00:00:00 2001
From: Emanuele Rocca <emanuele.rocca@arm.com>
Date: Fri, 19 Jul 2024 09:25:38 +0200
Subject: [PATCH 151/486] [Clang] make SVE types known to device targets too
 (#99446)

For the purpose of preprocessing and declarations in header files,
ensure clang accepts SVE types for both device and host targets.

Co-authored-by: Sander De Smalen <sander.desmalen@arm.com>
---
 clang/lib/AST/ASTContext.cpp       | 3 ++-
 clang/lib/Sema/Sema.cpp            | 4 +++-
 clang/test/PCH/aarch64-sve-types.c | 2 ++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index a8e599f7ebe04..98ce88eb6a9fd 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -1384,7 +1384,8 @@ void ASTContext::InitBuiltinTypes(const TargetInfo &Target,
 #include "clang/Basic/OpenCLExtensionTypes.def"
   }
 
-  if (Target.hasAArch64SVETypes()) {
+  if (Target.hasAArch64SVETypes() ||
+      (AuxTarget && AuxTarget->hasAArch64SVETypes())) {
 #define SVE_TYPE(Name, Id, SingletonId) \
     InitBuiltinType(SingletonId, BuiltinType::Id);
 #include "clang/Basic/AArch64SVEACLETypes.def"
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 46417964f0896..2e989f0ba6fe4 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -469,7 +469,9 @@ void Sema::Initialize() {
 #include "clang/Basic/OpenCLExtensionTypes.def"
   }
 
-  if (Context.getTargetInfo().hasAArch64SVETypes()) {
+  if (Context.getTargetInfo().hasAArch64SVETypes() ||
+      (Context.getAuxTargetInfo() &&
+       Context.getAuxTargetInfo()->hasAArch64SVETypes())) {
 #define SVE_TYPE(Name, Id, SingletonId) \
     addImplicitTypedef(Name, Context.SingletonId);
 #include "clang/Basic/AArch64SVEACLETypes.def"
diff --git a/clang/test/PCH/aarch64-sve-types.c b/clang/test/PCH/aarch64-sve-types.c
index 4c4549af0b6d6..249618c3a9708 100644
--- a/clang/test/PCH/aarch64-sve-types.c
+++ b/clang/test/PCH/aarch64-sve-types.c
@@ -1,6 +1,8 @@
 // RUN: %clang_cc1 -triple aarch64-linux-gnu -emit-pch -o %t %s
 // RUN: %clang_cc1 -triple aarch64-linux-gnu -include-pch %t \
 // RUN:   -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -aux-triple aarch64-linux-gnu \
+// RUN:   -x hip-cpp-output -emit-pch -o %t %s
 
 // expected-no-diagnostics
 

From c81366709574bb95bad86011a44e80a7f97f2c56 Mon Sep 17 00:00:00 2001
From: Mariya Podchishchaeva <mariya.podchishchaeva@intel.com>
Date: Fri, 19 Jul 2024 09:33:35 +0200
Subject: [PATCH 152/486] [clang] Fix static analyzer concerns in #embed code
 (#99331)

1. Dead code in `LookupEmbedFile`. The loop always exited on the first
iteration. This was also causing a bug of not checking all directories
provided by `--embed-dir`.

2. Use of uninitialized variable `CurTok` in `LexEmbedParameters`. It
was used to initialize the field which seems to be unused. Removed
unused field, this way `CurTok` should be initialized by Lex method.
---
 clang/include/clang/Lex/PPEmbedParameters.h  | 1 -
 clang/lib/Lex/PPDirectives.cpp               | 8 +++-----
 clang/test/Preprocessor/embed_search_paths.c | 4 ++++
 3 files changed, 7 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/Preprocessor/embed_search_paths.c

diff --git a/clang/include/clang/Lex/PPEmbedParameters.h b/clang/include/clang/Lex/PPEmbedParameters.h
index 51bf908524e7a..c4fb8d02f6f35 100644
--- a/clang/include/clang/Lex/PPEmbedParameters.h
+++ b/clang/include/clang/Lex/PPEmbedParameters.h
@@ -75,7 +75,6 @@ struct LexEmbedParametersResult {
   std::optional<PPEmbedParameterIfEmpty> MaybeIfEmptyParam;
   std::optional<PPEmbedParameterPrefix> MaybePrefixParam;
   std::optional<PPEmbedParameterSuffix> MaybeSuffixParam;
-  SourceRange ParamRange;
   int UnrecognizedParams;
 
   size_t PrefixTokenCount() const {
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index a53540b12dee6..4e77df9ec444c 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -1137,7 +1137,9 @@ Preprocessor::LookupEmbedFile(StringRef Filename, bool isAngled, bool OpenFile,
     SeparateComponents(LookupPath, Entry, Filename, false);
     llvm::Expected<FileEntryRef> ShouldBeEntry =
         FM.getFileRef(LookupPath, OpenFile);
-    return llvm::expectedToOptional(std::move(ShouldBeEntry));
+    if (ShouldBeEntry)
+      return llvm::expectedToOptional(std::move(ShouldBeEntry));
+    llvm::consumeError(ShouldBeEntry.takeError());
   }
   return std::nullopt;
 }
@@ -3624,12 +3626,10 @@ Preprocessor::LexEmbedParameters(Token &CurTok, bool ForHasEmbed) {
   LexEmbedParametersResult Result{};
   SmallVector<Token, 2> ParameterTokens;
   tok::TokenKind EndTokenKind = ForHasEmbed ? tok::r_paren : tok::eod;
-  Result.ParamRange = {CurTok.getLocation(), CurTok.getLocation()};
 
   auto DiagMismatchedBracesAndSkipToEOD =
       [&](tok::TokenKind Expected,
           std::pair<tok::TokenKind, SourceLocation> Matches) {
-        Result.ParamRange.setEnd(CurTok.getEndLoc());
         Diag(CurTok, diag::err_expected) << Expected;
         Diag(Matches.second, diag::note_matching) << Matches.first;
         if (CurTok.isNot(tok::eod))
@@ -3638,7 +3638,6 @@ Preprocessor::LexEmbedParameters(Token &CurTok, bool ForHasEmbed) {
 
   auto ExpectOrDiagAndSkipToEOD = [&](tok::TokenKind Kind) {
     if (CurTok.isNot(Kind)) {
-      Result.ParamRange.setEnd(CurTok.getEndLoc());
       Diag(CurTok, diag::err_expected) << Kind;
       if (CurTok.isNot(tok::eod))
         DiscardUntilEndOfDirective(CurTok);
@@ -3872,7 +3871,6 @@ Preprocessor::LexEmbedParameters(Token &CurTok, bool ForHasEmbed) {
       }
     }
   }
-  Result.ParamRange.setEnd(CurTok.getLocation());
   return Result;
 }
 
diff --git a/clang/test/Preprocessor/embed_search_paths.c b/clang/test/Preprocessor/embed_search_paths.c
new file mode 100644
index 0000000000000..5cc1bbf9f87a9
--- /dev/null
+++ b/clang/test/Preprocessor/embed_search_paths.c
@@ -0,0 +1,4 @@
+// RUN: %clang_cc1 -std=c23 %s -E -verify --embed-dir=%S --embed-dir=%S/Inputs
+// expected-no-diagnostics
+
+#embed <jk.txt>

From 25f4bd8872b2c15c641c8e915afb54b39d119d8a Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Fri, 19 Jul 2024 00:46:37 -0700
Subject: [PATCH 153/486] AMDGPU: Clear kill flags after FoldZeroHighBits
 (#99582)

After folding, all uses of the result register are going to be replaced
by the operand register. The kill flags on the uses of the result and
operand registers are no longer valid after the replacement, and need to
be cleared.
The only exception is, however, if the kill flag is set for the operand
register, we are sure the last use of the result register is the new
last use of the operand register, and thus we are safe to keep the kill
flags.
---
 llvm/lib/Target/AMDGPU/SIFoldOperands.cpp     |  4 +-
 .../fold-zero-high-bits-clear-kill-flags.mir  | 54 +++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/fold-zero-high-bits-clear-kill-flags.mir

diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 0e8c96625b221..9b2cab2eb73a3 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -1361,7 +1361,9 @@ bool SIFoldOperands::tryFoldZeroHighBits(MachineInstr &MI) const {
     return false;
 
   Register Dst = MI.getOperand(0).getReg();
-  MRI->replaceRegWith(Dst, SrcDef->getOperand(0).getReg());
+  MRI->replaceRegWith(Dst, Src1);
+  if (!MI.getOperand(2).isKill())
+    MRI->clearKillFlags(Src1);
   MI.eraseFromParent();
   return true;
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fold-zero-high-bits-clear-kill-flags.mir b/llvm/test/CodeGen/AMDGPU/fold-zero-high-bits-clear-kill-flags.mir
new file mode 100644
index 0000000000000..baaca76bfd8a8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fold-zero-high-bits-clear-kill-flags.mir
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -verify-machineinstrs -run-pass si-fold-operands -o - %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+
+---
+name:            fold_zero_high_bits_src1_alive
+tracksRegLiveness: true
+
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: fold_zero_high_bits_src1_alive
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[V_ADD_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U16_e64 [[COPY]], 1, 0, implicit $exec
+    ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+    ; GCN-NEXT: [[V_MUL_U32_U24_e64_:%[0-9]+]]:vgpr_32 = V_MUL_U32_U24_e64 [[V_ADD_U16_e64_]], 1, 0, implicit $exec
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[V_SUB_U16_e64_:%[0-9]+]]:vgpr_32 = V_SUB_U16_e64 [[V_ADD_U16_e64_]], [[COPY1]], 0, implicit $exec
+    %0:vgpr_32 = COPY $vgpr0
+    %1:sreg_32 = S_MOV_B32 1
+    %2:vgpr_32 = V_ADD_U16_e64 %0:vgpr_32, %1:sreg_32, 0, implicit $exec
+    %3:sreg_32 = S_MOV_B32 65535
+    %4:vgpr_32 = V_AND_B32_e64 %3:sreg_32, %2:vgpr_32, implicit $exec
+    %5:vgpr_32 = V_MUL_U32_U24_e64 killed %4:vgpr_32, %1:sreg_32, 0, implicit $exec
+    %6:vgpr_32 = COPY $vgpr1
+    %7:vgpr_32 = V_SUB_U16_e64 %2:vgpr_32, %6:vgpr_32, 0, implicit $exec
+...
+
+---
+name:            fold_zero_high_bits_src1_killed
+tracksRegLiveness: true
+
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: fold_zero_high_bits_src1_killed
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[V_ADD_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U16_e64 [[COPY]], 1, 0, implicit $exec
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[V_SUB_U16_e64_:%[0-9]+]]:vgpr_32 = V_SUB_U16_e64 [[V_ADD_U16_e64_]], [[COPY1]], 0, implicit $exec
+    ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+    ; GCN-NEXT: [[V_MUL_U32_U24_e64_:%[0-9]+]]:vgpr_32 = V_MUL_U32_U24_e64 killed [[V_ADD_U16_e64_]], 1, 0, implicit $exec
+    %0:vgpr_32 = COPY $vgpr0
+    %1:sreg_32 = S_MOV_B32 1
+    %2:vgpr_32 = V_ADD_U16_e64 %0:vgpr_32, %1:sreg_32, 0, implicit $exec
+    %6:vgpr_32 = COPY $vgpr1
+    %7:vgpr_32 = V_SUB_U16_e64 %2:vgpr_32, %6:vgpr_32, 0, implicit $exec
+    %3:sreg_32 = S_MOV_B32 65535
+    %4:vgpr_32 = V_AND_B32_e64 %3:sreg_32, killed %2:vgpr_32, implicit $exec
+    %5:vgpr_32 = V_MUL_U32_U24_e64 killed %4:vgpr_32, %1:sreg_32, 0, implicit $exec
+...

From 3a7c187031ea4999e426565fa53282cf536158dd Mon Sep 17 00:00:00 2001
From: Rainer Orth <ro@gcc.gnu.org>
Date: Fri, 19 Jul 2024 09:55:59 +0200
Subject: [PATCH 154/486] [safestack] Support multilib testing (#98002)

While working on my safestack patches, I noticed that only the default
multilib was tested even though all multilib versions of
`libclang_rt.safestack.a` were built.

This patch fixes this, patterned after the ubsan testing support.

Tested on `amd64-pc-solaris2.11` (`amd64` and `i386`),
`sparcv9-sun-solaris2.11` (`sparcv9` and `sparc`), `x86_64-pc-linux-gnu`
(`x86_64` and `i386`), and `sparc64-unknown-linux-gnu` (`sparcv9` and
`sparc`).
---
 compiler-rt/test/safestack/CMakeLists.txt     | 47 ++++++++++++-------
 compiler-rt/test/safestack/lit.cfg.py         |  2 +-
 compiler-rt/test/safestack/lit.site.cfg.py.in |  6 +++
 3 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/compiler-rt/test/safestack/CMakeLists.txt b/compiler-rt/test/safestack/CMakeLists.txt
index 89ba6e74884b1..fca06024c9856 100644
--- a/compiler-rt/test/safestack/CMakeLists.txt
+++ b/compiler-rt/test/safestack/CMakeLists.txt
@@ -1,29 +1,40 @@
 set(SAFESTACK_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(SAFESTACK_LIT_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
+set(SAFESTACK_TESTSUITES)
 set(SAFESTACK_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS})
 list(APPEND SAFESTACK_TEST_DEPS safestack)
-if(NOT COMPILER_RT_STANDALONE_BUILD)
-  # Some tests require LTO, so add a dependency on the relevant LTO plugin.
-  if(LLVM_ENABLE_PIC)
-    if(LLVM_BINUTILS_INCDIR)
-      list(APPEND SAFESTACK_TEST_DEPS
-        LLVMgold
-      )
-    endif()
-    if(APPLE)
-      list(APPEND SAFESTACK_TEST_DEPS
-        LTO
-      )
+
+macro(add_safestack_testsuite test_mode sanitizer arch)
+  set(SAFESTACK_LIT_TEST_MODE "${test_mode}")
+  set(CONFIG_NAME ${SAFESTACK_LIT_TEST_MODE})
+
+  if(NOT COMPILER_RT_STANDALONE_BUILD)
+    # Some tests require LTO, so add a dependency on the relevant LTO plugin.
+    if(LLVM_ENABLE_PIC)
+      if(LLVM_BINUTILS_INCDIR)
+        list(APPEND SAFESTACK_TEST_DEPS LLVMgold)
+      endif()
+      if(APPLE)
+        list(APPEND SAFESTACK_TEST_DEPS LTO)
+      endif()
     endif()
   endif()
-endif()
+  set(CONFIG_NAME ${CONFIG_NAME}-${arch})
+  configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py)
+  list(APPEND SAFESTACK_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
+endmacro()
+
+set(SAFESTACK_TEST_ARCH ${SAFESTACK_SUPPORTED_ARCH})
 
-configure_lit_site_cfg(
-  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
-  )
+foreach(arch ${SAFESTACK_TEST_ARCH})
+  set(SAFESTACK_TEST_TARGET_ARCH ${arch})
+  get_test_cc_for_arch(${arch} SAFESTACK_TEST_TARGET_CC SAFESTACK_TEST_TARGET_CFLAGS)
+  add_safestack_testsuite("Standalone" safestack ${arch})
+endforeach()
 
 add_lit_testsuite(check-safestack "Running the SafeStack tests"
-  ${CMAKE_CURRENT_BINARY_DIR}
+  ${SAFESTACK_TESTSUITES}
   DEPENDS ${SAFESTACK_TEST_DEPS})
diff --git a/compiler-rt/test/safestack/lit.cfg.py b/compiler-rt/test/safestack/lit.cfg.py
index 17dfae46a412b..4ab9c1ce70bac 100644
--- a/compiler-rt/test/safestack/lit.cfg.py
+++ b/compiler-rt/test/safestack/lit.cfg.py
@@ -3,7 +3,7 @@
 import os
 
 # Setup config name.
-config.name = "SafeStack"
+config.name = "SafeStack-" + config.name_suffix
 
 # Setup source root.
 config.test_source_root = os.path.dirname(__file__)
diff --git a/compiler-rt/test/safestack/lit.site.cfg.py.in b/compiler-rt/test/safestack/lit.site.cfg.py.in
index 3c8bf41b283b8..ee866bca2bad3 100644
--- a/compiler-rt/test/safestack/lit.site.cfg.py.in
+++ b/compiler-rt/test/safestack/lit.site.cfg.py.in
@@ -1,5 +1,11 @@
 @LIT_SITE_CFG_IN_HEADER@
 
+# Tool-specific config options.
+config.name_suffix = "@CONFIG_NAME@"
+config.safestack_lit_test_mode = "@SAFESTACK_LIT_TEST_MODE@"
+config.target_cflags = "@SAFESTACK_TEST_TARGET_CFLAGS@"
+config.target_arch = "@SAFESTACK_TEST_TARGET_ARCH@"
+
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
 

From ae2e66b03b7a793642d6291ccd8de2252637416a Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Fri, 19 Jul 2024 08:59:26 +0100
Subject: [PATCH 155/486] [AArch64] Use TargetStackID::ScalableVector instead
 of hard-coded values. NFC

---
 llvm/lib/Target/AArch64/AArch64FrameLowering.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 0589b14949bf4..dac03bc3c1d9e 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -3550,7 +3550,8 @@ void AArch64FrameLowering::determineStackHazardSlot(
       for (auto &MI : MBB) {
         std::optional<int> FI = getLdStFrameID(MI, MFI);
         if (FI && *FI >= 0 && *FI < (int)FrameObjects.size()) {
-          if (MFI.getStackID(*FI) == 2 || AArch64InstrInfo::isFpOrNEON(MI))
+          if (MFI.getStackID(*FI) == TargetStackID::ScalableVector ||
+              AArch64InstrInfo::isFpOrNEON(MI))
             FrameObjects[*FI] |= 2;
           else
             FrameObjects[*FI] |= 1;
@@ -4734,7 +4735,8 @@ void AArch64FrameLowering::orderFrameObjects(
       if (AFI.hasStackHazardSlotIndex()) {
         std::optional<int> FI = getLdStFrameID(MI, MFI);
         if (FI && *FI >= 0 && *FI < (int)FrameObjects.size()) {
-          if (MFI.getStackID(*FI) == 2 || AArch64InstrInfo::isFpOrNEON(MI))
+          if (MFI.getStackID(*FI) == TargetStackID::ScalableVector ||
+              AArch64InstrInfo::isFpOrNEON(MI))
             FrameObjects[*FI].Accesses |= FrameObject::AccessFPR;
           else
             FrameObjects[*FI].Accesses |= FrameObject::AccessGPR;

From a6b204b82745764e1460aa1dc26e69ff73195c60 Mon Sep 17 00:00:00 2001
From: Simon Tatham <simon.tatham@arm.com>
Date: Fri, 19 Jul 2024 09:01:25 +0100
Subject: [PATCH 156/486] [lld][AArch64] Fix handling of SHT_REL relocation
 addends. (#98291)

Normally, AArch64 ELF objects use the SHT_RELA type of relocation
section, with addends stored in each relocation. But some legacy AArch64
object producers still use SHT_REL in some situations, storing the
addend in the initial value of the data item or instruction immediate
field that the relocation will modify. LLD was mishandling relocations
of this type in multiple ways.

Firstly, many of the cases in the `getImplicitAddend` switch statement
were apparently based on a misunderstanding. The relocation types that
operate on instructions should be expecting to find an instruction of
the appropriate type, and should extract its immediate field. But many
of them were instead behaving as if they expected to find a raw 64-, 32-
or 16-bit value, and wanted to extract the right range of bits. For
example, the relocation for R_AARCH64_ADD_ABS_LO12_NC read a 16-bit word
and extracted its bottom 12 bits, presumably on the thinking that the
relocation writes the low 12 bits of the value it computes. But the
input addend for SHT_REL purposes occupies the immediate field of an
AArch64 ADD instruction, which meant it should have been reading a
32-bit AArch64 instruction encoding, and extracting bits 10-21 where the
immediate field lives. Worse, the R_AARCH64_MOVW_UABS_G2 relocation was
reading 64 bits from the input section, and since it's only relocating a
32-bit instruction, the second half of those bits would have been
completely unrelated!

Adding to that confusion, most of the values being read were first
sign-extended, and _then_ had a range of bits extracted, which doesn't
make much sense. They should have first extracted some bits from the
instruction encoding, and then sign-extended that 12-, 19-, or 21-bit
result (or whatever else) to a full 64-bit value.

Secondly, after the relocated value was computed, in most cases it was
being written into the target instruction field via a bitwise OR
operation. This meant that if the instruction field didn't initially
contain all zeroes, the wrong result would end up in it. That's not even
a 100% reliable strategy for SHT_RELA, which in some situations is used
for its repeatability (in the sense that applying the relocation twice
should cause the second answer to overwrite the first, so you can
relocate an image in advance to its most likely address, and then do it
again at load time if that turns out not to be available). But for
SHT_REL, when you expect nonzero immediate fields in normal use, it
couldn't possibly work. You could see the effect of this in the existing
test, which had a lot of FFFFFF in the expected output which there
wasn't any plausible justification for.

Finally, one relocation type was actually missing: there was no support
for R_AARCH64_ADR_PREL_LO21 at all.

So I've rewritten most of the cases in `getImplicitAddend`; replaced the
bitwise ORs with overwrites; and replaced the previous test with a much
more thorough one, obtained by writing an input assembly file with
explicitly specified relocations on instructions that also have
carefully selected immediate fields, and then doing some yaml2obj
seddery to turn the RELA relocation section into a REL one.
---
 lld/ELF/Arch/AArch64.cpp                      | 111 +++--
 .../ELF/aarch64-reloc-implicit-addend.test    | 413 ++++++++++++++----
 2 files changed, 399 insertions(+), 125 deletions(-)

diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index cf5c2380690f1..0106349b2d277 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -239,30 +239,81 @@ int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_AARCH64_IRELATIVE:
   case R_AARCH64_TLS_TPREL64:
     return read64(buf);
+
+    // The following relocation types all point at instructions, and
+    // relocate an immediate field in the instruction.
+    //
+    // The general rule, from AAELF64 §5.7.2 "Addends and PC-bias",
+    // says: "If the relocation relocates an instruction the immediate
+    // field of the instruction is extracted, scaled as required by
+    // the instruction field encoding, and sign-extended to 64 bits".
+
+    // The R_AARCH64_MOVW family operates on wide MOV/MOVK/MOVZ
+    // instructions, which have a 16-bit immediate field with its low
+    // bit in bit 5 of the instruction encoding. When the immediate
+    // field is used as an implicit addend for REL-type relocations,
+    // it is treated as added to the low bits of the output value, not
+    // shifted depending on the relocation type.
+    //
+    // This allows REL relocations to express the requirement 'please
+    // add 12345 to this symbol value and give me the four 16-bit
+    // chunks of the result', by putting the same addend 12345 in all
+    // four instructions. Carries between the 16-bit chunks are
+    // handled correctly, because the whole 64-bit addition is done
+    // once per relocation.
   case R_AARCH64_MOVW_UABS_G0:
   case R_AARCH64_MOVW_UABS_G0_NC:
-    return getBits(SignExtend64<16>(read16(buf)), 0, 15);
   case R_AARCH64_MOVW_UABS_G1:
   case R_AARCH64_MOVW_UABS_G1_NC:
-    return getBits(SignExtend64<32>(read32(buf)), 16, 31);
   case R_AARCH64_MOVW_UABS_G2:
   case R_AARCH64_MOVW_UABS_G2_NC:
-    return getBits(read64(buf), 32, 47);
   case R_AARCH64_MOVW_UABS_G3:
-    return getBits(read64(buf), 48, 63);
+    return SignExtend64<16>(getBits(read32(buf), 5, 20));
+
+    // R_AARCH64_TSTBR14 points at a TBZ or TBNZ instruction, which
+    // has a 14-bit offset measured in instructions, i.e. shifted left
+    // by 2.
   case R_AARCH64_TSTBR14:
-    return getBits(SignExtend64<32>(read32(buf)), 2, 15);
+    return SignExtend64<16>(getBits(read32(buf), 5, 18) << 2);
+
+    // R_AARCH64_CONDBR19 operates on the ordinary B.cond instruction,
+    // which has a 19-bit offset measured in instructions.
+    //
+    // R_AARCH64_LD_PREL_LO19 operates on the LDR (literal)
+    // instruction, which also has a 19-bit offset, measured in 4-byte
+    // chunks. So the calculation is the same as for
+    // R_AARCH64_CONDBR19.
   case R_AARCH64_CONDBR19:
   case R_AARCH64_LD_PREL_LO19:
-    return getBits(SignExtend64<32>(read32(buf)), 2, 20);
+    return SignExtend64<21>(getBits(read32(buf), 5, 23) << 2);
+
+    // R_AARCH64_ADD_ABS_LO12_NC operates on ADD (immediate). The
+    // immediate can optionally be shifted left by 12 bits, but this
+    // relocation is intended for the case where it is not.
   case R_AARCH64_ADD_ABS_LO12_NC:
-    return getBits(SignExtend64<16>(read16(buf)), 0, 11);
+    return SignExtend64<12>(getBits(read32(buf), 10, 21));
+
+    // R_AARCH64_ADR_PREL_LO21 operates on an ADR instruction, whose
+    // 21-bit immediate is split between two bits high up in the word
+    // (in fact the two _lowest_ order bits of the value) and 19 bits
+    // lower down.
+    //
+    // R_AARCH64_ADR_PREL_PG_HI21[_NC] operate on an ADRP instruction,
+    // which encodes the immediate in the same way, but will shift it
+    // left by 12 bits when the instruction executes. For the same
+    // reason as the MOVW family, we don't apply that left shift here.
+  case R_AARCH64_ADR_PREL_LO21:
   case R_AARCH64_ADR_PREL_PG_HI21:
   case R_AARCH64_ADR_PREL_PG_HI21_NC:
-    return getBits(SignExtend64<32>(read32(buf)), 12, 32);
+    return SignExtend64<21>((getBits(read32(buf), 5, 23) << 2) |
+                            getBits(read32(buf), 29, 30));
+
+    // R_AARCH64_{JUMP,CALL}26 operate on B and BL, which have a
+    // 26-bit offset measured in instructions.
   case R_AARCH64_JUMP26:
   case R_AARCH64_CALL26:
-    return getBits(SignExtend64<32>(read32(buf)), 2, 27);
+    return SignExtend64<28>(getBits(read32(buf), 0, 25) << 2);
+
   default:
     internalLinkerError(getErrorLocation(buf),
                         "cannot read addend for relocation " + toString(type));
@@ -366,11 +417,13 @@ static void write32AArch64Addr(uint8_t *l, uint64_t imm) {
   write32le(l, (read32le(l) & ~mask) | immLo | immHi);
 }
 
-static void or32le(uint8_t *p, int32_t v) { write32le(p, read32le(p) | v); }
+static void writeMaskedBits32le(uint8_t *p, int32_t v, uint32_t mask) {
+  write32le(p, (read32le(p) & ~mask) | v);
+}
 
 // Update the immediate field in a AARCH64 ldr, str, and add instruction.
-static void or32AArch64Imm(uint8_t *l, uint64_t imm) {
-  or32le(l, (imm & 0xFFF) << 10);
+static void write32Imm12(uint8_t *l, uint64_t imm) {
+  writeMaskedBits32le(l, (imm & 0xFFF) << 10, 0xFFF << 10);
 }
 
 // Update the immediate field in an AArch64 movk, movn or movz instruction
@@ -443,7 +496,7 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
       write32(loc, val);
     break;
   case R_AARCH64_ADD_ABS_LO12_NC:
-    or32AArch64Imm(loc, val);
+    write32Imm12(loc, val);
     break;
   case R_AARCH64_ADR_GOT_PAGE:
   case R_AARCH64_ADR_PREL_PG_HI21:
@@ -470,28 +523,28 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     [[fallthrough]];
   case R_AARCH64_CALL26:
     checkInt(loc, val, 28, rel);
-    or32le(loc, (val & 0x0FFFFFFC) >> 2);
+    writeMaskedBits32le(loc, (val & 0x0FFFFFFC) >> 2, 0x0FFFFFFC >> 2);
     break;
   case R_AARCH64_CONDBR19:
   case R_AARCH64_LD_PREL_LO19:
   case R_AARCH64_GOT_LD_PREL19:
     checkAlignment(loc, val, 4, rel);
     checkInt(loc, val, 21, rel);
-    or32le(loc, (val & 0x1FFFFC) << 3);
+    writeMaskedBits32le(loc, (val & 0x1FFFFC) << 3, 0x1FFFFC << 3);
     break;
   case R_AARCH64_LDST8_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
-    or32AArch64Imm(loc, getBits(val, 0, 11));
+    write32Imm12(loc, getBits(val, 0, 11));
     break;
   case R_AARCH64_LDST16_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
     checkAlignment(loc, val, 2, rel);
-    or32AArch64Imm(loc, getBits(val, 1, 11));
+    write32Imm12(loc, getBits(val, 1, 11));
     break;
   case R_AARCH64_LDST32_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
     checkAlignment(loc, val, 4, rel);
-    or32AArch64Imm(loc, getBits(val, 2, 11));
+    write32Imm12(loc, getBits(val, 2, 11));
     break;
   case R_AARCH64_LDST64_ABS_LO12_NC:
   case R_AARCH64_LD64_GOT_LO12_NC:
@@ -499,37 +552,39 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
   case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
   case R_AARCH64_TLSDESC_LD64_LO12:
     checkAlignment(loc, val, 8, rel);
-    or32AArch64Imm(loc, getBits(val, 3, 11));
+    write32Imm12(loc, getBits(val, 3, 11));
     break;
   case R_AARCH64_LDST128_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
     checkAlignment(loc, val, 16, rel);
-    or32AArch64Imm(loc, getBits(val, 4, 11));
+    write32Imm12(loc, getBits(val, 4, 11));
     break;
   case R_AARCH64_LD64_GOTPAGE_LO15:
     checkAlignment(loc, val, 8, rel);
-    or32AArch64Imm(loc, getBits(val, 3, 14));
+    write32Imm12(loc, getBits(val, 3, 14));
     break;
   case R_AARCH64_MOVW_UABS_G0:
     checkUInt(loc, val, 16, rel);
     [[fallthrough]];
   case R_AARCH64_MOVW_UABS_G0_NC:
-    or32le(loc, (val & 0xFFFF) << 5);
+    writeMaskedBits32le(loc, (val & 0xFFFF) << 5, 0xFFFF << 5);
     break;
   case R_AARCH64_MOVW_UABS_G1:
     checkUInt(loc, val, 32, rel);
     [[fallthrough]];
   case R_AARCH64_MOVW_UABS_G1_NC:
-    or32le(loc, (val & 0xFFFF0000) >> 11);
+    writeMaskedBits32le(loc, (val & 0xFFFF0000) >> 11, 0xFFFF0000 >> 11);
     break;
   case R_AARCH64_MOVW_UABS_G2:
     checkUInt(loc, val, 48, rel);
     [[fallthrough]];
   case R_AARCH64_MOVW_UABS_G2_NC:
-    or32le(loc, (val & 0xFFFF00000000) >> 27);
+    writeMaskedBits32le(loc, (val & 0xFFFF00000000) >> 27,
+                        0xFFFF00000000 >> 27);
     break;
   case R_AARCH64_MOVW_UABS_G3:
-    or32le(loc, (val & 0xFFFF000000000000) >> 43);
+    writeMaskedBits32le(loc, (val & 0xFFFF000000000000) >> 43,
+                        0xFFFF000000000000 >> 43);
     break;
   case R_AARCH64_MOVW_PREL_G0:
   case R_AARCH64_MOVW_SABS_G0:
@@ -562,15 +617,15 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     break;
   case R_AARCH64_TSTBR14:
     checkInt(loc, val, 16, rel);
-    or32le(loc, (val & 0xFFFC) << 3);
+    writeMaskedBits32le(loc, (val & 0xFFFC) << 3, 0xFFFC << 3);
     break;
   case R_AARCH64_TLSLE_ADD_TPREL_HI12:
     checkUInt(loc, val, 24, rel);
-    or32AArch64Imm(loc, val >> 12);
+    write32Imm12(loc, val >> 12);
     break;
   case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
   case R_AARCH64_TLSDESC_ADD_LO12:
-    or32AArch64Imm(loc, val);
+    write32Imm12(loc, val);
     break;
   case R_AARCH64_TLSDESC:
     // For R_AARCH64_TLSDESC the addend is stored in the second 64-bit word.
diff --git a/lld/test/ELF/aarch64-reloc-implicit-addend.test b/lld/test/ELF/aarch64-reloc-implicit-addend.test
index 804ed97a27371..23fc9209d2201 100644
--- a/lld/test/ELF/aarch64-reloc-implicit-addend.test
+++ b/lld/test/ELF/aarch64-reloc-implicit-addend.test
@@ -1,97 +1,316 @@
-## Test certain REL relocation types generated by legacy armasm.
-# RUN: yaml2obj %s -o %t.o
-# RUN: ld.lld %t.o -o %t
-# RUN: llvm-objdump -s %t | FileCheck %s
-
-# CHECK:      Contents of section .abs:
-# CHECK-NEXT:  [[#%x,]] 29002800 00002700 00000000 0000fcff  ).(...'.........
-# CHECK-NEXT:  [[#%x,]] ffffffff ffff                        ......
-# CHECK-NEXT: Contents of section .uabs:
-# CHECK-NEXT:  [[#%x,]] 40ffffff 40ffffff 20ffffff 20ffffff  @...@... ... ...
-# CHECK-NEXT:  [[#%x,]] 00ffffff 00ffffff                    ........
-# CHECK-NEXT: Contents of section .prel:
-# CHECK-NEXT:  [[#%x,]] 00ffffff fcfeffff f8feffff a0ffffff  ................
-# CHECK-NEXT:  [[#%x,]] 0010009f 0010009f                    ........
-# CHECK-NEXT: Contents of section .branch:
-# CHECK-NEXT:  [[#%x,]] f0ffffff f0ffffff fdffffff fcffff14  ................
-
----
-!ELF
-FileHeader:
-  Class: ELFCLASS64
-  Data: ELFDATA2LSB
-  Type: ET_REL
-  Machine: EM_AARCH64
-Sections:
-  - Name:    .abs
-    Type:    SHT_PROGBITS
-    Flags:   [ SHF_ALLOC ]
-    Content: fffffefffffffdfffffffffffffffcffffffffffffff
-  - Name:    .rel.abs
-    Type:    SHT_REL
-    Link:    .symtab
-    Info:    .abs
-    Relocations:
-      - {Offset: 0, Symbol: abs, Type: R_AARCH64_ABS16}
-      - {Offset: 2, Symbol: abs, Type: R_AARCH64_ABS32}
-      - {Offset: 6, Symbol: abs, Type: R_AARCH64_ABS64}
-      - {Offset: 14, Symbol: abs, Type: R_AARCH64_ADD_ABS_LO12_NC}
-
-  - Name:    .uabs
-    Type:    SHT_PROGBITS
-    Flags:   [ SHF_ALLOC ]
-    AddressAlign: 4
-    Content: 00ffffff00ffffff00ffffff00ffffff00ffffff00ffffff
-  - Name:    .rel.uabs
-    Type:    SHT_REL
-    Link:    .symtab
-    Info:    .uabs
-    Relocations:
-      - {Offset:  0, Symbol: abs, Type: R_AARCH64_MOVW_UABS_G0}
-      - {Offset:  4, Symbol: abs, Type: R_AARCH64_MOVW_UABS_G0_NC}
-      - {Offset:  8, Symbol: abs, Type: R_AARCH64_MOVW_UABS_G1}
-      - {Offset: 12, Symbol: abs, Type: R_AARCH64_MOVW_UABS_G1_NC}
-      - {Offset: 16, Symbol: abs, Type: R_AARCH64_MOVW_UABS_G2}
-      - {Offset: 20, Symbol: abs, Type: R_AARCH64_MOVW_UABS_G2_NC}
-
-  - Name:    .prel
-    Type:    SHT_PROGBITS
-    Flags:   [ SHF_ALLOC ]
-    AddressAlign: 4
-    Content: 00ffffff00ffffff00ffffff00ffffff00ffffff00ffffff
-  - Name:    .rel.prel
-    Type:    SHT_REL
-    Link:    .symtab
-    Info:    .prel
-    Relocations:
-      - {Offset:  0, Symbol: .prel, Type: R_AARCH64_PREL64}
-      - {Offset:  4, Symbol: .prel, Type: R_AARCH64_PREL32}
-      - {Offset:  8, Symbol: .prel, Type: R_AARCH64_PREL16}
-      - {Offset: 12, Symbol: .prel, Type: R_AARCH64_LD_PREL_LO19}
-      - {Offset: 16, Symbol: .prel, Type: R_AARCH64_ADR_PREL_PG_HI21}
-      - {Offset: 20, Symbol: .prel, Type: R_AARCH64_ADR_PREL_PG_HI21_NC}
-
-  - Name:    .branch
-    Type:    SHT_PROGBITS
-    Flags:   [ SHF_ALLOC ]
-    AddressAlign: 4
-    Content: f0fffffff0fffffff0fffffff0ffffff
-  - Name:    .rel.branch
-    Type:    SHT_REL
-    Link:    .symtab
-    Info:    .branch
-    Relocations:
-      - {Offset:  0, Symbol: .branch, Type: R_AARCH64_TSTBR14}
-      - {Offset:  4, Symbol: .branch, Type: R_AARCH64_CONDBR19}
-      - {Offset:  8, Symbol: .branch, Type: R_AARCH64_CALL26}
-      - {Offset: 12, Symbol: .branch, Type: R_AARCH64_JUMP26}
-
-Symbols:
-  - Name:    .branch
-    Section: .branch
-  - Name:    .prel
-    Section: .prel
-  - Name:    abs
-    Index:   SHN_ABS
-    Value:   42
-    Binding: STB_GLOBAL
+REQUIRES: aarch64
+
+## Test handling of addends taken from the relocated word or instruction
+## in AArch64 relocation sections of type SHT_REL. These can be generated
+## by assemblers other than LLVM, in particular the legacy 'armasm'.
+##
+## llvm-mc will only generate SHT_RELA when targeting AArch64. So to make
+## an input file with SHT_REL, we assemble our test source file, then
+## round-trip via YAML and do some seddery to change the type of the
+## relocation section. Since all the relocations were made manually with
+## .reloc directives containing no addend, this succeeds.
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=aarch64 relocs.s -o rela.o
+# RUN: obj2yaml rela.o -o rela.yaml
+# RUN: sed "s/\.rela/\.rel/;s/SHT_RELA/SHT_REL/" rela.yaml > rel.yaml
+# RUN: yaml2obj rel.yaml -o rel.o
+# RUN: llvm-mc -filetype=obj -triple=aarch64 symbols.s -o symbols.o
+# RUN: ld.lld rel.o symbols.o -o a.out --section-start=.data=0x100000 --section-start=.text=0x200000
+# RUN: llvm-objdump -s a.out | FileCheck %s --check-prefix=DATA
+# RUN: llvm-objdump -d a.out | FileCheck %s --check-prefix=CODE
+
+#--- symbols.s
+
+// Source file containing the values of target symbols for the relocations. If
+// we don't keep these in their own file, then llvm-mc is clever enough to
+// resolve some of the relocations during assembly, even though they're written
+// as explicit .reloc directives. But we want the relocations to be present in
+// the object file, so that yaml2obj can change their type and we can test
+// lld's handling of the result. So we ensure that llvm-mc can't see both the
+// .reloc and the target symbol value at the same time.
+
+.globl abs16
+.globl abs32
+.globl abs64
+.globl big64
+.globl pcrel
+.globl data
+.globl branchtarget
+.globl calltarget
+
+.equ abs16, 0x9999
+.equ data, 0x100000
+.equ branchtarget, 0x200100
+.equ calltarget, 0x02000100
+.equ pcrel, 0x245678
+.equ abs32, 0x88888888
+.equ abs64, 0x7777777777777777
+.equ big64, 0x77ffffffffffff77
+
+#--- relocs.s
+
+// Source file containing the test instructions and their relocations, with the
+// FileCheck comments interleaved.
+
+// DATA: Contents of section .data:
+.data
+
+// First test absolute data relocations. For each one I show the expected
+// value in a comment, and then expect a line in llvm-objdump -s containing
+// all the values together.
+
+        // 0x7777777777777777 + 0x1234567887654321 = 0x89abcdeffedcba98
+        .reloc ., R_AARCH64_ABS64, abs64
+        .xword 0x1234567887654321
+
+        // 0x88888888 + 0x12344321 = 0x9abccba9
+        .reloc ., R_AARCH64_ABS32, abs32
+        .word 0x12344321
+
+        // 0x9999 + 0x1234 = 0xabcd
+        .reloc ., R_AARCH64_ABS16, abs16
+        .hword 0x1234
+
+        // DATA-NEXT:  100000 98badcfe efcdab89 a9cbbc9a cdab
+
+        .balign 16
+
+// Test relative data relocs, each subtracting the address of the relocated
+// word.
+
+        // 0x100000 + 0x1234567887654321 - 0x100010 = 0x1234567887654311
+        .reloc ., R_AARCH64_PREL64, data
+        .xword 0x1234567887654321
+
+        // 0x100000 + 0x12344321 - 0x100018 = 0x12344309
+        .reloc ., R_AARCH64_PREL32, data
+        .word 0x12344321
+
+        // 0x100000 + 0x1234 - 0x10001c = 0x1218
+        .reloc ., R_AARCH64_PREL16, data
+        .hword 0x1234
+
+        // DATA-NEXT:  100010 11436587 78563412 09433412 1812
+
+// CODE: 0000000000200000 <_start>:
+.text
+.globl _start
+_start:
+
+// Full set of 4 instructions loading the constant 'abs64' and adding 0x1234 to
+// it.
+
+// Expected constant is 0x7777777777777777 + 0x1234 = 0x77777777777789ab
+
+        .reloc ., R_AARCH64_MOVW_UABS_G0_NC, abs64
+        movz x0, #0x1234
+        // CODE-NEXT:  200000: d2913560      mov     x0, #0x89ab
+        .reloc ., R_AARCH64_MOVW_UABS_G1_NC, abs64
+        movk x0, #0x1234, lsl #16
+        // CODE-NEXT:  200004: f2aeeee0      movk    x0, #0x7777, lsl #16
+        .reloc ., R_AARCH64_MOVW_UABS_G2_NC, abs64
+        movk x0, #0x1234, lsl #32
+        // CODE-NEXT:  200008: f2ceeee0      movk    x0, #0x7777, lsl #32
+        .reloc ., R_AARCH64_MOVW_UABS_G3,    abs64
+        movk x0, #0x1234, lsl #48
+        // CODE-NEXT:  20000c: f2eeeee0      movk    x0, #0x7777, lsl #48
+
+// The same, but this constant has ffff in the middle 32 bits, forcing carries
+// to be propagated.
+
+// Expected constant: 0x77ffffffffffff77 + 0x1234 = 0x78000000000011ab
+
+        .reloc ., R_AARCH64_MOVW_UABS_G0_NC, big64
+        movz x0, #0x1234
+        // CODE-NEXT:  200010: d2823560      mov     x0, #0x11ab            
+        .reloc ., R_AARCH64_MOVW_UABS_G1_NC, big64
+        movk x0, #0x1234, lsl #16
+        // CODE-NEXT:  200014: f2a00000      movk    x0, #0x0, lsl #16
+        .reloc ., R_AARCH64_MOVW_UABS_G2_NC, big64
+        movk x0, #0x1234, lsl #32
+        // CODE-NEXT:  200018: f2c00000      movk    x0, #0x0, lsl #32
+        .reloc ., R_AARCH64_MOVW_UABS_G3,    big64
+        movk x0, #0x1234, lsl #48
+        // CODE-NEXT:  20001c: f2ef0000      movk    x0, #0x7800, lsl #48
+
+// Demonstrate that offsets are treated as signed: this one is taken to be
+// -0x1234. (If it were +0xedcc then you'd be able to tell the difference by
+// the carry into the second halfword.)
+
+// Expected value: 0x7777777777777777 - 0x1234 = 0x7777777777776543
+
+        .reloc ., R_AARCH64_MOVW_UABS_G0_NC, abs64
+        movz x0, #0xedcc
+        // CODE-NEXT:  200020: d28ca860      mov     x0, #0x6543
+        .reloc ., R_AARCH64_MOVW_UABS_G1_NC, abs64
+        movk x0, #0xedcc, lsl #16
+        // CODE-NEXT:  200024: f2aeeee0      movk    x0, #0x7777, lsl #16
+        .reloc ., R_AARCH64_MOVW_UABS_G2_NC, abs64
+        movk x0, #0xedcc, lsl #32
+        // CODE-NEXT:  200028: f2ceeee0      movk    x0, #0x7777, lsl #32
+        .reloc ., R_AARCH64_MOVW_UABS_G3,    abs64
+        movk x0, #0xedcc, lsl #48
+        // CODE-NEXT:  20002c: f2eeeee0      movk    x0, #0x7777, lsl #48
+
+// Check various bits of the ADR immediate, including in particular the low 2
+// bits, which are not contiguous with the rest in the encoding.
+//
+// These values are all 0x245678 + 2^n, except the last one, where the set bit
+// of the addend is the top bit, counting as negative, i.e. we expect the value
+// 0x254678 - 0x100000 = 0x145678.
+
+        .reloc ., R_AARCH64_ADR_PREL_LO21, pcrel
+        adr x0, .+1
+        // CODE-NEXT:  200030: 3022b240      adr     x0, 0x245679
+        .reloc ., R_AARCH64_ADR_PREL_LO21, pcrel
+        adr x0, .+2
+        // CODE-NEXT:  200034: 5022b220      adr     x0, 0x24567a
+        .reloc ., R_AARCH64_ADR_PREL_LO21, pcrel
+        adr x0, .+4
+        // CODE-NEXT:  200038: 1022b220      adr     x0, 0x24567c
+        .reloc ., R_AARCH64_ADR_PREL_LO21, pcrel
+        adr x0, .+8
+        // CODE-NEXT:  20003c: 1022b220      adr     x0, 0x245680
+        .reloc ., R_AARCH64_ADR_PREL_LO21, pcrel
+        adr x0, .+1<<19
+        // CODE-NEXT:  200040: 1062b1c0      adr     x0, 0x2c5678
+        .reloc ., R_AARCH64_ADR_PREL_LO21, pcrel
+        adr x0, .-1<<20
+        // CODE-NEXT:  200044: 10a2b1a0      adr     x0, 0x145678
+
+// Now load the same set of values with ADRP+ADD. But because the real ADRP
+// instruction shifts its immediate, we must account for that.
+
+        .reloc ., R_AARCH64_ADR_PREL_PG_HI21, pcrel
+        adrp x0, 1<<12
+        // CODE-NEXT:  200048: b0000220      adrp    x0, 0x245000
+        .reloc ., R_AARCH64_ADD_ABS_LO12_NC,  pcrel
+        add x0, x0, #1
+        // CODE-NEXT:  20004c: 9119e400      add     x0, x0, #0x679
+        .reloc ., R_AARCH64_ADR_PREL_PG_HI21, pcrel
+        adrp x0, 2<<12
+        // CODE-NEXT:  200050: b0000220      adrp    x0, 0x245000
+        .reloc ., R_AARCH64_ADD_ABS_LO12_NC,  pcrel
+        add x0, x0, #2
+        // CODE-NEXT:  200054: 9119e800      add     x0, x0, #0x67a
+        .reloc ., R_AARCH64_ADR_PREL_PG_HI21, pcrel
+        adrp x0, 4<<12
+        // CODE-NEXT:  200058: b0000220      adrp    x0, 0x245000
+        .reloc ., R_AARCH64_ADD_ABS_LO12_NC,  pcrel
+        add x0, x0, #4
+        // CODE-NEXT:  20005c: 9119f000      add     x0, x0, #0x67c
+        .reloc ., R_AARCH64_ADR_PREL_PG_HI21, pcrel
+        adrp x0, 8<<12
+        // CODE-NEXT:  200060: b0000220      adrp    x0, 0x245000
+        .reloc ., R_AARCH64_ADD_ABS_LO12_NC,  pcrel
+        add x0, x0, #8
+        // CODE-NEXT:  200064: 911a0000      add     x0, x0, #0x680
+
+        // Starting here, the high bits won't fit in the ADD immediate, so that
+        // becomes 0, and only the ADRP immediate shows evidence of the addend.
+
+        .reloc ., R_AARCH64_ADR_PREL_PG_HI21, pcrel
+        adrp x0, 1<<(19+12)
+        // CODE-NEXT:  200068: b0000620      adrp    x0, 0x2c5000
+        .reloc ., R_AARCH64_ADD_ABS_LO12_NC,  pcrel
+        add x0, x0, #0
+        // CODE-NEXT:  20006c: 9119e000      add     x0, x0, #0x678
+
+        .reloc ., R_AARCH64_ADR_PREL_PG_HI21, pcrel
+        adrp x0, -1<<(20+12)
+        // CODE-NEXT:  200070: b0fffa20      adrp    x0, 0x145000
+        .reloc ., R_AARCH64_ADD_ABS_LO12_NC,  pcrel
+        add x0, x0, #0
+        // CODE-NEXT:  200074: 9119e000      add     x0, x0, #0x678
+
+        // Finally, an example with a full 21-bit addend.
+        // Expected value = 0x245678 + 0xfedcb - 0x100000 = 0x244443
+        .reloc ., R_AARCH64_ADR_PREL_PG_HI21, pcrel
+        adrp x0, (0xfedcb-0x100000)<<12
+        // CODE-NEXT:  200078: 90000220      adrp    x0, 0x244000
+        .reloc ., R_AARCH64_ADD_ABS_LO12_NC,  pcrel
+        add x0, x0, #0xdcb
+        // CODE-NEXT:  20007c: 91110c00      add     x0, x0, #0x443
+
+// PC-relative loads, in which the 19-bit offset is shifted. The offsets are
+// the same as the ADRs above, except for the first two, which can't be
+// expressed by pc-relative LDR with an offset shifted left 2.
+//
+// (The input syntax is confusing here. I'd normally expect to write this as
+// `ldr x0, [pc, #offset]`, but LLVM writes just `#offset`.)
+
+        .reloc ., R_AARCH64_LD_PREL_LO19,     pcrel
+        ldr w0, #4
+        // CODE-NEXT:  200080: 1822afe0      ldr     w0, 0x24567c
+        .reloc ., R_AARCH64_LD_PREL_LO19,     pcrel
+        ldr w0, #8
+        // CODE-NEXT:  200084: 1822afe0      ldr     w0, 0x245680
+        .reloc ., R_AARCH64_LD_PREL_LO19,     pcrel
+        ldr w0, #1<<19
+        // CODE-NEXT:  200088: 1862af80      ldr     w0, 0x2c5678
+        .reloc ., R_AARCH64_LD_PREL_LO19,     pcrel
+        ldr w0, #-1<<20
+        // CODE-NEXT:  20008c: 18a2af60      ldr     w0, 0x145678
+
+
+// For these, the branch target is 0x200100 plus powers of 2, except the offset
+// 2^15, which is negative, because the addend is treated as signed.
+
+        .reloc ., R_AARCH64_TSTBR14, branchtarget
+        tbnz x1, #63, #4
+        // CODE-NEXT:  200090: b7f803a1      tbnz    x1, #0x3f, 0x200104
+        .reloc ., R_AARCH64_TSTBR14, branchtarget
+        tbnz x1, #62, #8
+        // CODE-NEXT:  200094: b7f003a1      tbnz    x1, #0x3e, 0x200108
+        .reloc ., R_AARCH64_TSTBR14, branchtarget
+        tbnz x1, #61, #1<<14
+        // CODE-NEXT:  200098: b7ea0341      tbnz    x1, #0x3d, 0x204100
+        .reloc ., R_AARCH64_TSTBR14, branchtarget
+        tbnz x1, #60, #-1<<15
+        // CODE-NEXT:  20009c: b7e40321      tbnz    x1, #0x3c, 0x1f8100
+
+// CONDBR19 is used for both cbz/cbnz and B.cond, so test both at once. Base
+// offset is the same again (from 0x200100), but this time, offsets can go up
+// to 2^20.
+
+        .reloc ., R_AARCH64_CONDBR19, branchtarget
+        cbnz x2, #4
+        // CODE-NEXT:  2000a0: b5000322      cbnz    x2, 0x200104
+        .reloc ., R_AARCH64_CONDBR19, branchtarget
+        b.eq #8
+        // CODE-NEXT:  2000a4: 54000320      b.eq    0x200108
+        .reloc ., R_AARCH64_CONDBR19, branchtarget
+        cbz x2, #1<<19
+        // CODE-NEXT:  2000a8: b44002c2      cbz     x2, 0x280100
+        .reloc ., R_AARCH64_CONDBR19, branchtarget
+        b.vs #-1<<20
+        // CODE-NEXT:  2000ac: 548002a6      b.vs    0x100100
+
+// And for BL and B, the offsets go up to 2^25.
+
+        .reloc ., R_AARCH64_CALL26, calltarget
+        bl #4
+        // CODE-NEXT:  2000b0: 94780015      bl      0x2000104
+        .reloc ., R_AARCH64_CALL26, calltarget
+        bl #8
+        // CODE-NEXT:  2000b4: 94780015      bl      0x2000108
+        .reloc ., R_AARCH64_CALL26, calltarget
+        bl #1<<24
+        // CODE-NEXT:  2000b8: 94b80012      bl      0x3000100
+        .reloc ., R_AARCH64_CALL26, calltarget
+        bl #-1<<25
+        // CODE-NEXT:  2000bc: 97f80011      bl      0x100
+
+        .reloc ., R_AARCH64_JUMP26, calltarget
+        b #4
+        // CODE-NEXT:  2000c0: 14780011      b       0x2000104
+        .reloc ., R_AARCH64_JUMP26, calltarget
+        b #8
+        // CODE-NEXT:  2000c4: 14780011      b       0x2000108
+        .reloc ., R_AARCH64_JUMP26, calltarget
+        b #1<<24
+        // CODE-NEXT:  2000c8: 14b8000e      b       0x3000100
+        .reloc ., R_AARCH64_JUMP26, calltarget
+        b #-1<<25
+        // CODE-NEXT:  2000cc: 17f8000d      b       0x100

From 270f5e42b8daab586ffcd2b46ff41486199f6626 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 19 Jul 2024 09:21:40 +0100
Subject: [PATCH 157/486] [LV] Add tests where uniform recipe gets predicated
 for scalable VFs.

Currently the tests crash, due to a VPReplicateRecipe getting predicated
for scalable vectors.

Precommits tests for https://github.com/llvm/llvm-project/pull/98892.

Test cases for
 * https://github.com/llvm/llvm-project/issues/80416 and
 * https://github.com/llvm/llvm-project/issues/94328
---
 .../AArch64/divs-with-scalable-vfs.ll         | 93 +++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll

diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
new file mode 100644
index 0000000000000..a6a53b927696a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
@@ -0,0 +1,93 @@
+; REQUIRES: asserts
+; RUN: not --crash opt -p loop-vectorize -mtriple aarch64 -mcpu=neoverse-v1 -S %s
+
+; Test case for https://github.com/llvm/llvm-project/issues/94328.
+define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
+entry:
+  %conv61 = zext i32 %x to i64
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %div18 = sdiv i64 %M, %conv6
+  %conv20 = trunc i64 %div18 to i32
+  %mul30 = mul i64 %div18, %conv61
+  %sub31 = sub i64 %iv, %mul30
+  %conv34 = trunc i64 %sub31 to i32
+  %mul35 = mul i32 %x, %conv20
+  %add36 = add i32 %mul35, %conv34
+  %idxprom = sext i32 %add36 to i64
+  %gep = getelementptr double, ptr %dst, i64 %idxprom
+  store double 0.000000e+00, ptr %gep, align 8
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
+entry:
+  %conv61 = zext i32 %x to i64
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %c = icmp ule i64 %iv, %M
+  br i1 %c, label %then, label %loop.latch
+
+then:
+  %div18 = sdiv i64 %M, %conv6
+  %conv20 = trunc i64 %div18 to i32
+  %mul30 = mul i64 %div18, %conv61
+  %sub31 = sub i64 %iv, %mul30
+  %conv34 = trunc i64 %sub31 to i32
+  %mul35 = mul i32 %x, %conv20
+  %add36 = add i32 %mul35, %conv34
+  %idxprom = sext i32 %add36 to i64
+  %gep = getelementptr double, ptr %dst, i64 %idxprom
+  store double 0.000000e+00, ptr %gep, align 8
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Test case for https://github.com/llvm/llvm-project/issues/80416.
+define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
+entry:
+  %mul.1.i = mul i64 %x, %x
+  %mul.2.i = mul i64 %mul.1.i, %x
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %div.i = udiv i64 %iv, %mul.2.i
+  %rem.i = urem i64 %iv, %mul.2.i
+  %div.1.i = udiv i64 %rem.i, %mul.1.i
+  %rem.1.i = urem i64 %rem.i, %mul.1.i
+  %div.2.i = udiv i64 %rem.1.i, %x
+  %rem.2.i = urem i64 %rem.1.i, %x
+  %mul.i = mul i64 %x, %div.i
+  %add.i = add i64 %mul.i, %div.1.i
+  %mul.1.i9 = mul i64 %add.i, %x
+  %add.1.i = add i64 %mul.1.i9, %div.2.i
+  %mul.2.i11 = mul i64 %add.1.i, %x
+  %add.2.i = add i64 %mul.2.i11, %rem.2.i
+  %sext.i = shl i64 %add.2.i, 32
+  %conv6.i = ashr i64 %sext.i, 32
+  %gep = getelementptr i64, ptr %dst, i64 %conv6.i
+  store i64 %div.i, ptr %gep, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}

From e8fbefe15b101d7d30366cbe2ac068449923f5e2 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Fri, 19 Jul 2024 16:35:59 +0800
Subject: [PATCH 158/486] [TLI] Add basic support for remquo libcall (#99611)

This patch adds basic support for `remquo`. Constant folding support
will be submitted in a subsequent patch.

Related issue: https://github.com/llvm/llvm-project/issues/99497
---
 .../llvm/Analysis/TargetLibraryInfo.def       | 15 ++++++++++++++
 llvm/lib/Analysis/TargetLibraryInfo.cpp       |  2 ++
 llvm/lib/Transforms/Utils/BuildLibCalls.cpp   |  5 +++++
 .../Transforms/InferFunctionAttrs/annotate.ll |  9 +++++++++
 .../tools/llvm-tli-checker/ps4-tli-check.yaml | 20 +++++++++++++++----
 .../Analysis/TargetLibraryInfoTest.cpp        |  3 +++
 6 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
index 717693a7cf63c..623cdb4b6e0b7 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
@@ -1953,6 +1953,21 @@ TLI_DEFINE_ENUM_INTERNAL(remainderl)
 TLI_DEFINE_STRING_INTERNAL("remainderl")
 TLI_DEFINE_SIG_INTERNAL(LDbl, LDbl, LDbl)
 
+/// double remquo(double x, double y, int *quo);
+TLI_DEFINE_ENUM_INTERNAL(remquo)
+TLI_DEFINE_STRING_INTERNAL("remquo")
+TLI_DEFINE_SIG_INTERNAL(Dbl, Dbl, Dbl, Ptr)
+
+/// float remquof(float x, float y, int *quo);
+TLI_DEFINE_ENUM_INTERNAL(remquof)
+TLI_DEFINE_STRING_INTERNAL("remquof")
+TLI_DEFINE_SIG_INTERNAL(Flt, Flt, Flt, Ptr)
+
+/// long double remquol(long double x, long double y, int *quo);
+TLI_DEFINE_ENUM_INTERNAL(remquol)
+TLI_DEFINE_STRING_INTERNAL("remquol")
+TLI_DEFINE_SIG_INTERNAL(LDbl, LDbl, LDbl, Ptr)
+
 /// int remove(const char *path);
 TLI_DEFINE_ENUM_INTERNAL(remove)
 TLI_DEFINE_STRING_INTERNAL("remove")
diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index bf53f562ec419..5b9a7b0f33220 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -305,6 +305,7 @@ static void initializeLibCalls(TargetLibraryInfoImpl &TLI, const Triple &T,
       TLI.setUnavailable(LibFunc_modff);
       TLI.setUnavailable(LibFunc_powf);
       TLI.setUnavailable(LibFunc_remainderf);
+      TLI.setUnavailable(LibFunc_remquof);
       TLI.setUnavailable(LibFunc_sinf);
       TLI.setUnavailable(LibFunc_sinhf);
       TLI.setUnavailable(LibFunc_sqrtf);
@@ -335,6 +336,7 @@ static void initializeLibCalls(TargetLibraryInfoImpl &TLI, const Triple &T,
     TLI.setUnavailable(LibFunc_modfl);
     TLI.setUnavailable(LibFunc_powl);
     TLI.setUnavailable(LibFunc_remainderl);
+    TLI.setUnavailable(LibFunc_remquol);
     TLI.setUnavailable(LibFunc_sinl);
     TLI.setUnavailable(LibFunc_sinhl);
     TLI.setUnavailable(LibFunc_sqrtl);
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index 1ced3657184d4..0c45bd886af9d 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -1098,6 +1098,11 @@ bool llvm::inferNonMandatoryLibFuncAttrs(Function &F,
   case LibFunc_ldexpl:
     Changed |= setWillReturn(F);
     break;
+  case LibFunc_remquo:
+  case LibFunc_remquof:
+  case LibFunc_remquol:
+    Changed |= setDoesNotCapture(F, 2);
+    [[fallthrough]];
   case LibFunc_abs:
   case LibFunc_acos:
   case LibFunc_acosf:
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
index 0944402a91fd0..79962b45d253a 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -812,6 +812,15 @@ declare x86_fp80 @remainderl(x86_fp80, x86_fp80)
 ; CHECK: declare noundef i32 @remove(ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
 declare i32 @remove(ptr)
 
+; CHECK: declare double @remquo(double, double, ptr nocapture) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare double @remquo(double, double, ptr)
+
+; CHECK: declare float @remquof(float, float, ptr nocapture) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare float @remquof(float, float, ptr)
+
+; CHECK: declare x86_fp80 @remquol(x86_fp80, x86_fp80, ptr nocapture) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare x86_fp80 @remquol(x86_fp80, x86_fp80, ptr)
+
 ; CHECK: declare noundef i32 @rename(ptr nocapture noundef readonly, ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
 declare i32 @rename(ptr, ptr)
 
diff --git a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
index 95c23007b1a05..5840e024e9786 100644
--- a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
+++ b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
@@ -34,21 +34,21 @@
 #
 # CHECK: << Total TLI yes SDK no:  8
 # CHECK: >> Total TLI no  SDK yes: 0
-# CHECK: == Total TLI yes SDK yes: 239
+# CHECK: == Total TLI yes SDK yes: 242
 #
 # WRONG_DETAIL: << TLI yes SDK no : '_ZdaPv' aka operator delete[](void*)
 # WRONG_DETAIL: >> TLI no  SDK yes: '_ZdaPvj' aka operator delete[](void*, unsigned int)
 # WRONG_DETAIL-COUNT-8: << TLI yes SDK no : {{.*}}__hot_cold_t
 # WRONG_SUMMARY: << Total TLI yes SDK no:  9{{$}}
 # WRONG_SUMMARY: >> Total TLI no  SDK yes: 1{{$}}
-# WRONG_SUMMARY: == Total TLI yes SDK yes: 238
+# WRONG_SUMMARY: == Total TLI yes SDK yes: 241
 #
 ## The -COUNT suffix doesn't care if there are too many matches, so check
 ## the exact count first; the two directives should add up to that.
 ## Yes, this means additions to TLI will fail this test, but the argument
 ## to -COUNT can't be an expression.
-# AVAIL: TLI knows 480 symbols, 247 available
-# AVAIL-COUNT-247: {{^}} available
+# AVAIL: TLI knows 483 symbols, 250 available
+# AVAIL-COUNT-250: {{^}} available
 # AVAIL-NOT:       {{^}} available
 # UNAVAIL-COUNT-233: not available
 # UNAVAIL-NOT:       not available
@@ -763,6 +763,18 @@ DynamicSymbols:
     Type:            STT_FUNC
     Section:         .text
     Binding:         STB_GLOBAL
+  - Name:            remquo
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
+  - Name:            remquof
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
+  - Name:            remquol
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
   - Name:            rewind
     Type:            STT_FUNC
     Section:         .text
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index 1fe94e2aae059..1447291844dd2 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -281,6 +281,9 @@ TEST_F(TargetLibraryInfoTest, ValidProto) {
       "declare float @remainderf(float, float)\n"
       "declare x86_fp80 @remainderl(x86_fp80, x86_fp80)\n"
       "declare i32 @remove(i8*)\n"
+      "declare double @remquo(double, double, ptr)\n"
+      "declare float @remquof(float, float, ptr)\n"
+      "declare x86_fp80 @remquol(x86_fp80, x86_fp80, ptr)\n"
       "declare i32 @rename(i8*, i8*)\n"
       "declare void @rewind(%struct*)\n"
       "declare double @rint(double)\n"

From 39185da16228efb4a09a30d6825a6f508c4755a3 Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson@arm.com>
Date: Fri, 19 Jul 2024 09:54:52 +0100
Subject: [PATCH 159/486] [Clang][AArch64] Add missing SME/SVE2.1 feature
 macros (#98285)

The 2022 SME2.1and SVE2.1 feature macros are missing from Clang. Passing
'-target-feature +sve2p1' and 'target-feature +sme2p1' should prompt
Clang to define __ARM_FEATURE_SVE2p1 and __ARM_FEATURE_SME2p1
respectively, including their prerequisits..

This patch includes __ARM_FEATURE_SVE2p1 and __ARM_FEATURE_SME2p1, plus
a clang preprocessor test for each. It also ensures that the Clang macro
builder is used in a consistent fashion across Targets/AArch64.cpp.

The specification for SVE2.1 is documented in the latest (2024 Q1) ACLE
release: https://github.com/ARM-software/acle/releases . SME2p1 is not
yet featured in ACLE documentation but its features are described under
https://developer.arm.com/documentation/ddi0487/latest/
---
 clang/lib/Basic/Targets/AArch64.cpp           | 30 +++++++++++++++++--
 clang/lib/Basic/Targets/AArch64.h             |  2 ++
 .../Preprocessor/aarch64-target-features.c    | 15 ++++++++++
 3 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 2692ddec26ff4..6349fcf3dadd7 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -449,6 +449,9 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasSVE2)
     Builder.defineMacro("__ARM_FEATURE_SVE2", "1");
 
+  if (HasSVE2p1)
+    Builder.defineMacro("__ARM_FEATURE_SVE2p1", "1");
+
   if (HasSVE2 && HasSVE2AES)
     Builder.defineMacro("__ARM_FEATURE_SVE2_AES", "1");
 
@@ -467,8 +470,15 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 
   if (HasSME2) {
-    Builder.defineMacro("__ARM_FEATURE_SME");
-    Builder.defineMacro("__ARM_FEATURE_SME2");
+    Builder.defineMacro("__ARM_FEATURE_SME", "1");
+    Builder.defineMacro("__ARM_FEATURE_SME2", "1");
+    Builder.defineMacro("__ARM_FEATURE_LOCALLY_STREAMING", "1");
+  }
+
+  if (HasSME2p1) {
+    Builder.defineMacro("__ARM_FEATURE_SME", "1");
+    Builder.defineMacro("__ARM_FEATURE_SME2", "1");
+    Builder.defineMacro("__ARM_FEATURE_SME2p1", "1");
     Builder.defineMacro("__ARM_FEATURE_LOCALLY_STREAMING", "1");
   }
 
@@ -739,8 +749,10 @@ bool AArch64TargetInfo::hasFeature(StringRef Feature) const {
       .Case("sve2-bitperm", FPU & SveMode && HasSVE2BitPerm)
       .Case("sve2-sha3", FPU & SveMode && HasSVE2SHA3)
       .Case("sve2-sm4", FPU & SveMode && HasSVE2SM4)
+      .Case("sve2p1", FPU & SveMode && HasSVE2p1)
       .Case("sme", HasSME)
       .Case("sme2", HasSME2)
+      .Case("sme2p1", HasSME2p1)
       .Case("sme-f64f64", HasSMEF64F64)
       .Case("sme-i16i64", HasSMEI16I64)
       .Case("sme-fa64", HasSMEFA64)
@@ -816,6 +828,13 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasFullFP16 = true;
       HasSVE2 = true;
     }
+    if (Feature == "+sve2p1") {
+      FPU |= NeonMode;
+      FPU |= SveMode;
+      HasFullFP16 = true;
+      HasSVE2 = true;
+      HasSVE2p1 = true;
+    }
     if (Feature == "+sve2-aes") {
       FPU |= NeonMode;
       FPU |= SveMode;
@@ -867,6 +886,13 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasBFloat16 = true;
       HasFullFP16 = true;
     }
+    if (Feature == "+sme2p1") {
+      HasSME = true;
+      HasSME2 = true;
+      HasSME2p1 = true;
+      HasBFloat16 = true;
+      HasFullFP16 = true;
+    }
     if (Feature == "+sme-f64f64") {
       HasSME = true;
       HasSMEF64F64 = true;
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 71510fe289510..7bdf5a2b4106e 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -49,6 +49,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   bool HasMatMul = false;
   bool HasBFloat16 = false;
   bool HasSVE2 = false;
+  bool HasSVE2p1 = false;
   bool HasSVE2AES = false;
   bool HasSVE2SHA3 = false;
   bool HasSVE2SM4 = false;
@@ -70,6 +71,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   bool HasSME2 = false;
   bool HasSMEF64F64 = false;
   bool HasSMEI16I64 = false;
+  bool HasSME2p1 = false;
   bool HasSB = false;
   bool HasPredRes = false;
   bool HasSSBS = false;
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index 71cc36acf3f0e..87bd3e142d2c4 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -236,6 +236,15 @@
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv9-a+sve2-bitperm -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2BITPERM %s
 // CHECK-SVE2BITPERM: __ARM_FEATURE_SVE2_BITPERM 1
 
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv9-a+sve2p1 -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2p1 %s
+// CHECK-SVE2p1: __ARM_FEATURE_FP16_SCALAR_ARITHMETIC 1
+// CHECK-SVE2p1: __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 1
+// CHECK-SVE2p1: __ARM_FEATURE_SVE2 1
+// CHECK-SVE2p1: __ARM_FEATURE_SVE2p1 1
+// CHECK-SVE2p1: __ARM_NEON 1
+// CHECK-SVE2p1: __ARM_NEON_FP 0xE
+// CHECK-SVE2p1: __ARM_NEON_SVE_BRIDGE 1
+
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv8.2a+dotprod -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-DOTPROD %s
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv8.4a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-DOTPROD %s
 // CHECK-DOTPROD: __ARM_FEATURE_DOTPROD 1
@@ -694,3 +703,9 @@
 // CHECK-SME2: __ARM_FEATURE_LOCALLY_STREAMING 1
 // CHECK-SME2: __ARM_FEATURE_SME 1
 // CHECK-SME2: __ARM_FEATURE_SME2 1
+
+// RUN: %clang --target=aarch64 -march=armv9-a+sme2p1 -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SME2p1 %s
+// CHECK-SME2p1: __ARM_FEATURE_LOCALLY_STREAMING 1
+// CHECK-SME2p1: __ARM_FEATURE_SME 1
+// CHECK-SME2p1: __ARM_FEATURE_SME2 1
+// CHECK-SME2p1: __ARM_FEATURE_SME2p1 1

From f0617d2def86192c15c9194281ef3f843effed55 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Fri, 19 Jul 2024 09:59:06 +0100
Subject: [PATCH 160/486] [libcxx][test] Remove picolib UNSUPPORTED for
 now.pass.cpp (#99503)

This ARM semihosting call was implemented in QEMU by
https://gitlab.com/qemu-project/qemu/-/commit/4d834039c2107cb86931cb3f22ca3de6e4e42b06
and is present in the qemu-system-arm v8.1.3 used by the builders.
---
 libcxx/test/std/time/time.clock/time.clock.file/now.pass.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/libcxx/test/std/time/time.clock/time.clock.file/now.pass.cpp b/libcxx/test/std/time/time.clock/time.clock.file/now.pass.cpp
index 4ba17a48988b3..0852483a8b39f 100644
--- a/libcxx/test/std/time/time.clock/time.clock.file/now.pass.cpp
+++ b/libcxx/test/std/time/time.clock/time.clock.file/now.pass.cpp
@@ -10,9 +10,6 @@
 
 // UNSUPPORTED: availability-filesystem-missing
 
-// qemu: Unsupported SemiHosting SWI 0x30
-// UNSUPPORTED: LIBCXX-PICOLIBC-FIXME
-
 // <chrono>
 
 // file_clock

From e5df657bbf38f8fcd9dd8c9e79262ca184f2598b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alejandro=20=C3=81lvarez=20Ayll=C3=B3n?=
 <alejandro.alvarez@sonarsource.com>
Date: Fri, 19 Jul 2024 11:00:30 +0200
Subject: [PATCH 161/486] [Sema] Fix assertion error in
 Sema::FindInstantiatedDecl (#96509)

...when looking for a template instantiation with a non-type parameter of
unknown type and with a default value.

This can happen when a template non-type parameter has a broken
expression that gets replaced by a `RecoveryExpr`.
---
 clang/docs/ReleaseNotes.rst                        |  3 +++
 clang/lib/Sema/SemaTemplateInstantiateDecl.cpp     |  7 ++++++-
 ...instantiate-template-broken-nontype-default.cpp | 14 ++++++++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/SemaCXX/instantiate-template-broken-nontype-default.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index b0afadfa8d324..4efdb076ba768 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -857,6 +857,9 @@ Bug Fixes in This Version
 
 - ``typeof_unqual`` now properly removes type qualifiers from arrays and their element types. (#GH92667)
 
+- Fixed an assertion failure when a template non-type parameter contains
+  an invalid expression.
+
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 01432301633ed..97161febc15f7 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -6229,7 +6229,12 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
                   getTrivialTemplateArgumentLoc(UnpackedArg, QualType(), Loc));
           }
           QualType T = CheckTemplateIdType(TemplateName(TD), Loc, Args);
-          if (T.isNull())
+          // We may get a non-null type with errors, in which case
+          // `getAsCXXRecordDecl` will return `nullptr`. For instance, this
+          // happens when one of the template arguments is an invalid
+          // expression. We return early to avoid triggering the assertion
+          // about the `CodeSynthesisContext`.
+          if (T.isNull() || T->containsErrors())
             return nullptr;
           CXXRecordDecl *SubstRecord = T->getAsCXXRecordDecl();
 
diff --git a/clang/test/SemaCXX/instantiate-template-broken-nontype-default.cpp b/clang/test/SemaCXX/instantiate-template-broken-nontype-default.cpp
new file mode 100644
index 0000000000000..eb798e5887bda
--- /dev/null
+++ b/clang/test/SemaCXX/instantiate-template-broken-nontype-default.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify %s
+
+constexpr Missing a = 0; // expected-error {{unknown type name 'Missing'}}
+
+template < typename T, Missing b = a> // expected-error {{unknown type name 'Missing'}}
+class Klass { // expected-note {{candidate template ignored: could not match 'Klass<T, b>' against 'int'}} \
+                 expected-note {{implicit deduction guide declared as 'template <typename T, int b = <recovery-expr>()> Klass(Klass<T, b>) -> Klass<T, b>'}}
+  Klass(T);   // expected-note {{candidate template ignored: substitution failure [with T = int, b = <recovery-expr>()]}} \
+                 expected-note {{implicit deduction guide declared as 'template <typename T, int b = <recovery-expr>()> Klass(T) -> Klass<T, b>'}}
+};
+
+Klass foo{5}; // no-crash \
+                 expected-error {{no viable constructor or deduction guide for deduction of template arguments of 'Klass'}}
+

From 092dd9ca2d5149e3252098610c73e4fa42d831b9 Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson@arm.com>
Date: Fri, 19 Jul 2024 10:02:37 +0100
Subject: [PATCH 162/486] =?UTF-8?q?[AArch64]=20Remove=20redundant=20instru?=
 =?UTF-8?q?ctions=20in=20int-to-fp=20of=20lowest=20vector=E2=80=A6=20(#986?=
 =?UTF-8?q?02)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… element.
When converting the lowest element (that in lane 0) of a vector from an
integer to a floating-point value, LLVM should select the SIMD scalar
variant of CVTF
(https://developer.arm.com/documentation/dui0801/g/A64-SIMD-Scalar-Instructions/SCVTF--scalar--integer-)
to avoid the FPR to GPR register transfers that are required to use the
general floating-point variant
(https://developer.arm.com/documentation/dui0801/g/A64-Floating-point-Instructions/SCVTF--scalar--integer-).
This is possible as the lowest element can be referred to by the
corresponding scalar sub-register with the width of the vector's
constituent elements.

This patch adds new TableGen patterns to remove these redundant
instructions for AArch64, as well as back-end tests to ensure the new
preferred instruction selection result is produced. Existing tests that
relied on the previous selection result have also been updated.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  28 ++++
 llvm/test/CodeGen/AArch64/arm64-neon-copy.ll  |  15 +-
 .../AArch64/fixed-point-conv-vec-pat.ll       | 158 ++++++++++++++++++
 .../AArch64/sve-fixed-length-int-to-fp.ll     |   6 +-
 4 files changed, 193 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index dd11f74882115..fe4589f4fbdae 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -6084,6 +6084,34 @@ def : Pat<(f16 (any_sint_to_fp (i32 (any_fp_to_sint f16:$Rn)))),
 def : Pat<(f16 (any_uint_to_fp (i32 (any_fp_to_uint f16:$Rn)))),
           (UCVTFv1i16 (f16 (FCVTZUv1f16 f16:$Rn)))>;
 }
+
+// int -> float conversion of value in lane 0 of simd vector should use 
+// correct cvtf variant to avoid costly fpr <-> gpr register transfers.
+def : Pat<(f32 (sint_to_fp (i32 (vector_extract (v4i32 FPR128:$Rn), (i64 0))))),
+          (SCVTFv1i32 (i32 (EXTRACT_SUBREG (v4i32 FPR128:$Rn), ssub)))>;
+
+def : Pat<(f32 (uint_to_fp (i32 (vector_extract (v4i32 FPR128:$Rn), (i64 0))))),
+          (UCVTFv1i32 (i32 (EXTRACT_SUBREG (v4i32 FPR128:$Rn), ssub)))>;
+
+def : Pat<(f64 (sint_to_fp (i64 (vector_extract (v2i64 FPR128:$Rn), (i64 0))))),
+          (SCVTFv1i64 (i64 (EXTRACT_SUBREG (v2i64 FPR128:$Rn), dsub)))>;
+
+def : Pat<(f64 (uint_to_fp (i64 (vector_extract (v2i64 FPR128:$Rn), (i64 0))))),
+          (UCVTFv1i64 (i64 (EXTRACT_SUBREG (v2i64 FPR128:$Rn), dsub)))>;
+
+// fp16: integer extraction from vector must be at least 32-bits to be legal.
+// Actual extraction result is then an in-reg sign-extension of lower 16-bits.
+let Predicates = [HasNEONandIsStreamingSafe, HasFullFP16] in {
+def : Pat<(f16 (sint_to_fp (i32 (sext_inreg (i32 (vector_extract 
+                (v8i16 FPR128:$Rn), (i64 0))), i16)))), 
+          (SCVTFv1i16 (f16 (EXTRACT_SUBREG (v8i16 FPR128:$Rn), hsub)))>;
+
+// unsigned 32-bit extracted element is truncated to 16-bits using AND
+def : Pat<(f16 (uint_to_fp (i32 (and (i32 (vector_extract 
+                (v8i16 FPR128:$Rn), (i64 0))), (i32 65535))))),
+          (UCVTFv1i16 (f16 (EXTRACT_SUBREG (v8i16 FPR128:$Rn), hsub)))>;
+}
+
 // If an integer is about to be converted to a floating point value,
 // just load it on the floating point unit.
 // Here are the patterns for 8 and 16-bits to float.
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
index 43d5ab5ab54e1..c56f4409e3a62 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
@@ -1127,8 +1127,7 @@ define <8 x i8> @test_bitcastv1f64tov8i8(<1 x i64> %a) #0 {
 ; CHECK-SD-LABEL: test_bitcastv1f64tov8i8:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    fmov x8, d0
-; CHECK-SD-NEXT:    scvtf d0, x8
+; CHECK-SD-NEXT:    scvtf d0, d0
 ; CHECK-SD-NEXT:    neg v0.8b, v0.8b
 ; CHECK-SD-NEXT:    ret
 ;
@@ -1147,8 +1146,7 @@ define <4 x i16> @test_bitcastv1f64tov4i16(<1 x i64> %a) #0 {
 ; CHECK-SD-LABEL: test_bitcastv1f64tov4i16:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    fmov x8, d0
-; CHECK-SD-NEXT:    scvtf d0, x8
+; CHECK-SD-NEXT:    scvtf d0, d0
 ; CHECK-SD-NEXT:    neg v0.4h, v0.4h
 ; CHECK-SD-NEXT:    ret
 ;
@@ -1167,8 +1165,7 @@ define <2 x i32> @test_bitcastv1f64tov2i32(<1 x i64> %a) #0 {
 ; CHECK-SD-LABEL: test_bitcastv1f64tov2i32:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    fmov x8, d0
-; CHECK-SD-NEXT:    scvtf d0, x8
+; CHECK-SD-NEXT:    scvtf d0, d0
 ; CHECK-SD-NEXT:    neg v0.2s, v0.2s
 ; CHECK-SD-NEXT:    ret
 ;
@@ -1187,8 +1184,7 @@ define <1 x i64> @test_bitcastv1f64tov1i64(<1 x i64> %a) #0 {
 ; CHECK-SD-LABEL: test_bitcastv1f64tov1i64:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    fmov x8, d0
-; CHECK-SD-NEXT:    scvtf d0, x8
+; CHECK-SD-NEXT:    scvtf d0, d0
 ; CHECK-SD-NEXT:    neg d0, d0
 ; CHECK-SD-NEXT:    ret
 ;
@@ -1209,8 +1205,7 @@ define <2 x float> @test_bitcastv1f64tov2f32(<1 x i64> %a) #0 {
 ; CHECK-LABEL: test_bitcastv1f64tov2f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov x8, d0
-; CHECK-NEXT:    scvtf d0, x8
+; CHECK-NEXT:    scvtf d0, d0
 ; CHECK-NEXT:    fneg v0.2s, v0.2s
 ; CHECK-NEXT:    ret
   %vcvt.i = sitofp <1 x i64> %a to <1 x double>
diff --git a/llvm/test/CodeGen/AArch64/fixed-point-conv-vec-pat.ll b/llvm/test/CodeGen/AArch64/fixed-point-conv-vec-pat.ll
index dff216192a6c3..ab7c2afbbf871 100644
--- a/llvm/test/CodeGen/AArch64/fixed-point-conv-vec-pat.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-point-conv-vec-pat.ll
@@ -101,4 +101,162 @@ define <8 x half> @h_v8_s8(<8 x i16> %u) #0 {
   ret <8 x half> %v
 }
 
+; int-to-fp conversion of element in lane 0 should apply
+; cvtf on vector subregister to avoid fpr->gpr trip
+define float @l0_extract_f_v2s(<2 x i32> %u) {
+; CHECK-LABEL: l0_extract_f_v2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    scvtf s0, s0
+; CHECK-NEXT:    ret
+  %i = extractelement <2 x i32> %u, i64 0
+  %f = sitofp i32 %i to float
+  ret float %f
+}
+
+; cvtf to use ssub for bottom 32-bits from v2i32
+define float @l0_extract_f_v2u(<2 x i32> %u) {
+; CHECK-LABEL: l0_extract_f_v2u:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ucvtf s0, s0
+; CHECK-NEXT:    ret
+  %i = extractelement <2 x i32> %u, i64 0
+  %f = uitofp i32 %i to float
+  ret float %f
+}
+
+; Pattern should only apply when it is known to be lane 0
+define float @ln_extract_f_v2s(<2 x i32> %u, i64 %n) {
+; CHECK-LABEL: ln_extract_f_v2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    add x8, sp, #8
+; CHECK-NEXT:    str d0, [sp, #8]
+; CHECK-NEXT:    bfi x8, x0, #2, #1
+; CHECK-NEXT:    ldr s0, [x8]
+; CHECK-NEXT:    scvtf s0, s0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ret
+  %i = extractelement <2 x i32> %u, i64 %n
+  %f = sitofp i32 %i to float
+  ret float %f
+}
+
+; cvtf to use ssub for bottom 32-bits from v4i32
+define float @l0_extract_f_v4s(<4 x i32> %u) {
+; CHECK-LABEL: l0_extract_f_v4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf s0, s0
+; CHECK-NEXT:    ret
+  %i = extractelement <4 x i32> %u, i64 0
+  %f = sitofp i32 %i to float
+  ret float %f
+}
+
+define float @l0_extract_f_v4u(<4 x i32> %u) {
+; CHECK-LABEL: l0_extract_f_v4u:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf s0, s0
+; CHECK-NEXT:    ret
+  %i = extractelement <4 x i32> %u, i64 0
+  %f = uitofp i32 %i to float
+  ret float %f
+}
+
+define float @ln_extract_f_v4s(<4 x i32> %u, i64 %n) {
+; CHECK-LABEL: ln_extract_f_v4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str q0, [sp]
+; CHECK-NEXT:    bfi x8, x0, #2, #2
+; CHECK-NEXT:    ldr s0, [x8]
+; CHECK-NEXT:    scvtf s0, s0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ret
+  %i = extractelement <4 x i32> %u, i64 %n
+  %f = sitofp i32 %i to float
+  ret float %f
+}
+
+; cvtf to use dsub for bottom 64-bits from v2i64
+define double @l0_extract_d_v2s(<2 x i64> %u) {
+; CHECK-LABEL: l0_extract_d_v2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf d0, d0
+; CHECK-NEXT:    ret
+  %i = extractelement <2 x i64> %u, i64 0
+  %f = sitofp i64 %i to double
+  ret double %f
+}
+
+define double @l0_extract_d_v2u(<2 x i64> %u) {
+; CHECK-LABEL: l0_extract_d_v2u:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf d0, d0
+; CHECK-NEXT:    ret
+  %i = extractelement <2 x i64> %u, i64 0
+  %f = uitofp i64 %i to double
+  ret double %f
+}
+
+define double @ln_extract_d_v2s(<2 x i64> %u, i64 %n) {
+; CHECK-LABEL: ln_extract_d_v2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str q0, [sp]
+; CHECK-NEXT:    bfi x8, x0, #3, #1
+; CHECK-NEXT:    ldr d0, [x8]
+; CHECK-NEXT:    scvtf d0, d0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ret
+  %i = extractelement <2 x i64> %u, i64 %n
+  %f = sitofp i64 %i to double
+  ret double %f
+}
+
+; (fullfp16) cvtf to use hsub for bottom 16-bits from v8i16
+define half @l0_extract_h_v8s(<8 x i16> %u) #0 {
+; CHECK-LABEL: l0_extract_h_v8s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf h0, h0
+; CHECK-NEXT:    ret
+  %i = extractelement <8 x i16> %u, i32 0
+  %f = sitofp i16 %i to half
+  ret half %f
+}
+
+define half @l0_extract_h_v8u(<8 x i16> %u) #0 {
+; CHECK-LABEL: l0_extract_h_v8u:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf h0, h0
+; CHECK-NEXT:    ret
+  %i = extractelement <8 x i16> %u, i32 0
+  %f = uitofp i16 %i to half
+  ret half %f
+}
+
+define half @ln_extract_h_v8u(<8 x i16> %u, i32 %n) #0 {
+; CHECK-LABEL: ln_extract_h_v8u:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    str q0, [sp]
+; CHECK-NEXT:    bfi x8, x0, #1, #3
+; CHECK-NEXT:    ldrh w8, [x8]
+; CHECK-NEXT:    ucvtf h0, w8
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ret
+  %i = extractelement <8 x i16> %u, i32 %n
+  %f = uitofp i16 %i to half
+  ret half %f
+}
+
 attributes #0 = { "target-features"="+fullfp16"}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll
index 5bb012ae57503..573fe3d8b8a77 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll
@@ -827,8 +827,7 @@ define <1 x double> @ucvtf_v1i64_v1f64(<1 x i64> %op1) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ucvtf_v1i64_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov x8, d0
-; CHECK-NEXT:    ucvtf d0, x8
+; CHECK-NEXT:    ucvtf d0, d0
 ; CHECK-NEXT:    ret
   %res = uitofp <1 x i64> %op1 to <1 x double>
   ret <1 x double> %res
@@ -1752,8 +1751,7 @@ define <1 x double> @scvtf_v1i64_v1f64(<1 x i64> %op1) vscale_range(2,0) #0 {
 ; CHECK-LABEL: scvtf_v1i64_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov x8, d0
-; CHECK-NEXT:    scvtf d0, x8
+; CHECK-NEXT:    scvtf d0, d0
 ; CHECK-NEXT:    ret
   %res = sitofp <1 x i64> %op1 to <1 x double>
   ret <1 x double> %res

From ac6061e084250a377baa552842261797aa6da6a8 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Fri, 19 Jul 2024 10:06:23 +0100
Subject: [PATCH 163/486] [Analysis] Add new function
 isDereferenceableReadOnlyLoop (#97292)

I created this patch due to a reviewer request on PR #88385 to split off
the analysis changes, however without the other code in that PR I can
only test the new function with unit tests.
---
 llvm/include/llvm/Analysis/Loads.h    |  5 ++
 llvm/lib/Analysis/Loads.cpp           | 15 +++++
 llvm/unittests/Analysis/LoadsTest.cpp | 87 +++++++++++++++++++++++++++
 3 files changed, 107 insertions(+)

diff --git a/llvm/include/llvm/Analysis/Loads.h b/llvm/include/llvm/Analysis/Loads.h
index 6da1c5596b10e..33e817828b754 100644
--- a/llvm/include/llvm/Analysis/Loads.h
+++ b/llvm/include/llvm/Analysis/Loads.h
@@ -86,6 +86,11 @@ bool isDereferenceableAndAlignedInLoop(LoadInst *LI, Loop *L,
                                        ScalarEvolution &SE, DominatorTree &DT,
                                        AssumptionCache *AC = nullptr);
 
+/// Return true if the loop \p L cannot fault on any iteration and only
+/// contains read-only memory accesses.
+bool isDereferenceableReadOnlyLoop(Loop *L, ScalarEvolution *SE,
+                                   DominatorTree *DT, AssumptionCache *AC);
+
 /// Return true if we know that executing a load from this value cannot trap.
 ///
 /// If DT and ScanFrom are specified this method performs context-sensitive
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 1d54a66705a2a..255a7ae56dd3b 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -769,3 +769,18 @@ bool llvm::canReplacePointersIfEqual(const Value *From, const Value *To,
 
   return isPointerAlwaysReplaceable(From, To, DL);
 }
+
+bool llvm::isDereferenceableReadOnlyLoop(Loop *L, ScalarEvolution *SE,
+                                         DominatorTree *DT,
+                                         AssumptionCache *AC) {
+  for (BasicBlock *BB : L->blocks()) {
+    for (Instruction &I : *BB) {
+      if (auto *LI = dyn_cast<LoadInst>(&I)) {
+        if (!isDereferenceableAndAlignedInLoop(LI, L, *SE, *DT, AC))
+          return false;
+      } else if (I.mayReadFromMemory() || I.mayWriteToMemory() || I.mayThrow())
+        return false;
+    }
+  }
+  return true;
+}
diff --git a/llvm/unittests/Analysis/LoadsTest.cpp b/llvm/unittests/Analysis/LoadsTest.cpp
index 5da3feaf762f3..13377a8082096 100644
--- a/llvm/unittests/Analysis/LoadsTest.cpp
+++ b/llvm/unittests/Analysis/LoadsTest.cpp
@@ -7,8 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
@@ -114,3 +119,85 @@ define void @f(i32* %p1, i32* %p2, i64 %i) {
   EXPECT_TRUE(canReplacePointersInUseIfEqual(PtrToIntUse, P2, DL));
   EXPECT_TRUE(canReplacePointersInUseIfEqual(IcmpUse, P2, DL));
 }
+
+TEST(LoadsTest, IsDerefReadOnlyLoop) {
+  LLVMContext C;
+  std::unique_ptr<Module> M = parseIR(C,
+                                      R"IR(
+define i64 @f1() {
+entry:
+  %p1 = alloca [1024 x i8]
+  %p2 = alloca [1024 x i8]
+  br label %loop
+
+loop:
+  %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+  %ld1 = load i8, ptr %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+  %ld2 = load i8, ptr %arrayidx1, align 1
+  %cmp3 = icmp eq i8 %ld1, %ld2
+  br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+  %index.next = add i64 %index, 1
+  %exitcond = icmp ne i64 %index.next, 67
+  br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+  %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+  ret i64 %retval
+}
+
+define i64 @f2(ptr %p1) {
+entry:
+  %p2 = alloca [1024 x i8]
+  br label %loop
+
+loop:
+  %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+  %ld1 = load i8, ptr %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+  %ld2 = load i8, ptr %arrayidx1, align 1
+  %cmp3 = icmp eq i8 %ld1, %ld2
+  br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+  %index.next = add i64 %index, 1
+  %exitcond = icmp ne i64 %index.next, 67
+  br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+  %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+  ret i64 %retval
+}
+)IR");
+  auto *GV1 = M->getNamedValue("f1");
+  auto *GV2 = M->getNamedValue("f2");
+  ASSERT_TRUE(GV1 && GV2);
+  auto *F1 = dyn_cast<Function>(GV1);
+  auto *F2 = dyn_cast<Function>(GV2);
+  ASSERT_TRUE(F1 && F2);
+
+  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfo TLI(TLII);
+
+  auto IsDerefReadOnlyLoop = [&TLI](Function *F) -> bool {
+    AssumptionCache AC(*F);
+    DominatorTree DT(*F);
+    LoopInfo LI(DT);
+    ScalarEvolution SE(*F, TLI, AC, DT, LI);
+
+    Function::iterator FI = F->begin();
+    // First basic block is entry - skip it.
+    BasicBlock *Header = &*(++FI);
+    assert(Header->getName() == "loop");
+    Loop *L = LI.getLoopFor(Header);
+
+    return isDereferenceableReadOnlyLoop(L, &SE, &DT, &AC);
+  };
+
+  ASSERT_TRUE(IsDerefReadOnlyLoop(F1));
+  ASSERT_FALSE(IsDerefReadOnlyLoop(F2));
+}

From d2e8b1d717c465ed122e6135810a834c9626bfad Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 19 Jul 2024 11:05:57 +0200
Subject: [PATCH 164/486] [CVP] Regenerate test checks (NFC)

---
 .../CorrelatedValuePropagation/ashr.ll        | 139 +++++++++++++-----
 1 file changed, 105 insertions(+), 34 deletions(-)

diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/ashr.ll b/llvm/test/Transforms/CorrelatedValuePropagation/ashr.ll
index a5fb924404611..25bec18a5dbf3 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/ashr.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/ashr.ll
@@ -1,13 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=correlated-propagation -S | FileCheck %s
 
-; Check that debug locations are preserved. For more info see:
-;   https://llvm.org/docs/SourceLevelDebugging.html#fixing-errors
-; RUN: opt < %s -enable-debugify -passes=correlated-propagation -S 2>&1 | \
-; RUN:   FileCheck %s -check-prefix=DEBUG
-; DEBUG: CheckModuleDebugify: PASS
-
-; CHECK-LABEL: @test1
 define void @test1(i32 %n) {
+; CHECK-LABEL: define void @test1(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_COND:.*]]
+; CHECK:       [[FOR_COND]]:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ [[N]], %[[ENTRY]] ], [ [[SHR:%.*]], %[[FOR_BODY:.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], 1
+; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[SHR]] = lshr i32 [[A]], 5
+; CHECK-NEXT:    br label %[[FOR_COND]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.cond
 
@@ -17,7 +25,6 @@ for.cond:                                         ; preds = %for.body, %entry
   br i1 %cmp, label %for.body, label %for.end
 
 for.body:                                         ; preds = %for.cond
-; CHECK: lshr i32 %a, 5
   %shr = ashr i32 %a, 5
   br label %for.cond
 
@@ -26,8 +33,21 @@ for.end:                                          ; preds = %for.cond
 }
 
 ;; Negative test to show transform doesn't happen unless n > 0.
-; CHECK-LABEL: @test2
 define void @test2(i32 %n) {
+; CHECK-LABEL: define void @test2(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_COND:.*]]
+; CHECK:       [[FOR_COND]]:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ [[N]], %[[ENTRY]] ], [ [[SHR:%.*]], %[[FOR_BODY:.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], -2
+; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[SHR]] = ashr i32 [[A]], 2
+; CHECK-NEXT:    br label %[[FOR_COND]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.cond
 
@@ -37,7 +57,6 @@ for.cond:                                         ; preds = %for.body, %entry
   br i1 %cmp, label %for.body, label %for.end
 
 for.body:                                         ; preds = %for.cond
-; CHECK: ashr i32 %a, 2
   %shr = ashr i32 %a, 2
   br label %for.cond
 
@@ -46,14 +65,23 @@ for.end:                                          ; preds = %for.cond
 }
 
 ;; Non looping test case.
-; CHECK-LABEL: @test3
 define void @test3(i32 %n) {
+; CHECK-LABEL: define void @test3(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label %[[BB:.*]], label %[[EXIT:.*]]
+; CHECK:       [[BB]]:
+; CHECK-NEXT:    [[SHR:%.*]] = lshr exact i32 [[N]], 4
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp = icmp sgt i32 %n, 0
   br i1 %cmp, label %bb, label %exit
 
 bb:
-; CHECK: lshr exact i32 %n, 4
   %shr = ashr exact i32 %n, 4
   br label %exit
 
@@ -65,14 +93,26 @@ exit:
 ; at the point of ashr, we know that the operand is always greater than 0,
 ; because of the guard before it, so we can transform it to lshr.
 declare void @llvm.experimental.guard(i1,...)
-; CHECK-LABEL: @test4
 define void @test4(i32 %n) {
+; CHECK-LABEL: define void @test4(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP:.*]], label %[[EXIT:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ [[N]], %[[ENTRY]] ], [ [[SHR:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[A]], 2
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND]]) [ "deopt"() ]
+; CHECK-NEXT:    [[SHR]] = lshr i32 [[A]], 1
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp = icmp sgt i32 %n, 0
   br i1 %cmp, label %loop, label %exit
 
 loop:
-; CHECK: lshr i32 %a, 1
   %a = phi i32 [ %n, %entry ], [ %shr, %loop ]
   %cond = icmp sgt i32 %a, 2
   call void(i1,...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
@@ -85,14 +125,27 @@ exit:
 
 ; same test as above with assume instead of guard.
 declare void @llvm.assume(i1)
-; CHECK-LABEL: @test5
 define void @test5(i32 %n) {
+; CHECK-LABEL: define void @test5(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP:.*]], label %[[EXIT:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ [[N]], %[[ENTRY]] ], [ [[SHR:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ugt i32 [[A]], 4
+; CHECK-NEXT:    call void @llvm.assume(i1 [[COND]])
+; CHECK-NEXT:    [[SHR]] = lshr i32 [[A]], 1
+; CHECK-NEXT:    [[LOOPCOND:%.*]] = icmp ugt i32 [[SHR]], 8
+; CHECK-NEXT:    br i1 [[LOOPCOND]], label %[[LOOP]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp = icmp sgt i32 %n, 0
   br i1 %cmp, label %loop, label %exit
 
 loop:
-; CHECK: lshr i32 %a, 1
   %a = phi i32 [ %n, %entry ], [ %shr, %loop ]
   %cond = icmp sgt i32 %a, 4
   call void @llvm.assume(i1 %cond)
@@ -105,54 +158,72 @@ exit:
 }
 
 ; check that ashr of -1 or 0 is optimized away
-; CHECK-LABEL: @test6
 define i32 @test6(i32 %f, i32 %g) {
+; CHECK-LABEL: define i32 @test6(
+; CHECK-SAME: i32 [[F:%.*]], i32 [[G:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[F]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 2
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP1]])
+; CHECK-NEXT:    ret i32 [[F]]
+;
 entry:
   %0 = add i32 %f, 1
   %1 = icmp ult i32 %0, 2
   tail call void @llvm.assume(i1 %1)
-; CHECK: ret i32 %f
   %shr = ashr i32 %f, %g
   ret i32 %shr
 }
 
 ; same test as above with different numbers
-; CHECK-LABEL: @test7
 define i32 @test7(i32 %f, i32 %g) {
+; CHECK-LABEL: define i32 @test7(
+; CHECK-SAME: i32 [[F:%.*]], i32 [[G:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[F]], -2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 6
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP1]])
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[F]], -7
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
 entry:
   %0 = and i32 %f, -2
   %1 = icmp eq i32 %0, 6
   tail call void @llvm.assume(i1 %1)
   %sub = add nsw i32 %f, -7
-; CHECK: ret i32 %sub
   %shr = ashr i32 %sub, %g
   ret i32 %shr
 }
 
 ; check that ashr of -2 or 1 is not optimized away
-; CHECK-LABEL: @test8
 define i32 @test8(i32 %f, i32 %g, i1 %s) {
+; CHECK-LABEL: define i32 @test8(
+; CHECK-SAME: i32 [[F:%.*]], i32 [[G:%.*]], i1 [[S:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr i32 -2, [[F]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 1, [[G]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[S]], i32 [[TMP0]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
 entry:
-; CHECK: ashr i32 -2, %f
   %0 = ashr i32 -2, %f
-; CHECK: lshr i32 1, %g
   %1 = ashr i32 1, %g
   %2 = select i1 %s, i32 %0, i32 %1
   ret i32 %2
 }
 
 define i32 @may_including_undef(i1 %c.1, i1 %c.2) {
-; CHECK-LABEL: define i32 @may_including_undef
-; CHECK-SAME: (i1 [[C_1:%.*]], i1 [[C_2:%.*]]) {
-; CHECK-NEXT:    br i1 [[C_1]], label [[TRUE_1:%.*]], label [[FALSE:%.*]]
-; CHECK:       true.1:
-; CHECK-NEXT:    br i1 [[C_2]], label [[TRUE_2:%.*]], label [[EXIT:%.*]]
-; CHECK:       true.2:
-; CHECK-NEXT:    br label [[EXIT]]
-; CHECK:       false:
-; CHECK-NEXT:    br label [[EXIT]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 2, [[TRUE_1]] ], [ 4, [[TRUE_2]] ], [ undef, [[FALSE]] ]
+; CHECK-LABEL: define i32 @may_including_undef(
+; CHECK-SAME: i1 [[C_1:%.*]], i1 [[C_2:%.*]]) {
+; CHECK-NEXT:    br i1 [[C_1]], label %[[TRUE_1:.*]], label %[[FALSE:.*]]
+; CHECK:       [[TRUE_1]]:
+; CHECK-NEXT:    br i1 [[C_2]], label %[[TRUE_2:.*]], label %[[EXIT:.*]]
+; CHECK:       [[TRUE_2]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[FALSE]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 2, %[[TRUE_1]] ], [ 4, %[[TRUE_2]] ], [ undef, %[[FALSE]] ]
 ; CHECK-NEXT:    [[R:%.*]] = ashr i32 [[P]], 1
 ; CHECK-NEXT:    ret i32 [[R]]
 ;

From 6235698f47828747d3b1b0418e547e2e4ff9138f Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 19 Jul 2024 11:09:13 +0200
Subject: [PATCH 165/486] [CVP] Add tests for range return attributes (NFC)

---
 .../CorrelatedValuePropagation/basic.ll       | 452 +++++++++++-------
 1 file changed, 281 insertions(+), 171 deletions(-)

diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll b/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
index a3b1c89634835..c529bab4ef4a7 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
@@ -1,10 +1,11 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt < %s -passes=correlated-propagation -S | FileCheck %s
 ; PR2581
 
 define i32 @test1(i1 %C) {
-; CHECK-LABEL: @test1(
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[EXIT:%.*]], label [[BODY:%.*]]
+; CHECK-LABEL: define i32 @test1
+; CHECK-SAME: (i1 [[C:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[EXIT:%.*]], label [[BODY:%.*]]
 ; CHECK:       body:
 ; CHECK-NEXT:    ret i32 11
 ; CHECK:       exit:
@@ -23,7 +24,7 @@ exit:           ; preds = %0
 ; PR4420
 declare i1 @ext()
 define i1 @test2() {
-; CHECK-LABEL: @test2(
+; CHECK-LABEL: define i1 @test2() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COND:%.*]] = tail call i1 @ext()
 ; CHECK-NEXT:    br i1 [[COND]], label [[BB1:%.*]], label [[BB2:%.*]]
@@ -56,9 +57,10 @@ bb3:
 ; PR4855
 @gv = internal constant i8 7
 define i8 @test3(ptr %a) nounwind {
-; CHECK-LABEL: @test3(
+; CHECK-LABEL: define i8 @test3
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[A:%.*]], @gv
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[A]], @gv
 ; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB:%.*]]
 ; CHECK:       bb:
 ; CHECK-NEXT:    ret i8 0
@@ -80,9 +82,10 @@ bb2:
 
 ; PR1757
 define i32 @test4(i32) {
-; CHECK-LABEL: @test4(
+; CHECK-LABEL: define i32 @test4
+; CHECK-SAME: (i32 [[TMP0:%.*]]) {
 ; CHECK-NEXT:  EntryBlock:
-; CHECK-NEXT:    [[DOTDEMORGAN:%.*]] = icmp sgt i32 [[TMP0:%.*]], 2
+; CHECK-NEXT:    [[DOTDEMORGAN:%.*]] = icmp sgt i32 [[TMP0]], 2
 ; CHECK-NEXT:    br i1 [[DOTDEMORGAN]], label [[GREATERTHANTWO:%.*]], label [[LESSTHANOREQUALTOTWO:%.*]]
 ; CHECK:       GreaterThanTwo:
 ; CHECK-NEXT:    br i1 false, label [[IMPOSSIBLE:%.*]], label [[NOTTWOANDGREATERTHANTWO:%.*]]
@@ -113,14 +116,15 @@ LessThanOrEqualToTwo:
 
 declare ptr @f(ptr)
 define void @test5(ptr %x, ptr %y) {
-; CHECK-LABEL: @test5(
+; CHECK-LABEL: define void @test5
+; CHECK-SAME: (ptr [[X:%.*]], ptr [[Y:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[PRE:%.*]] = icmp eq ptr [[X:%.*]], null
+; CHECK-NEXT:    [[PRE:%.*]] = icmp eq ptr [[X]], null
 ; CHECK-NEXT:    br i1 [[PRE]], label [[RETURN:%.*]], label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[PHI:%.*]] = phi ptr [ [[F:%.*]], [[LOOP]] ], [ [[X]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[F]] = tail call ptr @f(ptr [[PHI]])
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne ptr [[F]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne ptr [[F]], [[Y]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP1]], ptr [[F]], ptr null
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq ptr [[SEL]], null
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[RETURN]], label [[LOOP]]
@@ -145,13 +149,14 @@ return:
 
 ; "false" case for CorrelatedValuePropagation
 define void @loop1(ptr %x, ptr %y) {
-; CHECK-LABEL: @loop1(
+; CHECK-LABEL: define void @loop1
+; CHECK-SAME: (ptr [[X:%.*]], ptr [[Y:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PHI:%.*]] = phi ptr [ [[F:%.*]], [[LOOP]] ], [ [[X:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi ptr [ [[F:%.*]], [[LOOP]] ], [ [[X]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[F]] = tail call ptr @f(ptr [[PHI]])
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne ptr [[F]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne ptr [[F]], [[Y]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP1]], ptr [[F]], ptr null
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq ptr [[SEL]], null
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[RETURN:%.*]], label [[LOOP]]
@@ -175,13 +180,14 @@ return:
 
 ; "true" case for CorrelatedValuePropagation
 define void @loop2(ptr %x, ptr %y) {
-; CHECK-LABEL: @loop2(
+; CHECK-LABEL: define void @loop2
+; CHECK-SAME: (ptr [[X:%.*]], ptr [[Y:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PHI:%.*]] = phi ptr [ [[F:%.*]], [[LOOP]] ], [ [[X:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi ptr [ [[F:%.*]], [[LOOP]] ], [ [[X]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[F]] = tail call ptr @f(ptr [[PHI]])
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq ptr [[F]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq ptr [[F]], [[Y]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP1]], ptr null, ptr [[F]]
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq ptr [[SEL]], null
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[RETURN:%.*]], label [[LOOP]]
@@ -204,9 +210,10 @@ return:
 }
 
 define i32 @switch1(i32 %s) {
-; CHECK-LABEL: @switch1(
+; CHECK-LABEL: define i32 @switch1
+; CHECK-SAME: (i32 [[S:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[S:%.*]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[S]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[NEGATIVE:%.*]], label [[OUT:%.*]]
 ; CHECK:       negative:
 ; CHECK-NEXT:    switch i32 [[S]], label [[OUT]] [
@@ -243,9 +250,10 @@ next:
 }
 
 define i32 @switch2(i32 %s) {
-; CHECK-LABEL: @switch2(
+; CHECK-LABEL: define i32 @switch2
+; CHECK-SAME: (i32 [[S:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[S:%.*]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[S]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[POSITIVE:%.*]], label [[OUT:%.*]]
 ; CHECK:       positive:
 ; CHECK-NEXT:    br label [[OUT]]
@@ -276,9 +284,10 @@ next:
 }
 
 define i32 @switch3(i32 %s) {
-; CHECK-LABEL: @switch3(
+; CHECK-LABEL: define i32 @switch3
+; CHECK-SAME: (i32 [[S:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[S:%.*]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[S]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[POSITIVE:%.*]], label [[OUT:%.*]]
 ; CHECK:       positive:
 ; CHECK-NEXT:    br label [[OUT]]
@@ -309,9 +318,10 @@ next:
 }
 
 define void @switch4(i32 %s) {
-; CHECK-LABEL: @switch4(
+; CHECK-LABEL: define void @switch4
+; CHECK-SAME: (i32 [[S:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[S:%.*]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[S]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[ZERO:%.*]], label [[OUT:%.*]]
 ; CHECK:       zero:
 ; CHECK-NEXT:    br label [[NEXT:%.*]]
@@ -339,9 +349,10 @@ next:
 }
 
 define void @switch_nonzero_zext(i8 %s) {
-; CHECK-LABEL: @switch_nonzero_zext(
+; CHECK-LABEL: define void @switch_nonzero_zext
+; CHECK-SAME: (i8 [[S:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[S:%.*]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[S]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[SWITCH:%.*]], label [[EXIT:%.*]]
 ; CHECK:       switch:
 ; CHECK-NEXT:    [[S_EXT:%.*]] = zext i8 [[S]] to i32
@@ -371,9 +382,10 @@ unreachable:
 }
 
 define void @switch_assume_nonzero(i32 %s) {
-; CHECK-LABEL: @switch_assume_nonzero(
+; CHECK-LABEL: define void @switch_assume_nonzero
+; CHECK-SAME: (i32 [[S:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[S:%.*]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[S]], 0
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
@@ -398,9 +410,10 @@ unreachable:
 }
 
 define void @switch_nonzero_phi(i1 %cond) {
-; CHECK-LABEL: @switch_nonzero_phi(
+; CHECK-LABEL: define void @switch_nonzero_phi
+; CHECK-SAME: (i1 [[COND:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    br label [[SWITCH:%.*]]
 ; CHECK:       else:
@@ -438,9 +451,10 @@ unreachable:
 }
 
 define i32 @switch_range(i32 %cond) {
-; CHECK-LABEL: @switch_range(
+; CHECK-LABEL: define i32 @switch_range
+; CHECK-SAME: (i32 [[COND:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[S:%.*]] = urem i32 [[COND:%.*]], 3
+; CHECK-NEXT:    [[S:%.*]] = urem i32 [[COND]], 3
 ; CHECK-NEXT:    [[S1:%.*]] = add nuw nsw i32 [[S]], 1
 ; CHECK-NEXT:    switch i32 [[S1]], label [[DEFAULT_UNREACHABLE:%.*]] [
 ; CHECK-NEXT:      i32 1, label [[EXIT1:%.*]]
@@ -477,9 +491,10 @@ unreachable:
 ; switch condition, we should not change the default.
 
 define i32 @switch_range_not_full(i32 %cond) {
-; CHECK-LABEL: @switch_range_not_full(
+; CHECK-LABEL: define i32 @switch_range_not_full
+; CHECK-SAME: (i32 [[COND:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[S:%.*]] = urem i32 [[COND:%.*]], 3
+; CHECK-NEXT:    [[S:%.*]] = urem i32 [[COND]], 3
 ; CHECK-NEXT:    [[S1:%.*]] = add nuw nsw i32 [[S]], 1
 ; CHECK-NEXT:    switch i32 [[S1]], label [[UNREACHABLE:%.*]] [
 ; CHECK-NEXT:      i32 1, label [[EXIT1:%.*]]
@@ -511,9 +526,10 @@ unreachable:
 ; PR51531
 
 define i8 @switch_defaultdest_multipleuse(i8 %t0) {
-; CHECK-LABEL: @switch_defaultdest_multipleuse(
+; CHECK-LABEL: define i8 @switch_defaultdest_multipleuse
+; CHECK-SAME: (i8 [[T0:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[O:%.*]] = or i8 [[T0:%.*]], 1
+; CHECK-NEXT:    [[O:%.*]] = or i8 [[T0]], 1
 ; CHECK-NEXT:    [[R:%.*]] = srem i8 1, [[O]]
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       default.unreachable:
@@ -534,7 +550,8 @@ exit:
 }
 
 define i1 @arg_attribute(ptr nonnull %a) {
-; CHECK-LABEL: @arg_attribute(
+; CHECK-LABEL: define i1 @arg_attribute
+; CHECK-SAME: (ptr nonnull [[A:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %cmp = icmp eq ptr %a, null
@@ -543,7 +560,7 @@ define i1 @arg_attribute(ptr nonnull %a) {
 
 declare nonnull ptr @return_nonnull()
 define i1 @call_attribute() {
-; CHECK-LABEL: @call_attribute(
+; CHECK-LABEL: define i1 @call_attribute() {
 ; CHECK-NEXT:    [[A:%.*]] = call ptr @return_nonnull()
 ; CHECK-NEXT:    ret i1 false
 ;
@@ -553,12 +570,13 @@ define i1 @call_attribute() {
 }
 
 define i1 @umin(i32 %a, i32 %b) {
-; CHECK-LABEL: @umin(
+; CHECK-LABEL: define i1 @umin
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[B:%.*]], 20
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[B]], 20
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[B_GUARD:%.*]], label [[OUT]]
 ; CHECK:       b_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ult i32 [[A]], [[B]]
@@ -585,12 +603,13 @@ out:
 }
 
 define i1 @smin(i32 %a, i32 %b) {
-; CHECK-LABEL: @smin(
+; CHECK-LABEL: define i1 @smin
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[B:%.*]], 20
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[B]], 20
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[B_GUARD:%.*]], label [[OUT]]
 ; CHECK:       b_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ule i32 [[A]], [[B]]
@@ -617,12 +636,13 @@ out:
 }
 
 define i1 @smax(i32 %a, i32 %b) {
-; CHECK-LABEL: @smax(
+; CHECK-LABEL: define i1 @smax
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[B:%.*]], 20
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[B]], 20
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[B_GUARD:%.*]], label [[OUT]]
 ; CHECK:       b_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp uge i32 [[A]], [[B]]
@@ -649,12 +669,13 @@ out:
 }
 
 define i1 @umax(i32 %a, i32 %b) {
-; CHECK-LABEL: @umax(
+; CHECK-LABEL: define i1 @umax
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[B:%.*]], 20
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[B]], 20
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[B_GUARD:%.*]], label [[OUT]]
 ; CHECK:       b_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp uge i32 [[A]], [[B]]
@@ -681,8 +702,9 @@ out:
 }
 
 define i1 @umin_lhs_overdefined_rhs_const(i32 %a) {
-; CHECK-LABEL: @umin_lhs_overdefined_rhs_const(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 42
+; CHECK-LABEL: define i1 @umin_lhs_overdefined_rhs_const
+; CHECK-SAME: (i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A]], 42
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[A]], i32 42
 ; CHECK-NEXT:    ret i1 true
 ;
@@ -693,8 +715,9 @@ define i1 @umin_lhs_overdefined_rhs_const(i32 %a) {
 }
 
 define i1 @umin_rhs_overdefined_lhs_const(i32 %a) {
-; CHECK-LABEL: @umin_rhs_overdefined_lhs_const(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[A:%.*]], 42
+; CHECK-LABEL: define i1 @umin_rhs_overdefined_lhs_const
+; CHECK-SAME: (i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[A]], 42
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 42, i32 [[A]]
 ; CHECK-NEXT:    ret i1 true
 ;
@@ -705,10 +728,11 @@ define i1 @umin_rhs_overdefined_lhs_const(i32 %a) {
 }
 
 define i1 @umin_lhs_overdefined_rhs_range(i32 %a, i32 %b) {
-; CHECK-LABEL: @umin_lhs_overdefined_rhs_range(
-; CHECK-NEXT:    [[ASSUME:%.*]] = icmp ult i32 [[B:%.*]], 42
+; CHECK-LABEL: define i1 @umin_lhs_overdefined_rhs_range
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[ASSUME:%.*]] = icmp ult i32 [[B]], 42
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[ASSUME]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A]], [[B]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[A]], i32 [[B]]
 ; CHECK-NEXT:    ret i1 true
 ;
@@ -721,10 +745,11 @@ define i1 @umin_lhs_overdefined_rhs_range(i32 %a, i32 %b) {
 }
 
 define i1 @umin_rhs_overdefined_lhs_range(i32 %a, i32 %b) {
-; CHECK-LABEL: @umin_rhs_overdefined_lhs_range(
-; CHECK-NEXT:    [[ASSUME:%.*]] = icmp ult i32 [[B:%.*]], 42
+; CHECK-LABEL: define i1 @umin_rhs_overdefined_lhs_range
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[ASSUME:%.*]] = icmp ult i32 [[B]], 42
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[ASSUME]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[A]], [[B]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[B]], i32 [[A]]
 ; CHECK-NEXT:    ret i1 true
 ;
@@ -737,9 +762,10 @@ define i1 @umin_rhs_overdefined_lhs_range(i32 %a, i32 %b) {
 }
 
 define i1 @clamp_low1(i32 noundef %a) {
-; CHECK-LABEL: @clamp_low1(
+; CHECK-LABEL: define i1 @clamp_low1
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp eq i32 [[A]], 5
@@ -764,9 +790,10 @@ out:
 }
 
 define i1 @clamp_low2(i32 noundef %a) {
-; CHECK-LABEL: @clamp_low2(
+; CHECK-LABEL: define i1 @clamp_low2
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ne i32 [[A]], 5
@@ -791,9 +818,10 @@ out:
 }
 
 define i1 @clamp_low3(i32 noundef %a) {
-; CHECK-LABEL: @clamp_low3(
+; CHECK-LABEL: define i1 @clamp_low3
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ugt i32 [[A]], 5
@@ -818,9 +846,10 @@ out:
 }
 
 define i1 @clamp_low4(i32 noundef %a) {
-; CHECK-LABEL: @clamp_low4(
+; CHECK-LABEL: define i1 @clamp_low4
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ule i32 [[A]], 5
@@ -845,9 +874,10 @@ out:
 }
 
 define i1 @clamp_high1(i32 noundef %a) {
-; CHECK-LABEL: @clamp_high1(
+; CHECK-LABEL: define i1 @clamp_high1
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp eq i32 [[A]], 5
@@ -872,9 +902,10 @@ out:
 }
 
 define i1 @clamp_high1_or(i32 noundef %a) {
-; CHECK-LABEL: @clamp_high1_or(
+; CHECK-LABEL: define i1 @clamp_high1_or
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp eq i32 [[A]], 5
@@ -899,9 +930,10 @@ out:
 }
 
 define i1 @clamp_high2(i32 noundef %a) {
-; CHECK-LABEL: @clamp_high2(
+; CHECK-LABEL: define i1 @clamp_high2
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ne i32 [[A]], 5
@@ -927,9 +959,10 @@ out:
 
 
 define i1 @clamp_high2_or_disjoint(i32 noundef %a) {
-; CHECK-LABEL: @clamp_high2_or_disjoint(
+; CHECK-LABEL: define i1 @clamp_high2_or_disjoint
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ne i32 [[A]], 5
@@ -955,9 +988,10 @@ out:
 
 
 define i1 @clamp_high3(i32 noundef %a) {
-; CHECK-LABEL: @clamp_high3(
+; CHECK-LABEL: define i1 @clamp_high3
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp slt i32 [[A]], 5
@@ -982,9 +1016,10 @@ out:
 }
 
 define i1 @clamp_high4(i32 noundef %a) {
-; CHECK-LABEL: @clamp_high4(
+; CHECK-LABEL: define i1 @clamp_high4
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp sge i32 [[A]], 5
@@ -1010,9 +1045,10 @@ out:
 
 ; Just showing arbitrary constants work, not really a clamp
 define i1 @not_clamp_high(i32 noundef %a) {
-; CHECK-LABEL: @not_clamp_high(
+; CHECK-LABEL: define i1 @not_clamp_high
+; CHECK-SAME: (i32 noundef [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A]], 5
 ; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
 ; CHECK:       a_guard:
 ; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ne i32 [[A]], 5
@@ -1037,9 +1073,10 @@ out:
 }
 
 define void @abs1(i32 %a, ptr %p) {
-; CHECK-LABEL: @abs1(
+; CHECK-LABEL: define void @abs1
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A]], 10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[A]], -20
 ; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
 ; CHECK-NEXT:    br i1 [[AND]], label [[GUARD:%.*]], label [[EXIT:%.*]]
@@ -1047,7 +1084,7 @@ define void @abs1(i32 %a, ptr %p) {
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[A]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
 ; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[A]]
-; CHECK-NEXT:    store i1 true, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[ABS]], 19
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
@@ -1082,9 +1119,10 @@ exit:
 }
 
 define void @abs2(i32 %a, ptr %p) {
-; CHECK-LABEL: @abs2(
+; CHECK-LABEL: define void @abs2
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A]], 10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[A]], -20
 ; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
 ; CHECK-NEXT:    br i1 [[AND]], label [[GUARD:%.*]], label [[EXIT:%.*]]
@@ -1092,7 +1130,7 @@ define void @abs2(i32 %a, ptr %p) {
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[A]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A]], 0
 ; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[A]], i32 [[SUB]]
-; CHECK-NEXT:    store i1 true, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[ABS]], 19
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
@@ -1127,9 +1165,10 @@ exit:
 }
 
 define void @nabs1(i32 %a, ptr %p) {
-; CHECK-LABEL: @nabs1(
+; CHECK-LABEL: define void @nabs1
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A]], 10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[A]], -20
 ; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
 ; CHECK-NEXT:    br i1 [[AND]], label [[GUARD:%.*]], label [[EXIT:%.*]]
@@ -1137,7 +1176,7 @@ define void @nabs1(i32 %a, ptr %p) {
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[A]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], 0
 ; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[A]]
-; CHECK-NEXT:    store i1 true, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[NABS]], -19
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
@@ -1172,9 +1211,10 @@ exit:
 }
 
 define void @nabs2(i32 %a, ptr %p) {
-; CHECK-LABEL: @nabs2(
+; CHECK-LABEL: define void @nabs2
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A]], 10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[A]], -20
 ; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
 ; CHECK-NEXT:    br i1 [[AND]], label [[GUARD:%.*]], label [[EXIT:%.*]]
@@ -1182,7 +1222,7 @@ define void @nabs2(i32 %a, ptr %p) {
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[A]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
 ; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i32 [[A]], i32 [[SUB]]
-; CHECK-NEXT:    store i1 true, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[NABS]], -19
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
@@ -1217,9 +1257,10 @@ exit:
 }
 
 define i1 @zext_unknown(i8 %a) {
-; CHECK-LABEL: @zext_unknown(
+; CHECK-LABEL: define i1 @zext_unknown
+; CHECK-SAME: (i8 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A32:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[A32:%.*]] = zext i8 [[A]] to i32
 ; CHECK-NEXT:    ret i1 true
 ;
 entry:
@@ -1229,9 +1270,10 @@ entry:
 }
 
 define i1 @trunc_unknown(i32 %a) {
-; CHECK-LABEL: @trunc_unknown(
+; CHECK-LABEL: define i1 @trunc_unknown
+; CHECK-SAME: (i32 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A8:%.*]] = trunc i32 [[A:%.*]] to i8
+; CHECK-NEXT:    [[A8:%.*]] = trunc i32 [[A]] to i8
 ; CHECK-NEXT:    [[A32:%.*]] = sext i8 [[A8]] to i32
 ; CHECK-NEXT:    ret i1 true
 ;
@@ -1243,12 +1285,13 @@ entry:
 }
 
 define void @trunc_icmp_ule(i32 %x, ptr %p) {
-; CHECK-LABEL: @trunc_icmp_ule(
-; CHECK-NEXT:    [[T:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-LABEL: define void @trunc_icmp_ule
+; CHECK-SAME: (i32 [[X:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[T:%.*]] = trunc i32 [[X]] to i8
 ; CHECK-NEXT:    [[C:%.*]] = icmp uge i8 [[T]], 5
 ; CHECK-NEXT:    br i1 [[C]], label [[TRUE:%.*]], label [[FALSE:%.*]]
 ; CHECK:       true:
-; CHECK-NEXT:    store i1 true, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i32 [[X]], 5
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[C3:%.*]] = icmp ule i32 [[X]], 5
@@ -1294,12 +1337,13 @@ false:
 }
 
 define void @trunc_icmp_eq(i32 %x, ptr %p) {
-; CHECK-LABEL: @trunc_icmp_eq(
-; CHECK-NEXT:    [[T:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-LABEL: define void @trunc_icmp_eq
+; CHECK-SAME: (i32 [[X:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[T:%.*]] = trunc i32 [[X]] to i8
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[T]], 5
 ; CHECK-NEXT:    br i1 [[C]], label [[TRUE:%.*]], label [[FALSE:%.*]]
 ; CHECK:       true:
-; CHECK-NEXT:    store i1 true, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i32 [[X]], 5
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[C3:%.*]] = icmp ule i32 [[X]], 5
@@ -1347,9 +1391,10 @@ false:
 ; TODO: missed optimization
 ; Make sure we exercise non-integer inputs to unary operators (i.e. crash check).
 define i1 @bitcast_unknown(float %a) {
-; CHECK-LABEL: @bitcast_unknown(
+; CHECK-LABEL: define i1 @bitcast_unknown
+; CHECK-SAME: (float [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A32:%.*]] = bitcast float [[A:%.*]] to i32
+; CHECK-NEXT:    [[A32:%.*]] = bitcast float [[A]] to i32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A32]], 128
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1360,9 +1405,10 @@ entry:
 }
 
 define i1 @bitcast_unknown2(ptr %p) {
-; CHECK-LABEL: @bitcast_unknown2(
+; CHECK-LABEL: define i1 @bitcast_unknown2
+; CHECK-SAME: (ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[P64:%.*]] = ptrtoint ptr [[P:%.*]] to i64
+; CHECK-NEXT:    [[P64:%.*]] = ptrtoint ptr [[P]] to i64
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[P64]], 128
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1374,9 +1420,10 @@ entry:
 
 
 define i1 @and_unknown(i32 %a) {
-; CHECK-LABEL: @and_unknown(
+; CHECK-LABEL: define i1 @and_unknown
+; CHECK-SAME: (i32 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 128
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A]], 128
 ; CHECK-NEXT:    ret i1 true
 ;
 entry:
@@ -1386,9 +1433,10 @@ entry:
 }
 
 define i1 @lshr_unknown(i32 %a) {
-; CHECK-LABEL: @lshr_unknown(
+; CHECK-LABEL: define i1 @lshr_unknown
+; CHECK-SAME: (i32 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = lshr i32 [[A:%.*]], 30
+; CHECK-NEXT:    [[AND:%.*]] = lshr i32 [[A]], 30
 ; CHECK-NEXT:    ret i1 true
 ;
 entry:
@@ -1398,9 +1446,10 @@ entry:
 }
 
 define i1 @urem_unknown(i32 %a) {
-; CHECK-LABEL: @urem_unknown(
+; CHECK-LABEL: define i1 @urem_unknown
+; CHECK-SAME: (i32 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[UREM:%.*]] = urem i32 [[A:%.*]], 30
+; CHECK-NEXT:    [[UREM:%.*]] = urem i32 [[A]], 30
 ; CHECK-NEXT:    ret i1 true
 ;
 entry:
@@ -1410,9 +1459,10 @@ entry:
 }
 
 define i1 @srem_unknown(i32 %a) {
-; CHECK-LABEL: @srem_unknown(
+; CHECK-LABEL: define i1 @srem_unknown
+; CHECK-SAME: (i32 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SREM:%.*]] = srem i32 [[A:%.*]], 30
+; CHECK-NEXT:    [[SREM:%.*]] = srem i32 [[A]], 30
 ; CHECK-NEXT:    br i1 undef, label [[EXIT1:%.*]], label [[EXIT2:%.*]]
 ; CHECK:       exit1:
 ; CHECK-NEXT:    ret i1 true
@@ -1431,9 +1481,10 @@ exit2:
 }
 
 define i1 @sdiv_unknown(i32 %a) {
-; CHECK-LABEL: @sdiv_unknown(
+; CHECK-LABEL: define i1 @sdiv_unknown
+; CHECK-SAME: (i32 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SREM:%.*]] = sdiv i32 [[A:%.*]], 123
+; CHECK-NEXT:    [[SREM:%.*]] = sdiv i32 [[A]], 123
 ; CHECK-NEXT:    br i1 undef, label [[EXIT1:%.*]], label [[EXIT2:%.*]]
 ; CHECK:       exit1:
 ; CHECK-NEXT:    ret i1 true
@@ -1452,9 +1503,10 @@ exit2:
 }
 
 define i1 @uadd_sat_unknown(i32 %a) {
-; CHECK-LABEL: @uadd_sat_unknown(
+; CHECK-LABEL: define i1 @uadd_sat_unknown
+; CHECK-SAME: (i32 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[A:%.*]], i32 100)
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[A]], i32 100)
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[VAL]], 100
 ; CHECK-NEXT:    br i1 undef, label [[EXIT1:%.*]], label [[EXIT2:%.*]]
 ; CHECK:       exit1:
@@ -1474,9 +1526,10 @@ exit2:
 }
 
 define i1 @usub_sat_unknown(i32 %a) {
-; CHECK-LABEL: @usub_sat_unknown(
+; CHECK-LABEL: define i1 @usub_sat_unknown
+; CHECK-SAME: (i32 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[A:%.*]], i32 100)
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[A]], i32 100)
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[VAL]], -101
 ; CHECK-NEXT:    br i1 undef, label [[EXIT1:%.*]], label [[EXIT2:%.*]]
 ; CHECK:       exit1:
@@ -1496,9 +1549,10 @@ exit2:
 }
 
 define i1 @sadd_sat_unknown(i32 %a) {
-; CHECK-LABEL: @sadd_sat_unknown(
+; CHECK-LABEL: define i1 @sadd_sat_unknown
+; CHECK-SAME: (i32 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[A:%.*]], i32 100)
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[A]], i32 100)
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[VAL]], -2147483548
 ; CHECK-NEXT:    br i1 undef, label [[EXIT1:%.*]], label [[EXIT2:%.*]]
 ; CHECK:       exit1:
@@ -1518,9 +1572,10 @@ exit2:
 }
 
 define i1 @ssub_sat_unknown(i32 %a) {
-; CHECK-LABEL: @ssub_sat_unknown(
+; CHECK-LABEL: define i1 @ssub_sat_unknown
+; CHECK-SAME: (i32 [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[A:%.*]], i32 100)
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[A]], i32 100)
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[VAL]], 2147483547
 ; CHECK-NEXT:    br i1 undef, label [[EXIT1:%.*]], label [[EXIT2:%.*]]
 ; CHECK:       exit1:
@@ -1540,14 +1595,15 @@ exit2:
 }
 
 define void @select_and(i32 %a, ptr %p) {
-; CHECK-LABEL: @select_and(
+; CHECK-LABEL: define void @select_and
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[A:%.*]], -10
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[A]], -10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[A]], 10
 ; CHECK-NEXT:    [[AND:%.*]] = select i1 [[CMP1]], i1 [[CMP2]], i1 false
 ; CHECK-NEXT:    br i1 [[AND]], label [[GUARD:%.*]], label [[EXIT:%.*]]
 ; CHECK:       guard:
-; CHECK-NEXT:    store i1 false, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 false, ptr [[P]], align 1
 ; CHECK-NEXT:    store i1 false, ptr [[P]], align 1
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -1571,15 +1627,16 @@ exit:
 }
 
 define void @select_and_wrong_const(i32 %a, ptr %p) {
-; CHECK-LABEL: @select_and_wrong_const(
+; CHECK-LABEL: define void @select_and_wrong_const
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[A:%.*]], -10
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[A]], -10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[A]], 10
 ; CHECK-NEXT:    [[AND:%.*]] = select i1 [[CMP1]], i1 [[CMP2]], i1 true
 ; CHECK-NEXT:    br i1 [[AND]], label [[GUARD:%.*]], label [[EXIT:%.*]]
 ; CHECK:       guard:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A]], 20
-; CHECK-NEXT:    store i1 [[C1]], ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 [[C1]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[A]], -20
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    br label [[EXIT]]
@@ -1604,15 +1661,16 @@ exit:
 }
 
 define void @select_and_wrong_operand(i32 %a, ptr %p) {
-; CHECK-LABEL: @select_and_wrong_operand(
+; CHECK-LABEL: define void @select_and_wrong_operand
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[A:%.*]], -10
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[A]], -10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[A]], 10
 ; CHECK-NEXT:    [[AND:%.*]] = select i1 [[CMP1]], i1 false, i1 [[CMP2]]
 ; CHECK-NEXT:    br i1 [[AND]], label [[GUARD:%.*]], label [[EXIT:%.*]]
 ; CHECK:       guard:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A]], 20
-; CHECK-NEXT:    store i1 [[C1]], ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 [[C1]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[A]], -20
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    br label [[EXIT]]
@@ -1637,14 +1695,15 @@ exit:
 }
 
 define void @select_or(i32 %a, ptr %p) {
-; CHECK-LABEL: @select_or(
+; CHECK-LABEL: define void @select_or
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], -10
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A]], -10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[A]], 10
 ; CHECK-NEXT:    [[OR:%.*]] = select i1 [[CMP1]], i1 true, i1 [[CMP2]]
 ; CHECK-NEXT:    br i1 [[OR]], label [[EXIT:%.*]], label [[GUARD:%.*]]
 ; CHECK:       guard:
-; CHECK-NEXT:    store i1 false, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 false, ptr [[P]], align 1
 ; CHECK-NEXT:    store i1 false, ptr [[P]], align 1
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -1668,15 +1727,16 @@ exit:
 }
 
 define void @select_or_wrong_const(i32 %a, ptr %p) {
-; CHECK-LABEL: @select_or_wrong_const(
+; CHECK-LABEL: define void @select_or_wrong_const
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], -10
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A]], -10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[A]], 10
 ; CHECK-NEXT:    [[OR:%.*]] = select i1 [[CMP1]], i1 false, i1 [[CMP2]]
 ; CHECK-NEXT:    br i1 [[OR]], label [[EXIT:%.*]], label [[GUARD:%.*]]
 ; CHECK:       guard:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A]], 20
-; CHECK-NEXT:    store i1 [[C1]], ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 [[C1]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[A]], -20
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    br label [[EXIT]]
@@ -1701,15 +1761,16 @@ exit:
 }
 
 define void @select_or_wrong_operand(i32 %a, ptr %p) {
-; CHECK-LABEL: @select_or_wrong_operand(
+; CHECK-LABEL: define void @select_or_wrong_operand
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], -10
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A]], -10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[A]], 10
 ; CHECK-NEXT:    [[OR:%.*]] = select i1 [[CMP1]], i1 [[CMP2]], i1 true
 ; CHECK-NEXT:    br i1 [[OR]], label [[EXIT:%.*]], label [[GUARD:%.*]]
 ; CHECK:       guard:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A]], 20
-; CHECK-NEXT:    store i1 [[C1]], ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 [[C1]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[A]], -20
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    br label [[EXIT]]
@@ -1734,13 +1795,14 @@ exit:
 }
 
 define void @or_union(i32 %a, ptr %p) {
-; CHECK-LABEL: @or_union(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], 10
+; CHECK-LABEL: define void @or_union
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A]], 10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[A]], 12
 ; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
 ; CHECK-NEXT:    br i1 [[OR]], label [[GUARD:%.*]], label [[EXIT:%.*]]
 ; CHECK:       guard:
-; CHECK-NEXT:    store i1 false, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 false, ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i32 [[A]], 10
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i32 [[A]], 11
@@ -1775,9 +1837,10 @@ exit:
 }
 
 define i1 @or_union_unknown_cond(i32 %a, i1 %c) {
-; CHECK-LABEL: @or_union_unknown_cond(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], 10
-; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[C:%.*]]
+; CHECK-LABEL: define i1 @or_union_unknown_cond
+; CHECK-SAME: (i32 [[A:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A]], 10
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[C]]
 ; CHECK-NEXT:    br i1 [[OR]], label [[GUARD:%.*]], label [[EXIT:%.*]]
 ; CHECK:       guard:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[A]], 10
@@ -1798,13 +1861,14 @@ exit:
 }
 
 define void @and_union(i32 %a, ptr %p) {
-; CHECK-LABEL: @and_union(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[A:%.*]], 10
+; CHECK-LABEL: define void @and_union
+; CHECK-SAME: (i32 [[A:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[A]], 10
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[A]], 12
 ; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
 ; CHECK-NEXT:    br i1 [[AND]], label [[EXIT:%.*]], label [[GUARD:%.*]]
 ; CHECK:       guard:
-; CHECK-NEXT:    store i1 false, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 false, ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i32 [[A]], 10
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i32 [[A]], 11
@@ -1839,9 +1903,10 @@ exit:
 }
 
 define i1 @and_union_unknown_cond(i32 %a, i1 %c) {
-; CHECK-LABEL: @and_union_unknown_cond(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[A:%.*]], 10
-; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[C:%.*]]
+; CHECK-LABEL: define i1 @and_union_unknown_cond
+; CHECK-SAME: (i32 [[A:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[A]], 10
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[C]]
 ; CHECK-NEXT:    br i1 [[AND]], label [[EXIT:%.*]], label [[GUARD:%.*]]
 ; CHECK:       guard:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[A]], 10
@@ -1862,14 +1927,15 @@ exit:
 }
 
 define void @select_assume(i32 %a, i32 %b, i1 %c, ptr %p) {
-; CHECK-LABEL: @select_assume(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[A:%.*]], 10
+; CHECK-LABEL: define void @select_assume
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i1 [[C:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[A]], 10
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[C1]])
-; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[B:%.*]], 20
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[B]], 20
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[C2]])
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[C:%.*]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i32 [[A]], i32 [[B]]
 ; CHECK-NEXT:    [[C3:%.*]] = icmp ult i32 [[S]], 19
-; CHECK-NEXT:    store i1 [[C3]], ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 [[C3]], ptr [[P]], align 1
 ; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
 ; CHECK-NEXT:    ret void
 ;
@@ -1886,10 +1952,11 @@ define void @select_assume(i32 %a, i32 %b, i1 %c, ptr %p) {
 }
 
 define void @xor(i8 %a, ptr %p) {
-; CHECK-LABEL: @xor(
-; CHECK-NEXT:    [[A_MASK:%.*]] = and i8 [[A:%.*]], 15
+; CHECK-LABEL: define void @xor
+; CHECK-SAME: (i8 [[A:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[A_MASK:%.*]] = and i8 [[A]], 15
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[A_MASK]], -86
-; CHECK-NEXT:    store i1 true, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i8 [[XOR]], -96
 ; CHECK-NEXT:    store i1 [[C2]], ptr [[P]], align 1
 ; CHECK-NEXT:    store i1 true, ptr [[P]], align 1
@@ -1911,8 +1978,9 @@ define void @xor(i8 %a, ptr %p) {
 }
 
 define i1 @xor_neg_cond(i32 %a) {
-; CHECK-LABEL: @xor_neg_cond(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], 10
+; CHECK-LABEL: define i1 @xor_neg_cond
+; CHECK-SAME: (i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A]], 10
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[CMP1]], true
 ; CHECK-NEXT:    br i1 [[XOR]], label [[EXIT:%.*]], label [[GUARD:%.*]]
 ; CHECK:       guard:
@@ -1933,8 +2001,9 @@ exit:
 }
 
 define i1 @xor_approx(i32 %a) {
-; CHECK-LABEL: @xor_approx(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[A:%.*]], 2
+; CHECK-LABEL: define i1 @xor_approx
+; CHECK-SAME: (i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[A]], 2
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[A]], 5
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ugt i32 [[A]], 7
 ; CHECK-NEXT:    [[CMP4:%.*]] = icmp ult i32 [[A]], 9
@@ -1968,8 +2037,9 @@ exit:
 }
 
 define i1 @binop_eval_order(i32 %x) {
-; CHECK-LABEL: @binop_eval_order(
-; CHECK-NEXT:    [[A:%.*]] = add nuw nsw i32 [[X:%.*]], 1
+; CHECK-LABEL: define i1 @binop_eval_order
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = add nuw nsw i32 [[X]], 1
 ; CHECK-NEXT:    [[B:%.*]] = add nuw nsw i32 [[A]], 1
 ; CHECK-NEXT:    [[C:%.*]] = add nuw nsw i32 [[A]], [[B]]
 ; CHECK-NEXT:    ret i1 true
@@ -1981,6 +2051,46 @@ define i1 @binop_eval_order(i32 %x) {
   ret i1 %d
 }
 
+define range(i32 0, 1024) i32 @range_larger(i8 %x) {
+; CHECK-LABEL: define range(i32 0, 1024) i32 @range_larger
+; CHECK-SAME: (i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT:    ret i32 [[ZEXT]]
+;
+  %zext = zext i8 %x to i32
+  ret i32 %zext
+}
+
+define range(i32 0, 128) i32 @range_smaller(i8 %x) {
+; CHECK-LABEL: define range(i32 0, 128) i32 @range_smaller
+; CHECK-SAME: (i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT:    ret i32 [[ZEXT]]
+;
+  %zext = zext i8 %x to i32
+  ret i32 %zext
+}
+
+define range(i32 128, 512) i32 @range_intersect(i8 %x) {
+; CHECK-LABEL: define range(i32 128, 512) i32 @range_intersect
+; CHECK-SAME: (i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT:    ret i32 [[ZEXT]]
+;
+  %zext = zext i8 %x to i32
+  ret i32 %zext
+}
+
+define range(i32 512, 1024) i32 @range_non_overlapping(i8 %x) {
+; CHECK-LABEL: define range(i32 512, 1024) i32 @range_non_overlapping
+; CHECK-SAME: (i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT:    ret i32 [[ZEXT]]
+;
+  %zext = zext i8 %x to i32
+  ret i32 %zext
+}
+
 declare i32 @llvm.uadd.sat.i32(i32, i32)
 declare i32 @llvm.usub.sat.i32(i32, i32)
 declare i32 @llvm.sadd.sat.i32(i32, i32)

From b7b0071680e60c60da9d4d858f944fd95d76fd42 Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov@arm.com>
Date: Fri, 19 Jul 2024 10:18:52 +0100
Subject: [PATCH 166/486] [AArch64][SVE] Improve code quality of vector
 unsigned/signed add reductions. (#97339)

For SVE we don't have to zero extend and sum part of the result before
issuing UADDV instruction. Also this change allows to handle bigger than
a legal vector type more efficiently and lower a fixed-length vector
type to SVE's UADDV where appropriate.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  72 ++++++++++++-
 llvm/test/CodeGen/AArch64/double_reduct.ll    |   9 +-
 llvm/test/CodeGen/AArch64/sve-doublereduct.ll |  41 +++----
 .../CodeGen/AArch64/sve-fixed-vector-zext.ll  |  44 +++++---
 llvm/test/CodeGen/AArch64/sve-int-reduce.ll   |  88 +++++++++++++++
 ...-streaming-mode-fixed-length-reductions.ll | 102 ++++++++++++++++++
 llvm/test/CodeGen/AArch64/vecreduce-add.ll    |  52 +++++----
 7 files changed, 330 insertions(+), 78 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index eef83a845e2c3..668a9acd1f3c6 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -17523,6 +17523,71 @@ static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
   return DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, UADDLP);
 }
 
+// Turn [sign|zero]_extend(vecreduce_add()) into SVE's  SADDV|UADDV
+// instructions.
+static SDValue
+performVecReduceAddExtCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
+                              const AArch64TargetLowering &TLI) {
+  if (N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND &&
+      N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND)
+    return SDValue();
+  bool IsSigned = N->getOperand(0).getOpcode() == ISD::SIGN_EXTEND;
+
+  SelectionDAG &DAG = DCI.DAG;
+  auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
+  SDValue VecOp = N->getOperand(0).getOperand(0);
+  EVT VecOpVT = VecOp.getValueType();
+  SDLoc DL(N);
+
+  // Split the input vectors if not legal, e.g.
+  // i32 (vecreduce_add (zext nxv32i8 %op to nxv32i32))
+  // ->
+  // i32 (add
+  //   (i32 vecreduce_add (zext nxv16i8 %op.lo to nxv16i32)),
+  //   (i32 vecreduce_add (zext nxv16i8 %op.hi to nxv16i32)))
+  if (TLI.getTypeAction(*DAG.getContext(), VecOpVT) ==
+      TargetLowering::TypeSplitVector) {
+    SDValue Lo, Hi;
+    std::tie(Lo, Hi) = DAG.SplitVector(VecOp, DL);
+    unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
+    EVT HalfVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
+        *DAG.getContext());
+    Lo = DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0),
+                     DAG.getNode(ExtOpc, DL, HalfVT, Lo));
+    Hi = DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0),
+                     DAG.getNode(ExtOpc, DL, HalfVT, Hi));
+    return DAG.getNode(ISD::ADD, DL, N->getValueType(0), Lo, Hi);
+  }
+
+  if (!TLI.isTypeLegal(VecOpVT))
+    return SDValue();
+
+  if (VecOpVT.isFixedLengthVector() &&
+      !TLI.useSVEForFixedLengthVectorVT(VecOpVT, !Subtarget.isNeonAvailable()))
+    return SDValue();
+
+  // The input type is legal so map VECREDUCE_ADD to UADDV/SADDV, e.g.
+  // i32 (vecreduce_add (zext nxv16i8 %op to nxv16i32))
+  // ->
+  // i32 (UADDV nxv16i8:%op)
+  EVT ElemType = N->getValueType(0);
+  SDValue Pg = getPredicateForVector(DAG, DL, VecOpVT);
+  if (VecOpVT.isFixedLengthVector()) {
+    EVT ContainerVT = getContainerForFixedLengthVector(DAG, VecOpVT);
+    VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
+  }
+  SDValue Res =
+      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64,
+                  DAG.getConstant(IsSigned ? Intrinsic::aarch64_sve_saddv
+                                           : Intrinsic::aarch64_sve_uaddv,
+                                  DL, MVT::i64),
+                  Pg, VecOp);
+  if (ElemType != MVT::i64)
+    Res = DAG.getAnyExtOrTrunc(Res, DL, ElemType);
+
+  return Res;
+}
+
 // Turn a v8i8/v16i8 extended vecreduce into a udot/sdot and vecreduce
 //   vecreduce.add(ext(A)) to vecreduce.add(DOT(zero, A, one))
 //   vecreduce.add(mul(ext(A), ext(B))) to vecreduce.add(DOT(zero, A, B))
@@ -25208,8 +25273,11 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
     return performInsertVectorEltCombine(N, DCI);
   case ISD::EXTRACT_VECTOR_ELT:
     return performExtractVectorEltCombine(N, DCI, Subtarget);
-  case ISD::VECREDUCE_ADD:
-    return performVecReduceAddCombine(N, DCI.DAG, Subtarget);
+  case ISD::VECREDUCE_ADD: {
+    if (SDValue Val = performVecReduceAddCombine(N, DCI.DAG, Subtarget))
+      return Val;
+    return performVecReduceAddExtCombine(N, DCI, *this);
+  }
   case AArch64ISD::UADDV:
     return performUADDVCombine(N, DAG);
   case AArch64ISD::SMULL:
diff --git a/llvm/test/CodeGen/AArch64/double_reduct.ll b/llvm/test/CodeGen/AArch64/double_reduct.ll
index b10114bc0ffa7..cf5e15da0b173 100644
--- a/llvm/test/CodeGen/AArch64/double_reduct.ll
+++ b/llvm/test/CodeGen/AArch64/double_reduct.ll
@@ -145,11 +145,10 @@ define i16 @add_ext_i16(<16 x i8> %a, <16 x i8> %b) {
 define i16 @add_ext_v32i16(<32 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: add_ext_v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uaddl2 v3.8h, v0.16b, v1.16b
-; CHECK-NEXT:    uaddl v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    add v0.8h, v0.8h, v3.8h
-; CHECK-NEXT:    uadalp v0.8h, v2.16b
-; CHECK-NEXT:    addv h0, v0.8h
+; CHECK-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-NEXT:    uadalp v1.8h, v0.16b
+; CHECK-NEXT:    uadalp v1.8h, v2.16b
+; CHECK-NEXT:    addv h0, v1.8h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
   %ae = zext <32 x i8> %a to <32 x i16>
diff --git a/llvm/test/CodeGen/AArch64/sve-doublereduct.ll b/llvm/test/CodeGen/AArch64/sve-doublereduct.ll
index 7bc31d44bb654..b289dfbec527c 100644
--- a/llvm/test/CodeGen/AArch64/sve-doublereduct.ll
+++ b/llvm/test/CodeGen/AArch64/sve-doublereduct.ll
@@ -103,17 +103,12 @@ define i32 @add_i32(<vscale x 8 x i32> %a, <vscale x 4 x i32> %b) {
 define i16 @add_ext_i16(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: add_ext_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpkhi z2.h, z0.b
-; CHECK-NEXT:    uunpklo z0.h, z0.b
-; CHECK-NEXT:    uunpkhi z3.h, z1.b
-; CHECK-NEXT:    uunpklo z1.h, z1.b
-; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    add z0.h, z0.h, z2.h
-; CHECK-NEXT:    add z1.h, z1.h, z3.h
-; CHECK-NEXT:    add z0.h, z0.h, z1.h
-; CHECK-NEXT:    uaddv d0, p0, z0.h
-; CHECK-NEXT:    fmov x0, d0
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    uaddv d0, p0, z0.b
+; CHECK-NEXT:    uaddv d1, p0, z1.b
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov w9, s1
+; CHECK-NEXT:    add w0, w8, w9
 ; CHECK-NEXT:    ret
   %ae = zext <vscale x 16 x i8> %a to <vscale x 16 x i16>
   %be = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>
@@ -126,21 +121,15 @@ define i16 @add_ext_i16(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 define i16 @add_ext_v32i16(<vscale x 32 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: add_ext_v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpklo z3.h, z1.b
-; CHECK-NEXT:    uunpklo z4.h, z0.b
-; CHECK-NEXT:    uunpkhi z1.h, z1.b
-; CHECK-NEXT:    uunpkhi z0.h, z0.b
-; CHECK-NEXT:    uunpkhi z5.h, z2.b
-; CHECK-NEXT:    uunpklo z2.h, z2.b
-; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    add z0.h, z0.h, z1.h
-; CHECK-NEXT:    add z1.h, z4.h, z3.h
-; CHECK-NEXT:    add z0.h, z1.h, z0.h
-; CHECK-NEXT:    add z1.h, z2.h, z5.h
-; CHECK-NEXT:    add z0.h, z0.h, z1.h
-; CHECK-NEXT:    uaddv d0, p0, z0.h
-; CHECK-NEXT:    fmov x0, d0
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    uaddv d1, p0, z1.b
+; CHECK-NEXT:    uaddv d0, p0, z0.b
+; CHECK-NEXT:    uaddv d2, p0, z2.b
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    add w8, w9, w8
+; CHECK-NEXT:    fmov w9, s2
+; CHECK-NEXT:    add w0, w8, w9
 ; CHECK-NEXT:    ret
   %ae = zext <vscale x 32 x i8> %a to <vscale x 32 x i16>
   %be = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-vector-zext.ll b/llvm/test/CodeGen/AArch64/sve-fixed-vector-zext.ll
index 1ab2589bccd5b..24c817d410301 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-vector-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-vector-zext.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 
 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v1 -O3 -aarch64-sve-vector-bits-min=256 -verify-machineinstrs | FileCheck %s --check-prefixes=SVE256
 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v1 -O3 -aarch64-sve-vector-bits-min=128 -verify-machineinstrs | FileCheck %s --check-prefixes=NEON
@@ -6,24 +7,31 @@
 
 define internal i32 @test(ptr nocapture readonly %p1, i32 %i1, ptr nocapture readonly %p2, i32 %i2) {
 ; SVE256-LABEL: test:
-; SVE256:       ld1b    { z0.h }, p0/z,
-; SVE256:       ld1b    { z1.h }, p0/z,
-; SVE256:       sub z0.h, z0.h, z1.h
-; SVE256-NEXT:  sunpklo z1.s, z0.h
-; SVE256-NEXT:  ext z0.b, z0.b, z0.b, #16
-; SVE256-NEXT:  sunpklo z0.s, z0.h
-; SVE256-NEXT:  add z0.s, z1.s, z0.s
-; SVE256-NEXT:  uaddv   d0, p1, z0.s
+; SVE256:       // %bb.0: // %L.entry
+; SVE256-NEXT:    ptrue p0.h, vl16
+; SVE256-NEXT:    mov w9, wzr
+; SVE256-NEXT:    mov w10, wzr
+; SVE256-NEXT:    mov w8, wzr
+; SVE256-NEXT:    mov w11, #-16 // =0xfffffff0
+; SVE256-NEXT:    .p2align 5, , 16
+; SVE256-NEXT:  .LBB0_1: // %L1
+; SVE256-NEXT:    // =>This Inner Loop Header: Depth=1
+; SVE256-NEXT:    sxtw x12, w9
+; SVE256-NEXT:    sxtw x13, w10
+; SVE256-NEXT:    adds w11, w11, #1
+; SVE256-NEXT:    add w10, w10, w3
+; SVE256-NEXT:    ld1b { z0.h }, p0/z, [x0, x12]
+; SVE256-NEXT:    ld1b { z1.h }, p0/z, [x2, x13]
+; SVE256-NEXT:    add w9, w9, w1
+; SVE256-NEXT:    sub z0.h, z0.h, z1.h
+; SVE256-NEXT:    saddv d0, p0, z0.h
+; SVE256-NEXT:    fmov w12, s0
+; SVE256-NEXT:    add w8, w12, w8
+; SVE256-NEXT:    b.lo .LBB0_1
+; SVE256-NEXT:  // %bb.2: // %L2
+; SVE256-NEXT:    mov w0, w8
+; SVE256-NEXT:    ret
 
-; NEON-LABEL: test:
-; NEON:       ldr q0, [x0, w9, sxtw]
-; NEON:       ldr q1, [x2, w10, sxtw]
-; NEON:       usubl2  v2.8h, v0.16b, v1.16b
-; NEON-NEXT:  usubl   v0.8h, v0.8b, v1.8b
-; NEON:       saddl2  v1.4s, v0.8h, v2.8h
-; NEON-NEXT:  saddl   v0.4s, v0.4h, v2.4h
-; NEON-NEXT:  add v0.4s, v0.4s, v1.4s
-; NEON-NEXT:  addv    s0, v0.4s
 
 L.entry:
   br label %L1
@@ -55,3 +63,5 @@ L2:                                          ; preds = %L1
 }
 
 declare  i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; NEON: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/sve-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve-int-reduce.ll
index 8c1b5225b7f25..c8dd719aa03c6 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-reduce.ll
@@ -188,6 +188,94 @@ define i64 @uaddv_nxv2i64(<vscale x 2 x i64> %a) {
   ret i64 %res
 }
 
+define i32 @uaddv_nxv16i8_nxv16i32(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: uaddv_nxv16i8_nxv16i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    uaddv d0, p0, z0.b
+; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %1 = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> %1)
+  ret i32 %2
+}
+
+define i64 @uaddv_nxv16i16_nxv16i64(<vscale x 16 x i16> %a) {
+; CHECK-LABEL: uaddv_nxv16i16_nxv16i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    uaddv d1, p0, z1.h
+; CHECK-NEXT:    uaddv d0, p0, z0.h
+; CHECK-NEXT:    fmov x8, d1
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    add x0, x9, x8
+; CHECK-NEXT:    ret
+  %1 = zext <vscale x 16 x i16> %a to <vscale x 16 x i64>
+  %2 = call i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64> %1)
+  ret i64 %2
+}
+
+define i32 @uaddv_nxv16i16_nxv16i32(<vscale x 32 x i16> %a) {
+; CHECK-LABEL: uaddv_nxv16i16_nxv16i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    uaddv d3, p0, z3.h
+; CHECK-NEXT:    uaddv d2, p0, z2.h
+; CHECK-NEXT:    uaddv d1, p0, z1.h
+; CHECK-NEXT:    uaddv d0, p0, z0.h
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s2
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w11, s0
+; CHECK-NEXT:    add w8, w9, w8
+; CHECK-NEXT:    add w9, w11, w10
+; CHECK-NEXT:    add w0, w9, w8
+; CHECK-NEXT:    ret
+  %1 = zext <vscale x 32 x i16> %a to <vscale x 32 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.nxv32i64(<vscale x 32 x i32> %1)
+  ret i32 %2
+}
+
+define i32 @saddv_nxv16i8_nxv16i32(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: saddv_nxv16i8_nxv16i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    saddv d0, p0, z0.b
+; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %1 = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> %1)
+  ret i32 %2
+}
+
+define i32 @uaddv_nxv32i16_nxv32i32(ptr %a) {
+; CHECK-LABEL: uaddv_nxv32i16_nxv32i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT:    ld1h { z2.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1h { z3.h }, p0/z, [x0]
+; CHECK-NEXT:    uaddv d0, p0, z0.h
+; CHECK-NEXT:    uaddv d1, p0, z1.h
+; CHECK-NEXT:    uaddv d2, p0, z2.h
+; CHECK-NEXT:    uaddv d3, p0, z3.h
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov w9, s1
+; CHECK-NEXT:    fmov w10, s2
+; CHECK-NEXT:    fmov w11, s3
+; CHECK-NEXT:    add w8, w9, w8
+; CHECK-NEXT:    add w9, w11, w10
+; CHECK-NEXT:    add w0, w9, w8
+; CHECK-NEXT:    ret
+  %1 = load <vscale x 32 x i16>, ptr %a, align 16
+  %2 = zext <vscale x 32 x i16> %1 to <vscale x 32 x i32>
+  %3 = call i32 @llvm.vector.reduce.add.nxv32i32(<vscale x 32 x i32> %2)
+  ret i32 %3
+}
+
 ; UMINV
 
 define i8 @umin_nxv16i8(<vscale x 16 x i8> %a) {
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll
new file mode 100644
index 0000000000000..608b3bdeac75a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mattr=+sve < %s | FileCheck %s -check-prefixes=CHECK,NO_STREAMING
+; RUN: llc -mattr=+sve -force-streaming-compatible -aarch64-sve-vector-bits-min=128 -aarch64-sve-vector-bits-max=128  < %s | FileCheck %s -check-prefixes=CHECK,SVE_128
+; RUN: llc -mattr=+sve -force-streaming-compatible -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,SVE_MIN_256
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define i32 @reduce_uadd_v16i8(<32 x i8> %a) #0 {
+; NO_STREAMING-LABEL: reduce_uadd_v16i8:
+; NO_STREAMING:       // %bb.0:
+; NO_STREAMING-NEXT:    ushll2 v2.8h, v1.16b, #0
+; NO_STREAMING-NEXT:    ushll v1.8h, v1.8b, #0
+; NO_STREAMING-NEXT:    ushll2 v3.8h, v0.16b, #0
+; NO_STREAMING-NEXT:    ushll v0.8h, v0.8b, #0
+; NO_STREAMING-NEXT:    uaddl2 v4.4s, v1.8h, v2.8h
+; NO_STREAMING-NEXT:    uaddl v1.4s, v1.4h, v2.4h
+; NO_STREAMING-NEXT:    uaddl2 v2.4s, v0.8h, v3.8h
+; NO_STREAMING-NEXT:    uaddl v0.4s, v0.4h, v3.4h
+; NO_STREAMING-NEXT:    add v1.4s, v1.4s, v4.4s
+; NO_STREAMING-NEXT:    add v0.4s, v0.4s, v2.4s
+; NO_STREAMING-NEXT:    add v0.4s, v0.4s, v1.4s
+; NO_STREAMING-NEXT:    addv s0, v0.4s
+; NO_STREAMING-NEXT:    fmov w0, s0
+; NO_STREAMING-NEXT:    ret
+;
+; SVE_128-LABEL: reduce_uadd_v16i8:
+; SVE_128:       // %bb.0:
+; SVE_128-NEXT:    ptrue p0.b
+; SVE_128-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE_128-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE_128-NEXT:    uaddv d1, p0, z1.b
+; SVE_128-NEXT:    uaddv d0, p0, z0.b
+; SVE_128-NEXT:    fmov x8, d1
+; SVE_128-NEXT:    fmov x9, d0
+; SVE_128-NEXT:    add w0, w9, w8
+; SVE_128-NEXT:    ret
+;
+; SVE_MIN_256-LABEL: reduce_uadd_v16i8:
+; SVE_MIN_256:       // %bb.0:
+; SVE_MIN_256-NEXT:    ptrue p0.b, vl16
+; SVE_MIN_256-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE_MIN_256-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE_MIN_256-NEXT:    splice z0.b, p0, z0.b, z1.b
+; SVE_MIN_256-NEXT:    ptrue p0.b, vl32
+; SVE_MIN_256-NEXT:    uaddv d0, p0, z0.b
+; SVE_MIN_256-NEXT:    fmov x0, d0
+; SVE_MIN_256-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; SVE_MIN_256-NEXT:    ret
+  %1 = zext <32 x i8> %a to <32 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.v16i32(<32 x i32> %1)
+  ret i32 %2
+}
+
+define i32 @reduce_sadd_v16i8(<32 x i8> %a) #0 {
+; NO_STREAMING-LABEL: reduce_sadd_v16i8:
+; NO_STREAMING:       // %bb.0:
+; NO_STREAMING-NEXT:    sshll2 v2.8h, v1.16b, #0
+; NO_STREAMING-NEXT:    sshll v1.8h, v1.8b, #0
+; NO_STREAMING-NEXT:    sshll2 v3.8h, v0.16b, #0
+; NO_STREAMING-NEXT:    sshll v0.8h, v0.8b, #0
+; NO_STREAMING-NEXT:    saddl2 v4.4s, v1.8h, v2.8h
+; NO_STREAMING-NEXT:    saddl v1.4s, v1.4h, v2.4h
+; NO_STREAMING-NEXT:    saddl2 v2.4s, v0.8h, v3.8h
+; NO_STREAMING-NEXT:    saddl v0.4s, v0.4h, v3.4h
+; NO_STREAMING-NEXT:    add v1.4s, v1.4s, v4.4s
+; NO_STREAMING-NEXT:    add v0.4s, v0.4s, v2.4s
+; NO_STREAMING-NEXT:    add v0.4s, v0.4s, v1.4s
+; NO_STREAMING-NEXT:    addv s0, v0.4s
+; NO_STREAMING-NEXT:    fmov w0, s0
+; NO_STREAMING-NEXT:    ret
+;
+; SVE_128-LABEL: reduce_sadd_v16i8:
+; SVE_128:       // %bb.0:
+; SVE_128-NEXT:    ptrue p0.b
+; SVE_128-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE_128-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE_128-NEXT:    saddv d1, p0, z1.b
+; SVE_128-NEXT:    saddv d0, p0, z0.b
+; SVE_128-NEXT:    fmov x8, d1
+; SVE_128-NEXT:    fmov x9, d0
+; SVE_128-NEXT:    add w0, w9, w8
+; SVE_128-NEXT:    ret
+;
+; SVE_MIN_256-LABEL: reduce_sadd_v16i8:
+; SVE_MIN_256:       // %bb.0:
+; SVE_MIN_256-NEXT:    ptrue p0.b, vl16
+; SVE_MIN_256-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE_MIN_256-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE_MIN_256-NEXT:    splice z0.b, p0, z0.b, z1.b
+; SVE_MIN_256-NEXT:    ptrue p0.b, vl32
+; SVE_MIN_256-NEXT:    saddv d0, p0, z0.b
+; SVE_MIN_256-NEXT:    fmov x0, d0
+; SVE_MIN_256-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; SVE_MIN_256-NEXT:    ret
+  %1 = sext <32 x i8> %a to <32 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.v16i32(<32 x i32> %1)
+  ret i32 %2
+}
+
+attributes #0 = { "target-features"="+sve" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
index c81fd26a77525..27e786eb1ced1 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -1968,10 +1968,9 @@ define i32 @test_udot_v16i8(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-SD-BASE:       // %bb.0: // %entry
 ; CHECK-SD-BASE-NEXT:    umull2 v2.8h, v1.16b, v0.16b
 ; CHECK-SD-BASE-NEXT:    umull v0.8h, v1.8b, v0.8b
-; CHECK-SD-BASE-NEXT:    uaddl2 v1.4s, v0.8h, v2.8h
-; CHECK-SD-BASE-NEXT:    uaddl v0.4s, v0.4h, v2.4h
-; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-SD-BASE-NEXT:    addv s0, v0.4s
+; CHECK-SD-BASE-NEXT:    uaddlp v1.4s, v2.8h
+; CHECK-SD-BASE-NEXT:    uadalp v1.4s, v0.8h
+; CHECK-SD-BASE-NEXT:    addv s0, v1.4s
 ; CHECK-SD-BASE-NEXT:    fmov w0, s0
 ; CHECK-SD-BASE-NEXT:    ret
 ;
@@ -2296,10 +2295,9 @@ define i32 @test_sdot_v16i8(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-SD-BASE:       // %bb.0: // %entry
 ; CHECK-SD-BASE-NEXT:    smull2 v2.8h, v1.16b, v0.16b
 ; CHECK-SD-BASE-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-SD-BASE-NEXT:    saddl2 v1.4s, v0.8h, v2.8h
-; CHECK-SD-BASE-NEXT:    saddl v0.4s, v0.4h, v2.4h
-; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-SD-BASE-NEXT:    addv s0, v0.4s
+; CHECK-SD-BASE-NEXT:    saddlp v1.4s, v2.8h
+; CHECK-SD-BASE-NEXT:    sadalp v1.4s, v0.8h
+; CHECK-SD-BASE-NEXT:    addv s0, v1.4s
 ; CHECK-SD-BASE-NEXT:    fmov w0, s0
 ; CHECK-SD-BASE-NEXT:    ret
 ;
@@ -3868,10 +3866,9 @@ entry:
 define i16 @add_v32i8_v32i16_zext(<32 x i8> %x) {
 ; CHECK-SD-LABEL: add_v32i8_v32i16_zext:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    uaddl2 v2.8h, v0.16b, v1.16b
-; CHECK-SD-NEXT:    uaddl v0.8h, v0.8b, v1.8b
-; CHECK-SD-NEXT:    add v0.8h, v0.8h, v2.8h
-; CHECK-SD-NEXT:    addv h0, v0.8h
+; CHECK-SD-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-SD-NEXT:    uadalp v1.8h, v0.16b
+; CHECK-SD-NEXT:    addv h0, v1.8h
 ; CHECK-SD-NEXT:    fmov w0, s0
 ; CHECK-SD-NEXT:    ret
 ;
@@ -3994,10 +3991,9 @@ entry:
 define i16 @add_v32i8_v32i16_sext(<32 x i8> %x) {
 ; CHECK-SD-LABEL: add_v32i8_v32i16_sext:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    saddl2 v2.8h, v0.16b, v1.16b
-; CHECK-SD-NEXT:    saddl v0.8h, v0.8b, v1.8b
-; CHECK-SD-NEXT:    add v0.8h, v0.8h, v2.8h
-; CHECK-SD-NEXT:    addv h0, v0.8h
+; CHECK-SD-NEXT:    saddlp v1.8h, v1.16b
+; CHECK-SD-NEXT:    sadalp v1.8h, v0.16b
+; CHECK-SD-NEXT:    addv h0, v1.8h
 ; CHECK-SD-NEXT:    fmov w0, s0
 ; CHECK-SD-NEXT:    ret
 ;
@@ -4238,14 +4234,14 @@ define i32 @add_v32i8_v32i32_zext(<32 x i8> %x) {
 ; CHECK-SD-BASE-LABEL: add_v32i8_v32i32_zext:
 ; CHECK-SD-BASE:       // %bb.0: // %entry
 ; CHECK-SD-BASE-NEXT:    ushll2 v2.8h, v1.16b, #0
-; CHECK-SD-BASE-NEXT:    ushll2 v3.8h, v0.16b, #0
 ; CHECK-SD-BASE-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-BASE-NEXT:    ushll2 v3.8h, v0.16b, #0
 ; CHECK-SD-BASE-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-SD-BASE-NEXT:    uaddl2 v4.4s, v3.8h, v2.8h
-; CHECK-SD-BASE-NEXT:    uaddl v2.4s, v3.4h, v2.4h
-; CHECK-SD-BASE-NEXT:    uaddl2 v5.4s, v0.8h, v1.8h
-; CHECK-SD-BASE-NEXT:    uaddl v0.4s, v0.4h, v1.4h
-; CHECK-SD-BASE-NEXT:    add v1.4s, v5.4s, v4.4s
+; CHECK-SD-BASE-NEXT:    uaddl2 v4.4s, v1.8h, v2.8h
+; CHECK-SD-BASE-NEXT:    uaddl v1.4s, v1.4h, v2.4h
+; CHECK-SD-BASE-NEXT:    uaddl2 v2.4s, v0.8h, v3.8h
+; CHECK-SD-BASE-NEXT:    uaddl v0.4s, v0.4h, v3.4h
+; CHECK-SD-BASE-NEXT:    add v1.4s, v1.4s, v4.4s
 ; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-BASE-NEXT:    addv s0, v0.4s
@@ -4511,14 +4507,14 @@ define i32 @add_v32i8_v32i32_sext(<32 x i8> %x) {
 ; CHECK-SD-BASE-LABEL: add_v32i8_v32i32_sext:
 ; CHECK-SD-BASE:       // %bb.0: // %entry
 ; CHECK-SD-BASE-NEXT:    sshll2 v2.8h, v1.16b, #0
-; CHECK-SD-BASE-NEXT:    sshll2 v3.8h, v0.16b, #0
 ; CHECK-SD-BASE-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-SD-BASE-NEXT:    sshll2 v3.8h, v0.16b, #0
 ; CHECK-SD-BASE-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-SD-BASE-NEXT:    saddl2 v4.4s, v3.8h, v2.8h
-; CHECK-SD-BASE-NEXT:    saddl v2.4s, v3.4h, v2.4h
-; CHECK-SD-BASE-NEXT:    saddl2 v5.4s, v0.8h, v1.8h
-; CHECK-SD-BASE-NEXT:    saddl v0.4s, v0.4h, v1.4h
-; CHECK-SD-BASE-NEXT:    add v1.4s, v5.4s, v4.4s
+; CHECK-SD-BASE-NEXT:    saddl2 v4.4s, v1.8h, v2.8h
+; CHECK-SD-BASE-NEXT:    saddl v1.4s, v1.4h, v2.4h
+; CHECK-SD-BASE-NEXT:    saddl2 v2.4s, v0.8h, v3.8h
+; CHECK-SD-BASE-NEXT:    saddl v0.4s, v0.4h, v3.4h
+; CHECK-SD-BASE-NEXT:    add v1.4s, v1.4s, v4.4s
 ; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-BASE-NEXT:    addv s0, v0.4s

From 17f98baf704f90dbf892fe0b0150715f09252499 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 19 Jul 2024 10:29:36 +0100
Subject: [PATCH 167/486] [LV] Add test with users both demanding all lanes and
 first-lane-only.

Add a test case where scalar steps are used by both a VPReplicateRecipe
(demands all scalar lanes) and a VPInstruction that only demands the
first lane.

Test case for https://github.com/llvm/llvm-project/issues/88849.
---
 ...demanding-all-lanes-and-first-lane-only.ll | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll

diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll
new file mode 100644
index 0000000000000..c673e9690a667
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll
@@ -0,0 +1,37 @@
+; REQUIRES: asserts
+; RUN: not --crash opt -p loop-vectorize -mtriple=s390x-unknown-linux -mcpu=z16 %s
+
+target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
+
+@src = external global [8 x i32], align 4
+
+; Test case where scalar steps are used by both a VPReplicateRecipe (demands
+; all scalar lanes) and a VPInstruction that only demands the first lane.
+; Test case for https://github.com/llvm/llvm-project/issues/88849.
+define void @test_scalar_iv_steps_used_by_replicate_and_first_lane_only_vpinst(ptr noalias %dst, ptr noalias %src.1) {
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %mul.iv = mul nsw i64 %iv, 4
+  %gep.src.1 = getelementptr inbounds i8, ptr %src.1, i64 %mul.iv
+  %l.1 = load i8, ptr %gep.src.1, align 1
+  %c = icmp eq i8 %l.1, 0
+  br i1 %c, label %then, label %loop.latch
+
+then:
+  %iv.or = or disjoint i64 %iv, 4
+  %gep.src = getelementptr inbounds [8 x i32], ptr @src, i64 0, i64 %iv.or
+  %l.2 = load i32, ptr %gep.src, align 4
+  store i32 %l.2, ptr %dst, align 4
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 4
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}

From a27037becd1bdea568e2f970d4b85fa5e02f3b08 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Fri, 19 Jul 2024 11:36:06 +0200
Subject: [PATCH 168/486] [lldb] Forward-declare lldb-private::SaveCoreOptions

to avoid including private headers from the API headers.

This fixes API tests which link against the lldb library.
---
 lldb/include/lldb/API/SBSaveCoreOptions.h | 1 -
 lldb/include/lldb/lldb-forward.h          | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/include/lldb/API/SBSaveCoreOptions.h b/lldb/include/lldb/API/SBSaveCoreOptions.h
index b8659bf128a78..e77496bd3a4a0 100644
--- a/lldb/include/lldb/API/SBSaveCoreOptions.h
+++ b/lldb/include/lldb/API/SBSaveCoreOptions.h
@@ -10,7 +10,6 @@
 #define LLDB_API_SBSAVECOREOPTIONS_H
 
 #include "lldb/API/SBDefines.h"
-#include "lldb/Symbol/SaveCoreOptions.h"
 
 namespace lldb {
 
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index 9c946eaa19f0b..1024501e05bca 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -182,6 +182,7 @@ class RegisterTypeBuilder;
 class RegisterValue;
 class RegularExpression;
 class RichManglingContext;
+class SaveCoreOptions;
 class Scalar;
 class ScriptInterpreter;
 class ScriptInterpreterLocker;

From 7f763b162ff0f72bbd98e73da9b10d5d3941247b Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Fri, 19 Jul 2024 10:41:24 +0100
Subject: [PATCH 169/486] [MLIR][OpenMP] NFC: Sort clause definitions (#99504)

This patch moves the `filter` clause definition to keep alphabetical
sorting of OpenMPClauses.td.
---
 .../mlir/Dialect/OpenMP/OpenMPClauses.td      | 56 +++++++++----------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
index 99150bc5dff39..5b201687b4ea3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
@@ -305,6 +305,34 @@ class OpenMP_DoacrossClauseSkip<
 
 def OpenMP_DoacrossClause : OpenMP_DoacrossClauseSkip<>;
 
+//===----------------------------------------------------------------------===//
+// V5.2: [10.5.1] `filter` clause
+//===----------------------------------------------------------------------===//
+
+class OpenMP_FilterClauseSkip<
+    bit traits = false, bit arguments = false, bit assemblyFormat = false,
+    bit description = false, bit extraClassDeclaration = false
+  > : OpenMP_Clause</*isRequired=*/false, traits, arguments, assemblyFormat,
+                    description, extraClassDeclaration> {
+  let arguments = (ins
+    Optional<IntLikeType>:$filtered_thread_id
+  );
+
+  let assemblyFormat = [{
+    `filter` `(` $filtered_thread_id `:` type($filtered_thread_id) `)`
+  }];
+
+  let description = [{
+    If `filter` is specified, the masked construct masks the execution of
+    the region to only the thread id filtered. Other threads executing the
+    parallel region are not expected to execute the region specified within
+    the `masked` directive. If `filter` is not specified, master thread is
+    expected to execute the region enclosed within `masked` directive.
+  }];
+}
+
+def OpenMP_FilterClause : OpenMP_FilterClauseSkip<>;
+
 //===----------------------------------------------------------------------===//
 // V5.2: [12.3] `final` clause
 //===----------------------------------------------------------------------===//
@@ -1204,32 +1232,4 @@ class OpenMP_UseDevicePtrClauseSkip<
 
 def OpenMP_UseDevicePtrClause : OpenMP_UseDevicePtrClauseSkip<>;
 
-//===----------------------------------------------------------------------===//
-// V5.2: [10.5.1] `filter` clause
-//===----------------------------------------------------------------------===//
-
-class OpenMP_FilterClauseSkip<
-    bit traits = false, bit arguments = false, bit assemblyFormat = false,
-    bit description = false, bit extraClassDeclaration = false
-  > : OpenMP_Clause</*isRequired=*/false, traits, arguments, assemblyFormat,
-                    description, extraClassDeclaration> {
-  let arguments = (ins
-    Optional<IntLikeType>:$filtered_thread_id
-  );
-
-  let assemblyFormat = [{
-    `filter` `(` $filtered_thread_id `:` type($filtered_thread_id) `)`
-  }];
-
-  let description = [{
-    If `filter` is specified, the masked construct masks the execution of
-    the region to only the thread id filtered. Other threads executing the
-    parallel region are not expected to execute the region specified within
-    the `masked` directive. If `filter` is not specified, master thread is
-    expected to execute the region enclosed within `masked` directive.
-  }];
-}
-
-def OpenMP_FilterClause : OpenMP_FilterClauseSkip<>;
-
 #endif // OPENMP_CLAUSES

From 6cc8774228a4162ebadfa636086a171f82e51a8c Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Fri, 19 Jul 2024 11:43:25 +0200
Subject: [PATCH 170/486] [CodeGen][ARM64EC] Add support for hybrid_patchable
 attribute. (#92965)

---
 llvm/include/llvm/Bitcode/LLVMBitCodes.h      |   1 +
 llvm/include/llvm/CodeGen/AsmPrinter.h        |   2 +-
 llvm/include/llvm/IR/Attributes.td            |   3 +
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp     |   2 +
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp    |   4 +-
 .../AArch64/AArch64Arm64ECCallLowering.cpp    | 138 +++++++-
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp |  27 ++
 .../AArch64/AArch64CallingConvention.td       |   2 +-
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |   1 +
 .../AArch64/arm64ec-hybrid-patchable.ll       | 315 ++++++++++++++++++
 10 files changed, 484 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll

diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 184bbe32df695..fb88f2fe75adb 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -757,6 +757,7 @@ enum AttributeKindCodes {
   ATTR_KIND_RANGE = 92,
   ATTR_KIND_SANITIZE_NUMERICAL_STABILITY = 93,
   ATTR_KIND_INITIALIZES = 94,
+  ATTR_KIND_HYBRID_PATCHABLE = 95,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index dc00bd57d655d..1c4e9e9111441 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -892,7 +892,6 @@ class AsmPrinter : public MachineFunctionPass {
   virtual void emitModuleCommandLines(Module &M);
 
   GCMetadataPrinter *getOrCreateGCPrinter(GCStrategy &S);
-  virtual void emitGlobalAlias(const Module &M, const GlobalAlias &GA);
   void emitGlobalIFunc(Module &M, const GlobalIFunc &GI);
 
 private:
@@ -900,6 +899,7 @@ class AsmPrinter : public MachineFunctionPass {
   bool shouldEmitLabelForBasicBlock(const MachineBasicBlock &MBB) const;
 
 protected:
+  virtual void emitGlobalAlias(const Module &M, const GlobalAlias &GA);
   virtual bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const {
     return false;
   }
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 0457f0c388d26..e1bd193891c1e 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -112,6 +112,9 @@ def ElementType : TypeAttr<"elementtype", [ParamAttr]>;
 /// symbol.
 def FnRetThunkExtern : EnumAttr<"fn_ret_thunk_extern", [FnAttr]>;
 
+/// Function has a hybrid patchable thunk.
+def HybridPatchable : EnumAttr<"hybrid_patchable", [FnAttr]>;
+
 /// Pass structure in an alloca.
 def InAlloca : TypeAttr<"inalloca", [ParamAttr]>;
 
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index b3ebe70e8c52f..324dcbca8137e 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -727,6 +727,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_HOT;
   case Attribute::ElementType:
     return bitc::ATTR_KIND_ELEMENTTYPE;
+  case Attribute::HybridPatchable:
+    return bitc::ATTR_KIND_HYBRID_PATCHABLE;
   case Attribute::InlineHint:
     return bitc::ATTR_KIND_INLINE_HINT;
   case Attribute::InReg:
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 1f59ec545b4f7..b46a6d348413b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2859,8 +2859,8 @@ bool AsmPrinter::emitSpecialLLVMGlobal(const GlobalVariable *GV) {
     auto *Arr = cast<ConstantArray>(GV->getInitializer());
     for (auto &U : Arr->operands()) {
       auto *C = cast<Constant>(U);
-      auto *Src = cast<Function>(C->getOperand(0)->stripPointerCasts());
-      auto *Dst = cast<Function>(C->getOperand(1)->stripPointerCasts());
+      auto *Src = cast<GlobalValue>(C->getOperand(0)->stripPointerCasts());
+      auto *Dst = cast<GlobalValue>(C->getOperand(1)->stripPointerCasts());
       int Kind = cast<ConstantInt>(C->getOperand(2))->getZExtValue();
 
       if (Src->hasDLLImportStorageClass()) {
diff --git a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
index 4ff52eb252d20..310b152ef9817 100644
--- a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/IR/CallingConv.h"
+#include "llvm/IR/GlobalAlias.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Mangler.h"
@@ -70,15 +71,21 @@ class AArch64Arm64ECCallLowering : public ModulePass {
   Function *buildEntryThunk(Function *F);
   void lowerCall(CallBase *CB);
   Function *buildGuestExitThunk(Function *F);
-  bool processFunction(Function &F, SetVector<Function *> &DirectCalledFns);
+  Function *buildPatchableThunk(GlobalAlias *UnmangledAlias,
+                                GlobalAlias *MangledAlias);
+  bool processFunction(Function &F, SetVector<GlobalValue *> &DirectCalledFns,
+                       DenseMap<GlobalAlias *, GlobalAlias *> &FnsMap);
   bool runOnModule(Module &M) override;
 
 private:
   int cfguard_module_flag = 0;
   FunctionType *GuardFnType = nullptr;
   PointerType *GuardFnPtrType = nullptr;
+  FunctionType *DispatchFnType = nullptr;
+  PointerType *DispatchFnPtrType = nullptr;
   Constant *GuardFnCFGlobal = nullptr;
   Constant *GuardFnGlobal = nullptr;
+  Constant *DispatchFnGlobal = nullptr;
   Module *M = nullptr;
 
   Type *PtrTy;
@@ -672,6 +679,66 @@ Function *AArch64Arm64ECCallLowering::buildGuestExitThunk(Function *F) {
   return GuestExit;
 }
 
+Function *
+AArch64Arm64ECCallLowering::buildPatchableThunk(GlobalAlias *UnmangledAlias,
+                                                GlobalAlias *MangledAlias) {
+  llvm::raw_null_ostream NullThunkName;
+  FunctionType *Arm64Ty, *X64Ty;
+  Function *F = cast<Function>(MangledAlias->getAliasee());
+  SmallVector<ThunkArgTranslation> ArgTranslations;
+  getThunkType(F->getFunctionType(), F->getAttributes(),
+               Arm64ECThunkType::GuestExit, NullThunkName, Arm64Ty, X64Ty,
+               ArgTranslations);
+  std::string ThunkName(MangledAlias->getName());
+  if (ThunkName[0] == '?' && ThunkName.find("@") != std::string::npos) {
+    ThunkName.insert(ThunkName.find("@"), "$hybpatch_thunk");
+  } else {
+    ThunkName.append("$hybpatch_thunk");
+  }
+
+  Function *GuestExit =
+      Function::Create(Arm64Ty, GlobalValue::WeakODRLinkage, 0, ThunkName, M);
+  GuestExit->setComdat(M->getOrInsertComdat(ThunkName));
+  GuestExit->setSection(".wowthk$aa");
+  BasicBlock *BB = BasicBlock::Create(M->getContext(), "", GuestExit);
+  IRBuilder<> B(BB);
+
+  // Load the global symbol as a pointer to the check function.
+  LoadInst *DispatchLoad = B.CreateLoad(DispatchFnPtrType, DispatchFnGlobal);
+
+  // Create new dispatch call instruction.
+  Function *ExitThunk =
+      buildExitThunk(F->getFunctionType(), F->getAttributes());
+  CallInst *Dispatch =
+      B.CreateCall(DispatchFnType, DispatchLoad,
+                   {UnmangledAlias, ExitThunk, UnmangledAlias->getAliasee()});
+
+  // Ensure that the first arguments are passed in the correct registers.
+  Dispatch->setCallingConv(CallingConv::CFGuard_Check);
+
+  Value *DispatchRetVal = B.CreateBitCast(Dispatch, PtrTy);
+  SmallVector<Value *> Args;
+  for (Argument &Arg : GuestExit->args())
+    Args.push_back(&Arg);
+  CallInst *Call = B.CreateCall(Arm64Ty, DispatchRetVal, Args);
+  Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
+
+  if (Call->getType()->isVoidTy())
+    B.CreateRetVoid();
+  else
+    B.CreateRet(Call);
+
+  auto SRetAttr = F->getAttributes().getParamAttr(0, Attribute::StructRet);
+  auto InRegAttr = F->getAttributes().getParamAttr(0, Attribute::InReg);
+  if (SRetAttr.isValid() && !InRegAttr.isValid()) {
+    GuestExit->addParamAttr(0, SRetAttr);
+    Call->addParamAttr(0, SRetAttr);
+  }
+
+  MangledAlias->setAliasee(GuestExit);
+  return GuestExit;
+}
+
 // Lower an indirect call with inline code.
 void AArch64Arm64ECCallLowering::lowerCall(CallBase *CB) {
   assert(Triple(CB->getModule()->getTargetTriple()).isOSWindows() &&
@@ -727,17 +794,57 @@ bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) {
 
   GuardFnType = FunctionType::get(PtrTy, {PtrTy, PtrTy}, false);
   GuardFnPtrType = PointerType::get(GuardFnType, 0);
+  DispatchFnType = FunctionType::get(PtrTy, {PtrTy, PtrTy, PtrTy}, false);
+  DispatchFnPtrType = PointerType::get(DispatchFnType, 0);
   GuardFnCFGlobal =
       M->getOrInsertGlobal("__os_arm64x_check_icall_cfg", GuardFnPtrType);
   GuardFnGlobal =
       M->getOrInsertGlobal("__os_arm64x_check_icall", GuardFnPtrType);
+  DispatchFnGlobal =
+      M->getOrInsertGlobal("__os_arm64x_dispatch_call", DispatchFnPtrType);
+
+  DenseMap<GlobalAlias *, GlobalAlias *> FnsMap;
+  SetVector<GlobalAlias *> PatchableFns;
 
-  SetVector<Function *> DirectCalledFns;
+  for (Function &F : Mod) {
+    if (!F.hasFnAttribute(Attribute::HybridPatchable) || F.isDeclaration() ||
+        F.hasLocalLinkage() || F.getName().ends_with("$hp_target"))
+      continue;
+
+    // Rename hybrid patchable functions and change callers to use a global
+    // alias instead.
+    if (std::optional<std::string> MangledName =
+            getArm64ECMangledFunctionName(F.getName().str())) {
+      std::string OrigName(F.getName());
+      F.setName(MangledName.value() + "$hp_target");
+
+      // The unmangled symbol is a weak alias to an undefined symbol with the
+      // "EXP+" prefix. This undefined symbol is resolved by the linker by
+      // creating an x86 thunk that jumps back to the actual EC target. Since we
+      // can't represent that in IR, we create an alias to the target instead.
+      // The "EXP+" symbol is set as metadata, which is then used by
+      // emitGlobalAlias to emit the right alias.
+      auto *A =
+          GlobalAlias::create(GlobalValue::LinkOnceODRLinkage, OrigName, &F);
+      F.replaceAllUsesWith(A);
+      F.setMetadata("arm64ec_exp_name",
+                    MDNode::get(M->getContext(),
+                                MDString::get(M->getContext(),
+                                              "EXP+" + MangledName.value())));
+      A->setAliasee(&F);
+
+      FnsMap[A] = GlobalAlias::create(GlobalValue::LinkOnceODRLinkage,
+                                      MangledName.value(), &F);
+      PatchableFns.insert(A);
+    }
+  }
+
+  SetVector<GlobalValue *> DirectCalledFns;
   for (Function &F : Mod)
     if (!F.isDeclaration() &&
         F.getCallingConv() != CallingConv::ARM64EC_Thunk_Native &&
         F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64)
-      processFunction(F, DirectCalledFns);
+      processFunction(F, DirectCalledFns, FnsMap);
 
   struct ThunkInfo {
     Constant *Src;
@@ -755,14 +862,20 @@ bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) {
           {&F, buildEntryThunk(&F), Arm64ECThunkType::Entry});
     }
   }
-  for (Function *F : DirectCalledFns) {
+  for (GlobalValue *O : DirectCalledFns) {
+    auto GA = dyn_cast<GlobalAlias>(O);
+    auto F = dyn_cast<Function>(GA ? GA->getAliasee() : O);
     ThunkMapping.push_back(
-        {F, buildExitThunk(F->getFunctionType(), F->getAttributes()),
+        {O, buildExitThunk(F->getFunctionType(), F->getAttributes()),
          Arm64ECThunkType::Exit});
-    if (!F->hasDLLImportStorageClass())
+    if (!GA && !F->hasDLLImportStorageClass())
       ThunkMapping.push_back(
           {buildGuestExitThunk(F), F, Arm64ECThunkType::GuestExit});
   }
+  for (GlobalAlias *A : PatchableFns) {
+    Function *Thunk = buildPatchableThunk(A, FnsMap[A]);
+    ThunkMapping.push_back({Thunk, A, Arm64ECThunkType::GuestExit});
+  }
 
   if (!ThunkMapping.empty()) {
     SmallVector<Constant *> ThunkMappingArrayElems;
@@ -785,7 +898,8 @@ bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) {
 }
 
 bool AArch64Arm64ECCallLowering::processFunction(
-    Function &F, SetVector<Function *> &DirectCalledFns) {
+    Function &F, SetVector<GlobalValue *> &DirectCalledFns,
+    DenseMap<GlobalAlias *, GlobalAlias *> &FnsMap) {
   SmallVector<CallBase *, 8> IndirectCalls;
 
   // For ARM64EC targets, a function definition's name is mangled differently
@@ -837,6 +951,16 @@ bool AArch64Arm64ECCallLowering::processFunction(
         continue;
       }
 
+      // Use mangled global alias for direct calls to patchable functions.
+      if (GlobalAlias *A = dyn_cast<GlobalAlias>(CB->getCalledOperand())) {
+        auto I = FnsMap.find(A);
+        if (I != FnsMap.end()) {
+          CB->setCalledOperand(I->second);
+          DirectCalledFns.insert(I->first);
+          continue;
+        }
+      }
+
       IndirectCalls.push_back(CB);
       ++Arm64ECCallsLowered;
     }
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 6f6b551993b6d..63358c1568a35 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -201,6 +201,7 @@ class AArch64AsmPrinter : public AsmPrinter {
   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
 
   void emitFunctionBodyEnd() override;
+  void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
 
   MCSymbol *GetCPISymbol(unsigned CPID) const override;
   void emitEndOfAsmFile(Module &M) override;
@@ -1263,6 +1264,32 @@ void AArch64AsmPrinter::emitFunctionEntryLabel() {
   }
 }
 
+void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
+                                        const GlobalAlias &GA) {
+  if (auto F = dyn_cast_or_null<Function>(GA.getAliasee())) {
+    // Global aliases must point to a definition, but unmangled patchable
+    // symbols are special and need to point to an undefined symbol with "EXP+"
+    // prefix. Such undefined symbol is resolved by the linker by creating
+    // x86 thunk that jumps back to the actual EC target.
+    if (MDNode *Node = F->getMetadata("arm64ec_exp_name")) {
+      StringRef ExpStr = cast<MDString>(Node->getOperand(0))->getString();
+      MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(ExpStr);
+      MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(GA.getName());
+      OutStreamer->beginCOFFSymbolDef(Sym);
+      OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_EXTERNAL);
+      OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_FUNCTION
+                                      << COFF::SCT_COMPLEX_TYPE_SHIFT);
+      OutStreamer->endCOFFSymbolDef();
+      OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
+      OutStreamer->emitAssignment(
+          Sym, MCSymbolRefExpr::create(ExpSym, MCSymbolRefExpr::VK_None,
+                                       MMI->getContext()));
+      return;
+    }
+  }
+  AsmPrinter::emitGlobalAlias(M, GA);
+}
+
 /// Small jump tables contain an unsigned byte or half, representing the offset
 /// from the lowest-addressed possible destination to the desired basic
 /// block. Since all instructions are 4-byte aligned, this is further compressed
diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.td b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
index 2f7e226fd09b2..6f885f4588c4b 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.td
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
@@ -333,7 +333,7 @@ def CC_AArch64_Win64_CFGuard_Check : CallingConv<[
 
 let Entry = 1 in
 def CC_AArch64_Arm64EC_CFGuard_Check : CallingConv<[
-  CCIfType<[i64], CCAssignToReg<[X11, X10]>>
+  CCIfType<[i64], CCAssignToReg<[X11, X10, X9]>>
 ]>;
 
 let Entry = 1 in
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index a05e955943f7c..5bca5cf8ff91f 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -932,6 +932,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::DisableSanitizerInstrumentation:
       case Attribute::FnRetThunkExtern:
       case Attribute::Hot:
+      case Attribute::HybridPatchable:
       case Attribute::NoRecurse:
       case Attribute::InlineHint:
       case Attribute::MinSize:
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll b/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll
new file mode 100644
index 0000000000000..e5387d40b9c64
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll
@@ -0,0 +1,315 @@
+; RUN: llc -mtriple=arm64ec-pc-windows-msvc < %s | FileCheck %s
+; RUN: llc -mtriple=arm64ec-pc-windows-msvc -filetype=obj -o %t.o < %s
+; RUN: llvm-objdump -t %t.o | FileCheck --check-prefix=SYM %s
+
+define dso_local ptr @func() hybrid_patchable nounwind {
+; SYM: [ 8](sec  4)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #func$hp_target
+; CHECK-LABEL:     .def    "#func$hp_target";
+; CHECK:           .section        .text,"xr",discard,"#func$hp_target"
+; CHECK-NEXT:      .globl  "#func$hp_target"               // -- Begin function #func$hp_target
+; CHECK-NEXT:      .p2align        2
+; CHECK-NEXT:  "#func$hp_target":                      // @"#func$hp_target"
+; CHECK-NEXT:      // %bb.0:
+; CHECK-NEXT:      adrp x0, func
+; CHECK-NEXT:      add x0, x0, :lo12:func
+; CHECK-NEXT:      ret
+  ret ptr @func
+}
+
+define void @has_varargs(...) hybrid_patchable nounwind {
+; SYM: [11](sec  5)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #has_varargs$hp_target
+; CHECK-LABEL:     .def "#has_varargs$hp_target";
+; CHECK:           .section .text,"xr",discard,"#has_varargs$hp_target"
+; CHECK-NEXT:      .globl  "#has_varargs$hp_target"        // -- Begin function #has_varargs$hp_target
+; CHECK-NEXT:      .p2align 2
+; CHECK-NEXT:  "#has_varargs$hp_target":               // @"#has_varargs$hp_target"
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:      sub     sp, sp, #32
+; CHECK-NEXT:      stp     x0, x1, [x4, #-32]
+; CHECK-NEXT:      stp     x2, x3, [x4, #-16]
+; CHECK-NEXT:      add     sp, sp, #32
+; CHECK-NEXT:      ret
+  ret void
+}
+
+define void @has_sret(ptr sret([100 x i8])) hybrid_patchable nounwind {
+; SYM: [14](sec  6)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #has_sret$hp_target
+; CHECK-LABEL:     .def    "#has_sret$hp_target";
+; CHECK:           .section        .text,"xr",discard,"#has_sret$hp_target"
+; CHECK-NEXT:      .globl  "#has_sret$hp_target"           // -- Begin function #has_sret$hp_target
+; CHECK-NEXT:      .p2align        2
+; CHECK-NEXT:  "#has_sret$hp_target":                  // @"#has_sret$hp_target"
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:      ret
+  ret void
+}
+
+define dllexport void @exp() hybrid_patchable nounwind {
+; CHECK-LABEL:     .def    "#exp$hp_target";
+; CHECK:           .section        .text,"xr",discard,"#exp$hp_target"
+; CHECK-NEXT:      .globl  "#exp$hp_target"                // -- Begin function #exp$hp_target
+; CHECK-NEXT:      .p2align        2
+; CHECK-NEXT:  "#exp$hp_target":                       // @"#exp$hp_target"
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:      ret
+  ret void
+}
+
+; hybrid_patchable attribute is ignored on internal functions
+define internal i32 @static_func() hybrid_patchable nounwind {
+; CHECK-LABEL:     .def    static_func;
+; CHECK:       static_func:                            // @static_func
+; CHECK-NEXT:      // %bb.0:
+; CHECK-NEXT:      mov     w0, #2                          // =0x2
+; CHECK-NEXT:      ret
+  ret i32 2
+}
+
+define dso_local void @caller() nounwind {
+; CHECK-LABEL:     .def    "#caller";
+; CHECK:           .section        .text,"xr",discard,"#caller"
+; CHECK-NEXT:      .globl  "#caller"                       // -- Begin function #caller
+; CHECK-NEXT:      .p2align        2
+; CHECK-NEXT:  "#caller":                              // @"#caller"
+; CHECK-NEXT:      .weak_anti_dep  caller
+; CHECK-NEXT:  .set caller, "#caller"{{$}}
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:      str     x30, [sp, #-16]!                // 8-byte Folded Spill
+; CHECK-NEXT:      bl      "#func"
+; CHECK-NEXT:      bl      static_func
+; CHECK-NEXT:      adrp    x8, __os_arm64x_check_icall
+; CHECK-NEXT:      adrp    x11, func
+; CHECK-NEXT:      add     x11, x11, :lo12:func
+; CHECK-NEXT:      ldr     x8, [x8, :lo12:__os_arm64x_check_icall]
+; CHECK-NEXT:      adrp    x10, ($iexit_thunk$cdecl$v$v)
+; CHECK-NEXT:      add     x10, x10, :lo12:($iexit_thunk$cdecl$v$v)
+; CHECK-NEXT:      str     x11, [sp, #8]
+; CHECK-NEXT:      blr     x8
+; CHECK-NEXT:      blr     x11
+; CHECK-NEXT:      ldr     x30, [sp], #16                  // 8-byte Folded Reload
+; CHECK-NEXT:      ret
+  %1 = call i32 @func()
+  %2 = call i32 @static_func()
+  %3 = alloca ptr, align 8
+  store ptr @func, ptr %3, align 8
+  %4 = load ptr, ptr %3, align 8
+  call void %4()
+  ret void
+}
+
+; CHECK-LABEL:       def    "#func$hybpatch_thunk";
+; CHECK:            .section        .wowthk$aa,"xr",discard,"#func$hybpatch_thunk"
+; CHECK-NEXT:       .globl  "#func$hybpatch_thunk"          // -- Begin function #func$hybpatch_thunk
+; CHECK-NEXT:       .p2align        2
+; CHECK-NEXT:   "#func$hybpatch_thunk":                 // @"#func$hybpatch_thunk"
+; CHECK-NEXT:   .seh_proc "#func$hybpatch_thunk"
+; CHECK-NEXT:   // %bb.0:
+; CHECK-NEXT:       str     x30, [sp, #-16]!                // 8-byte Folded Spill
+; CHECK-NEXT:       .seh_save_reg_x x30, 16
+; CHECK-NEXT:       .seh_endprologue
+; CHECK-NEXT:       adrp    x8, __os_arm64x_dispatch_call
+; CHECK-NEXT:       adrp    x11, func
+; CHECK-NEXT:       add     x11, x11, :lo12:func
+; CHECK-NEXT:       ldr     x8, [x8, :lo12:__os_arm64x_dispatch_call]
+; CHECK-NEXT:       adrp    x10, ($iexit_thunk$cdecl$i8$v)
+; CHECK-NEXT:       add     x10, x10, :lo12:($iexit_thunk$cdecl$i8$v)
+; CHECK-NEXT:       adrp    x9, "#func$hp_target"
+; CHECK-NEXT:       add     x9, x9, :lo12:"#func$hp_target"
+; CHECK-NEXT:       blr     x8
+; CHECK-NEXT:       .seh_startepilogue
+; CHECK-NEXT:       ldr     x30, [sp], #16                  // 8-byte Folded Reload
+; CHECK-NEXT:       .seh_save_reg_x x30, 16
+; CHECK-NEXT:       .seh_endepilogue
+; CHECK-NEXT:       br      x11
+; CHECK-NEXT:       .seh_endfunclet
+; CHECK-NEXT:       .seh_endproc
+
+; CHECK-LABEL:      .def    "#has_varargs$hybpatch_thunk";
+; CHECK:            .section        .wowthk$aa,"xr",discard,"#has_varargs$hybpatch_thunk"
+; CHECK-NEXT:       .globl  "#has_varargs$hybpatch_thunk"   // -- Begin function #has_varargs$hybpatch_thunk
+; CHECK-NEXT:       .p2align        2
+; CHECK-NEXT:"#has_varargs$hybpatch_thunk":          // @"#has_varargs$hybpatch_thunk"
+; CHECK-NEXT:.seh_proc "#has_varargs$hybpatch_thunk"
+; CHECK-NEXT:// %bb.0:
+; CHECK-NEXT:       str     x30, [sp, #-16]!                // 8-byte Folded Spill
+; CHECK-NEXT:       .seh_save_reg_x x30, 16
+; CHECK-NEXT:       .seh_endprologue
+; CHECK-NEXT:       adrp    x8, __os_arm64x_dispatch_call
+; CHECK-NEXT:       adrp    x11, has_varargs
+; CHECK-NEXT:       add     x11, x11, :lo12:has_varargs
+; CHECK-NEXT:       ldr     x8, [x8, :lo12:__os_arm64x_dispatch_call]
+; CHECK-NEXT:       adrp    x10, ($iexit_thunk$cdecl$v$varargs)
+; CHECK-NEXT:       add     x10, x10, :lo12:($iexit_thunk$cdecl$v$varargs)
+; CHECK-NEXT:       adrp    x9, "#has_varargs$hp_target"
+; CHECK-NEXT:       add     x9, x9, :lo12:"#has_varargs$hp_target"
+; CHECK-NEXT:       blr     x8
+; CHECK-NEXT:       .seh_startepilogue
+; CHECK-NEXT:       ldr     x30, [sp], #16                  // 8-byte Folded Reload
+; CHECK-NEXT:       .seh_save_reg_x x30, 16
+; CHECK-NEXT:       .seh_endepilogue
+; CHECK-NEXT:       br      x11
+; CHECK-NEXT:       .seh_endfunclet
+; CHECK-NEXT:       .seh_endproc
+
+; CHECK-LABEL:     .def    "#has_sret$hybpatch_thunk";
+; CHECK:           .section        .wowthk$aa,"xr",discard,"#has_sret$hybpatch_thunk"
+; CHECK-NEXT:      .globl  "#has_sret$hybpatch_thunk"      // -- Begin function #has_sret$hybpatch_thunk
+; CHECK-NEXT:      .p2align        2
+; CHECK-NEXT:  "#has_sret$hybpatch_thunk":             // @"#has_sret$hybpatch_thunk"
+; CHECK-NEXT:  .seh_proc "#has_sret$hybpatch_thunk"
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:      str     x30, [sp, #-16]!                // 8-byte Folded Spill
+; CHECK-NEXT:      .seh_save_reg_x x30, 16
+; CHECK-NEXT:      .seh_endprologue
+; CHECK-NEXT:      adrp    x9, __os_arm64x_dispatch_call
+; CHECK-NEXT:      adrp    x11, has_sret
+; CHECK-NEXT:      add     x11, x11, :lo12:has_sret
+; CHECK-NEXT:      ldr     x12, [x9, :lo12:__os_arm64x_dispatch_call]
+; CHECK-NEXT:      adrp    x10, ($iexit_thunk$cdecl$m100$v)
+; CHECK-NEXT:      add     x10, x10, :lo12:($iexit_thunk$cdecl$m100$v)
+; CHECK-NEXT:      adrp    x9, "#has_sret$hp_target"
+; CHECK-NEXT:      add     x9, x9, :lo12:"#has_sret$hp_target"
+; CHECK-NEXT:      blr     x12
+; CHECK-NEXT:      .seh_startepilogue
+; CHECK-NEXT:      ldr     x30, [sp], #16                  // 8-byte Folded Reload
+; CHECK-NEXT:      .seh_save_reg_x x30, 16
+; CHECK-NEXT:      .seh_endepilogue
+; CHECK-NEXT:      br      x11
+; CHECK-NEXT:      .seh_endfunclet
+; CHECK-NEXT:      .seh_endproc
+
+; CHECK-LABEL:     .def    "#exp$hybpatch_thunk";
+; CHECK:           .section        .wowthk$aa,"xr",discard,"#exp$hybpatch_thunk"
+; CHECK-NEXT:      .globl  "#exp$hybpatch_thunk"           // -- Begin function #exp$hybpatch_thunk
+; CHECK-NEXT:      .p2align        2
+; CHECK-NEXT:  "#exp$hybpatch_thunk":                // @"#exp$hybpatch_thunk"
+; CHECK-NEXT:  .seh_proc "#exp$hybpatch_thunk"
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:      str     x30, [sp, #-16]!                // 8-byte Folded Spill
+; CHECK-NEXT:      .seh_save_reg_x x30, 16
+; CHECK-NEXT:      .seh_endprologue
+; CHECK-NEXT:      adrp    x8, __os_arm64x_dispatch_call
+; CHECK-NEXT:      adrp    x11, exp
+; CHECK-NEXT:      add     x11, x11, :lo12:exp
+; CHECK-NEXT:      ldr     x8, [x8, :lo12:__os_arm64x_dispatch_call]
+; CHECK-NEXT:      adrp    x10, ($iexit_thunk$cdecl$v$v)
+; CHECK-NEXT:      add     x10, x10, :lo12:($iexit_thunk$cdecl$v$v)
+; CHECK-NEXT:      adrp    x9, "#exp$hp_target"
+; CHECK-NEXT:      add     x9, x9, :lo12:"#exp$hp_target"
+; CHECK-NEXT:      blr     x8
+; CHECK-NEXT:      .seh_startepilogue
+; CHECK-NEXT:      ldr     x30, [sp], #16                  // 8-byte Folded Reload
+; CHECK-NEXT:      .seh_save_reg_x x30, 16
+; CHECK-NEXT:      .seh_endepilogue
+; CHECK-NEXT:      br      x11
+; CHECK-NEXT:      .seh_endfunclet
+; CHECK-NEXT:      .seh_endproc
+
+; Verify the hybrid bitmap
+; CHECK-LABEL:     .section        .hybmp$x,"yi"
+; CHECK-NEXT:      .symidx "#func$hp_target"
+; CHECK-NEXT:      .symidx $ientry_thunk$cdecl$i8$v
+; CHECK-NEXT:      .word   1
+; CHECK-NEXT:      .symidx "#has_varargs$hp_target"
+; CHECK-NEXT:      .symidx $ientry_thunk$cdecl$v$varargs
+; CHECK-NEXT:      .word   1
+; CHECK-NEXT:      .symidx "#has_sret$hp_target"
+; CHECK-NEXT:      .symidx $ientry_thunk$cdecl$m100$v
+; CHECK-NEXT:      .word   1
+; CHECK-NEXT:      .symidx "#exp$hp_target"
+; CHECK-NEXT:      .symidx $ientry_thunk$cdecl$v$v
+; CHECK-NEXT:      .word   1
+; CHECK-NEXT:      .symidx "#caller"
+; CHECK-NEXT:      .symidx $ientry_thunk$cdecl$v$v
+; CHECK-NEXT:      .word   1
+; CHECK-NEXT:      .symidx func
+; CHECK-NEXT:      .symidx $iexit_thunk$cdecl$i8$v
+; CHECK-NEXT:      .word   4
+; CHECK-NEXT:      .symidx "#func$hybpatch_thunk"
+; CHECK-NEXT:      .symidx func
+; CHECK-NEXT:      .word   0
+; CHECK-NEXT:      .symidx "#has_varargs$hybpatch_thunk"
+; CHECK-NEXT:      .symidx has_varargs
+; CHECK-NEXT:      .word   0
+; CHECK-NEXT:      .symidx "#has_sret$hybpatch_thunk"
+; CHECK-NEXT:      .symidx has_sret
+; CHECK-NEXT:      .word   0
+; CHECK-NEXT:      .symidx "#exp$hybpatch_thunk"
+; CHECK-NEXT:      .symidx exp
+; CHECK-NEXT:      .word   0
+; CHECK-NEXT:      .section        .drectve,"yni"
+; CHECK-NEXT:      .ascii  " /EXPORT:\"#exp$hp_target,EXPORTAS,exp$hp_target\""
+
+; CHECK-NEXT:      .def    func;
+; CHECK-NEXT:      .scl    2;
+; CHECK-NEXT:      .type   32;
+; CHECK-NEXT:      .endef
+; CHECK-NEXT:      .weak  func
+; CHECK-NEXT:  .set func, "EXP+#func"{{$}}
+; CHECK-NEXT:      .weak  "#func"
+; CHECK-NEXT:      .def    "#func";
+; CHECK-NEXT:      .scl    2;
+; CHECK-NEXT:      .type   32;
+; CHECK-NEXT:      .endef
+; CHECK-NEXT:  .set "#func", "#func$hybpatch_thunk"{{$}}
+; CHECK-NEXT:      .def    has_varargs;
+; CHECK-NEXT:      .scl    2;
+; CHECK-NEXT:      .type   32;
+; CHECK-NEXT:      .endef
+; CHECK-NEXT:      .weak   has_varargs
+; CHECK-NEXT:  .set has_varargs, "EXP+#has_varargs"
+; CHECK-NEXT:      .weak   "#has_varargs"
+; CHECK-NEXT:      .def    "#has_varargs";
+; CHECK-NEXT:      .scl    2;
+; CHECK-NEXT:      .type   32;
+; CHECK-NEXT:      .endef
+; CHECK-NEXT:  .set "#has_varargs", "#has_varargs$hybpatch_thunk"
+; CHECK-NEXT:      .def    has_sret;
+; CHECK-NEXT:      .scl    2;
+; CHECK-NEXT:      .type   32;
+; CHECK-NEXT:      .endef
+; CHECK-NEXT:      .weak   has_sret
+; CHECK-NEXT:  .set has_sret, "EXP+#has_sret"
+; CHECK-NEXT:      .weak   "#has_sret"
+; CHECK-NEXT:      .def    "#has_sret";
+; CHECK-NEXT:      .scl    2;
+; CHECK-NEXT:      .type   32;
+; CHECK-NEXT:      .endef
+; CHECK-NEXT:  .set "#has_sret", "#has_sret$hybpatch_thunk"
+; CHECK-NEXT:      .def    exp;
+; CHECK-NEXT:      .scl    2;
+; CHECK-NEXT:      .type   32;
+; CHECK-NEXT:      .endef
+; CHECK-NEXT:      .weak   exp
+; CHECK-NEXT:  .set exp, "EXP+#exp"
+; CHECK-NEXT:      .weak   "#exp"
+; CHECK-NEXT:      .def    "#exp";
+; CHECK-NEXT:      .scl    2;
+; CHECK-NEXT:      .type   32;
+; CHECK-NEXT:      .endef
+; CHECK-NEXT:  .set "#exp", "#exp$hybpatch_thunk"
+
+; SYM:      [53](sec 15)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #func$hybpatch_thunk
+; SYM:      [58](sec 16)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #has_varargs$hybpatch_thunk
+; SYM:      [68](sec 18)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #has_sret$hybpatch_thunk
+; SYM:      [78](sec 20)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #exp$hybpatch_thunk
+; SYM:      [110](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 func
+; SYM-NEXT: AUX indx 112 srch 3
+; SYM-NEXT: [112](sec  0)(fl 0x00)(ty   0)(scl   2) (nx 0) 0x00000000 EXP+#func
+; SYM:      [116](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 #func
+; SYM-NEXT: AUX indx 53 srch 3
+; SYM:      [122](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 has_varargs
+; SYM-NEXT: AUX indx 124 srch 3
+; SYM-NEXT: [124](sec  0)(fl 0x00)(ty   0)(scl   2) (nx 0) 0x00000000 EXP+#has_varargs
+; SYM-NEXT: [125](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 has_sret
+; SYM-NEXT: AUX indx 127 srch 3
+; SYM-NEXT: [127](sec  0)(fl 0x00)(ty   0)(scl   2) (nx 0) 0x00000000 EXP+#has_sret
+; SYM-NEXT: [128](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 exp
+; SYM-NEXT: AUX indx 130 srch 3
+; SYM-NEXT: [130](sec  0)(fl 0x00)(ty   0)(scl   2) (nx 0) 0x00000000 EXP+#exp
+; SYM-NEXT: [131](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 #has_varargs
+; SYM-NEXT: AUX indx 58 srch 3
+; SYM-NEXT: [133](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 #has_sret
+; SYM-NEXT: AUX indx 68 srch 3
+; SYM-NEXT: [135](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 #exp
+; SYM-NEXT: AUX indx 78 srch 3

From 5a45fed188dce2ae6c4da23ba61c9b8df87f08f4 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 19 Jul 2024 12:42:57 +0300
Subject: [PATCH 171/486] [clang][NFC] Fix typo in `-Ofast` deprecation warning

A follow-up for #98736
---
 clang/include/clang/Basic/DiagnosticDriverKinds.td | 2 +-
 clang/include/clang/Driver/Options.td              | 2 +-
 clang/test/Driver/Ofast.c                          | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 26a9792e6a608..3d8240f8357b4 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -445,7 +445,7 @@ def warn_drv_deprecated_arg_no_relaxed_template_template_args : Warning<
   "argument '-fno-relaxed-template-template-args' is deprecated">,
   InGroup<DeprecatedNoRelaxedTemplateTemplateArgs>;
 def warn_drv_deprecated_arg_ofast : Warning<
-  "argument '-Ofast' is deprecated; use '-O3 -ffast math' for the same behavior,"
+  "argument '-Ofast' is deprecated; use '-O3 -ffast-math' for the same behavior,"
   " or '-O3' to enable only conforming optimizations">,
   InGroup<DeprecatedOFast>;
 def warn_drv_deprecated_custom : Warning<
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 12b20fe04675b..fecfa2f3c7dba 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -932,7 +932,7 @@ def O_flag : Flag<["-"], "O">, Visibility<[ClangOption, CC1Option, FC1Option]>,
   Alias<O>, AliasArgs<["1"]>;
 def Ofast : Joined<["-"], "Ofast">, Group<O_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption]>,
-  HelpText<"Deprecated; use '-O3 -ffast math' for the same behavior,"
+  HelpText<"Deprecated; use '-O3 -ffast-math' for the same behavior,"
   " or '-O3' to enable only conforming optimizations">;
 def P : Flag<["-"], "P">,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
diff --git a/clang/test/Driver/Ofast.c b/clang/test/Driver/Ofast.c
index 4c63caf9865d5..91de296a4c3ff 100644
--- a/clang/test/Driver/Ofast.c
+++ b/clang/test/Driver/Ofast.c
@@ -10,7 +10,7 @@
 // RUN: %clang -Ofast -fno-strict-aliasing -### %s 2>&1 | FileCheck -check-prefix=CHECK-OFAST-NO-STRICT-ALIASING %s
 // RUN: %clang -Ofast -fno-vectorize -### %s 2>&1 | FileCheck -check-prefix=CHECK-OFAST-NO-VECTORIZE %s
 
-// CHECK-OFAST: use '-O3 -ffast math' for the same behavior, or '-O3' to enable only conforming optimizations
+// CHECK-OFAST: use '-O3 -ffast-math' for the same behavior, or '-O3' to enable only conforming optimizations
 // CHECK-OFAST: -cc1
 // CHECK-OFAST-NOT: -relaxed-aliasing
 // CHECK-OFAST: -ffast-math
@@ -25,21 +25,21 @@
 // CHECK-OFAST-O2-NOT: -Ofast
 // CHECK-OFAST-O2: -vectorize-loops
 
-// CHECK-OFAST-NO-FAST-MATH: use '-O3 -ffast math' for the same behavior, or '-O3' to enable only conforming optimizations
+// CHECK-OFAST-NO-FAST-MATH: use '-O3 -ffast-math' for the same behavior, or '-O3' to enable only conforming optimizations
 // CHECK-OFAST-NO-FAST-MATH: -cc1
 // CHECK-OFAST-NO-FAST-MATH-NOT: -relaxed-aliasing
 // CHECK-OFAST-NO-FAST-MATH-NOT: -ffast-math
 // CHECK-OFAST-NO-FAST-MATH: -Ofast
 // CHECK-OFAST-NO-FAST-MATH: -vectorize-loops
 
-// CHECK-OFAST-NO-STRICT-ALIASING: use '-O3 -ffast math' for the same behavior, or '-O3' to enable only conforming optimizations
+// CHECK-OFAST-NO-STRICT-ALIASING: use '-O3 -ffast-math' for the same behavior, or '-O3' to enable only conforming optimizations
 // CHECK-OFAST-NO-STRICT-ALIASING: -cc1
 // CHECK-OFAST-NO-STRICT-ALIASING: -relaxed-aliasing
 // CHECK-OFAST-NO-STRICT-ALIASING: -ffast-math
 // CHECK-OFAST-NO-STRICT-ALIASING: -Ofast
 // CHECK-OFAST-NO-STRICT-ALIASING: -vectorize-loops
 
-// CHECK-OFAST-NO-VECTORIZE: use '-O3 -ffast math' for the same behavior, or '-O3' to enable only conforming optimizations
+// CHECK-OFAST-NO-VECTORIZE: use '-O3 -ffast-math' for the same behavior, or '-O3' to enable only conforming optimizations
 // CHECK-OFAST-NO-VECTORIZE: -cc1
 // CHECK-OFAST-NO-VECTORIZE-NOT: -relaxed-aliasing
 // CHECK-OFAST-NO-VECTORIZE: -ffast-math

From 02cb5bcab4f5232bbc5ecacb3d5d94fd487baa23 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 19 Jul 2024 02:47:47 -0700
Subject: [PATCH 172/486] [ADT] Teach set_intersect to erase with iterators
 (#99569)

Without this patch, we erase an element in S1 by value even though we
have an interator pointing to it.  This patch tries to use erase(iter)
to avoid redundant lookups.
---
 llvm/include/llvm/ADT/SetOperations.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/ADT/SetOperations.h b/llvm/include/llvm/ADT/SetOperations.h
index 40e4b05fd7425..86a27b683ebc1 100644
--- a/llvm/include/llvm/ADT/SetOperations.h
+++ b/llvm/include/llvm/ADT/SetOperations.h
@@ -60,11 +60,11 @@ template <class S1Ty, class S2Ty> void set_intersect(S1Ty &S1, const S2Ty &S2) {
   if constexpr (detail::HasMemberRemoveIf<S1Ty, decltype(Pred)>) {
     S1.remove_if(Pred);
   } else {
-    for (typename S1Ty::iterator I = S1.begin(); I != S1.end();) {
-      const auto &E = *I;
-      ++I;
-      if (!S2.count(E))
-        S1.erase(E); // Erase element if not in S2
+    typename S1Ty::iterator Next;
+    for (typename S1Ty::iterator I = S1.begin(); I != S1.end(); I = Next) {
+      Next = std::next(I);
+      if (!S2.count(*I))
+        S1.erase(I); // Erase element if not in S2
     }
   }
 }

From 0f231567719c99caa99164d8f91bad50883dab03 Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Fri, 19 Jul 2024 17:47:58 +0800
Subject: [PATCH 173/486] [LoongArch] Support parsing the
 `%le_{hi20,add,lo12}_r` modifiers

Reviewed By: SixWeining

Pull Request: https://github.com/llvm/llvm-project/pull/99485
---
 .../AsmParser/LoongArchAsmParser.cpp          | 16 +++++++
 .../Target/LoongArch/LoongArchInstrInfo.td    | 28 +++++++++++
 .../MCTargetDesc/LoongArchFixupKinds.h        |  6 +++
 .../MCTargetDesc/LoongArchMCCodeEmitter.cpp   | 48 +++++++++++++++++++
 .../MCTargetDesc/LoongArchMCExpr.cpp          |  9 ++++
 .../LoongArch/MCTargetDesc/LoongArchMCExpr.h  |  3 ++
 .../MC/LoongArch/Relocations/relocations.s    | 15 ++++++
 7 files changed, 125 insertions(+)

diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
index 73cb80245bca4..208bd3db8f14e 100644
--- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
+++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
@@ -245,6 +245,16 @@ class LoongArchOperand : public MCParsedAsmOperand {
            VK == LoongArchMCExpr::VK_LoongArch_None;
   }
 
+  bool isTPRelAddSymbol() const {
+    int64_t Imm;
+    LoongArchMCExpr::VariantKind VK = LoongArchMCExpr::VK_LoongArch_None;
+    // Must be of 'immediate' type but not a constant.
+    if (!isImm() || evaluateConstantImm(getImm(), Imm, VK))
+      return false;
+    return LoongArchAsmParser::classifySymbolRef(getImm(), VK) &&
+           VK == LoongArchMCExpr::VK_LoongArch_TLS_LE_ADD_R;
+  }
+
   bool isUImm1() const { return isUImm<1>(); }
   bool isUImm2() const { return isUImm<2>(); }
   bool isUImm2plus1() const { return isUImm<2, 1>(); }
@@ -276,6 +286,7 @@ class LoongArchOperand : public MCParsedAsmOperand {
                        VK == LoongArchMCExpr::VK_LoongArch_PCALA_LO12 ||
                        VK == LoongArchMCExpr::VK_LoongArch_GOT_PC_LO12 ||
                        VK == LoongArchMCExpr::VK_LoongArch_TLS_IE_PC_LO12 ||
+                       VK == LoongArchMCExpr::VK_LoongArch_TLS_LE_LO12_R ||
                        VK == LoongArchMCExpr::VK_LoongArch_TLS_DESC_PC_LO12 ||
                        VK == LoongArchMCExpr::VK_LoongArch_TLS_DESC_LD;
     return IsConstantImm
@@ -391,6 +402,7 @@ class LoongArchOperand : public MCParsedAsmOperand {
                        VK == LoongArchMCExpr::VK_LoongArch_TLS_LD_HI20 ||
                        VK == LoongArchMCExpr::VK_LoongArch_TLS_IE_HI20 ||
                        VK == LoongArchMCExpr::VK_LoongArch_TLS_LE_HI20 ||
+                       VK == LoongArchMCExpr::VK_LoongArch_TLS_LE_HI20_R ||
                        VK == LoongArchMCExpr::VK_LoongArch_TLS_DESC_HI20;
     return IsConstantImm
                ? isInt<20>(Imm) && IsValidKind
@@ -1686,6 +1698,10 @@ bool LoongArchAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     SMLoc ErrorLoc = ((LoongArchOperand &)*Operands[ErrorInfo]).getStartLoc();
     return Error(ErrorLoc, "operand must be a bare symbol name");
   }
+  case Match_InvalidTPRelAddSymbol: {
+    SMLoc ErrorLoc = ((LoongArchOperand &)*Operands[ErrorInfo]).getStartLoc();
+    return Error(ErrorLoc, "operand must be a symbol with %le_add_r modifier");
+  }
   }
   llvm_unreachable("Unknown match type detected!");
 }
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index 67c3dfd3b2599..97f0e8d6a10c7 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -456,6 +456,19 @@ def bare_symbol : Operand<GRLenVT> {
   let ParserMatchClass = BareSymbol;
 }
 
+def TPRelAddSymbol : AsmOperandClass {
+  let Name = "TPRelAddSymbol";
+  let RenderMethod = "addImmOperands";
+  let DiagnosticType = "InvalidTPRelAddSymbol";
+  let ParserMethod = "parseOperandWithModifier";
+}
+
+// A bare symbol with the %le_add_r variant.
+def tprel_add_symbol : Operand<GRLenVT> {
+  let ParserMatchClass = TPRelAddSymbol;
+}
+
+
 // Standalone (codegen-only) immleaf patterns.
 
 // A 12-bit signed immediate plus one where the imm range will be [-2047, 2048].
@@ -1570,6 +1583,21 @@ def PseudoTAIL36 : Pseudo<(outs), (ins GPR:$tmp, bare_symbol:$dst), [],
                           "tail36", "$tmp, $dst">,
                    Requires<[IsLA64]>;
 
+// This is a special case of the ADD_W/D instruction used to facilitate the use
+// of a fourth operand to emit a relocation on a symbol relating to this
+// instruction. The relocation does not affect any bits of the instruction itself
+// but is used as a hint to the linker.
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 0 in {
+def PseudoAddTPRel_W : Pseudo<(outs GPR:$rd),
+                              (ins GPR:$rj, GPR:$rk, tprel_add_symbol:$sym), [],
+                              "add.w", "$rd, $rj, $rk, $sym">,
+                              Requires<[IsLA32]>;
+def PseudoAddTPRel_D : Pseudo<(outs GPR:$rd),
+                              (ins GPR:$rj, GPR:$rk, tprel_add_symbol:$sym), [],
+                              "add.d", "$rd, $rj, $rk, $sym">,
+                              Requires<[IsLA64]>;
+}
+
 /// Load address (la*) macro instructions.
 
 // Define isCodeGenOnly = 0 to expose them to tablegened assembly parser.
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
index fb0587bf3bed7..29ed14f6e4d62 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
@@ -136,6 +136,12 @@ enum Fixups {
   fixup_loongarch_tls_desc_ld,
   // 12-bit fixup corresponding to %desc_call(foo) for instruction jirl.
   fixup_loongarch_tls_desc_call,
+  // 20-bit fixup corresponding to %le_hi20_r(foo) for instruction lu12i.w.
+  fixup_loongarch_tls_le_hi20_r,
+  // Fixup corresponding to %le_add_r(foo) for instruction PseudoAddTPRel_W/D.
+  fixup_loongarch_tls_le_add_r,
+  // 12-bit fixup corresponding to %le_lo12_r(foo) for instruction addi.w/d.
+  fixup_loongarch_tls_le_lo12_r,
 };
 } // end namespace LoongArch
 } // end namespace llvm
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
index 83812dc3c62a8..efbfce35dbfca 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
@@ -49,6 +49,10 @@ class LoongArchMCCodeEmitter : public MCCodeEmitter {
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const;
 
+  void expandAddTPRel(const MCInst &MI, SmallVectorImpl<char> &CB,
+                      SmallVectorImpl<MCFixup> &Fixups,
+                      const MCSubtargetInfo &STI) const;
+
   /// TableGen'erated function for getting the binary encoding for an
   /// instruction.
   uint64_t getBinaryCodeForInstr(const MCInst &MI,
@@ -134,6 +138,9 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO,
     case LoongArchMCExpr::VK_LoongArch_None:
     case LoongArchMCExpr::VK_LoongArch_Invalid:
       llvm_unreachable("Unhandled fixup kind!");
+    case LoongArchMCExpr::VK_LoongArch_TLS_LE_ADD_R:
+      llvm_unreachable("VK_LoongArch_TLS_LE_ADD_R should not represent an "
+                       "instruction operand");
     case LoongArchMCExpr::VK_LoongArch_B16:
       FixupKind = LoongArch::fixup_loongarch_b16;
       break;
@@ -274,6 +281,12 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO,
     case LoongArchMCExpr::VK_LoongArch_TLS_DESC_CALL:
       FixupKind = LoongArch::fixup_loongarch_tls_desc_call;
       break;
+    case LoongArchMCExpr::VK_LoongArch_TLS_LE_HI20_R:
+      FixupKind = LoongArch::fixup_loongarch_tls_le_hi20_r;
+      break;
+    case LoongArchMCExpr::VK_LoongArch_TLS_LE_LO12_R:
+      FixupKind = LoongArch::fixup_loongarch_tls_le_lo12_r;
+      break;
     }
   } else if (Kind == MCExpr::SymbolRef &&
              cast<MCSymbolRefExpr>(Expr)->getKind() ==
@@ -346,6 +359,38 @@ void LoongArchMCCodeEmitter::expandToVectorLDI(
   support::endian::write(CB, Binary, llvm::endianness::little);
 }
 
+void LoongArchMCCodeEmitter::expandAddTPRel(const MCInst &MI,
+                                            SmallVectorImpl<char> &CB,
+                                            SmallVectorImpl<MCFixup> &Fixups,
+                                            const MCSubtargetInfo &STI) const {
+  MCOperand Rd = MI.getOperand(0);
+  MCOperand Rj = MI.getOperand(1);
+  MCOperand Rk = MI.getOperand(2);
+  MCOperand Symbol = MI.getOperand(3);
+  assert(Symbol.isExpr() &&
+         "Expected expression as third input to TP-relative add");
+
+  const LoongArchMCExpr *Expr = dyn_cast<LoongArchMCExpr>(Symbol.getExpr());
+  assert(Expr &&
+         Expr->getKind() == LoongArchMCExpr::VK_LoongArch_TLS_LE_ADD_R &&
+         "Expected %le_add_r relocation on TP-relative symbol");
+
+  // Emit the correct %le_add_r relocation for the symbol.
+  // TODO: Emit R_LARCH_RELAX for %le_add_r where the relax feature is enabled.
+  Fixups.push_back(MCFixup::create(
+      0, Expr, MCFixupKind(LoongArch::fixup_loongarch_tls_le_add_r),
+      MI.getLoc()));
+
+  // Emit a normal ADD instruction with the given operands.
+  unsigned ADD = MI.getOpcode() == LoongArch::PseudoAddTPRel_D
+                     ? LoongArch::ADD_D
+                     : LoongArch::ADD_W;
+  MCInst TmpInst =
+      MCInstBuilder(ADD).addOperand(Rd).addOperand(Rj).addOperand(Rk);
+  uint32_t Binary = getBinaryCodeForInstr(TmpInst, Fixups, STI);
+  support::endian::write(CB, Binary, llvm::endianness::little);
+}
+
 void LoongArchMCCodeEmitter::encodeInstruction(
     const MCInst &MI, SmallVectorImpl<char> &CB,
     SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
@@ -366,6 +411,9 @@ void LoongArchMCCodeEmitter::encodeInstruction(
   case LoongArch::PseudoXVREPLI_W:
   case LoongArch::PseudoXVREPLI_D:
     return expandToVectorLDI<LoongArch::XVLDI>(MI, CB, Fixups, STI);
+  case LoongArch::PseudoAddTPRel_W:
+  case LoongArch::PseudoAddTPRel_D:
+    return expandAddTPRel(MI, CB, Fixups, STI);
   }
 
   switch (Size) {
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp
index 9b537794c7ab6..98b9b1ab6d3f4 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp
@@ -160,6 +160,12 @@ StringRef LoongArchMCExpr::getVariantKindName(VariantKind Kind) {
     return "desc_ld";
   case VK_LoongArch_TLS_DESC_CALL:
     return "desc_call";
+  case VK_LoongArch_TLS_LE_HI20_R:
+    return "le_hi20_r";
+  case VK_LoongArch_TLS_LE_ADD_R:
+    return "le_add_r";
+  case VK_LoongArch_TLS_LE_LO12_R:
+    return "le_lo12_r";
   }
 }
 
@@ -213,6 +219,9 @@ LoongArchMCExpr::getVariantKindForName(StringRef name) {
       .Case("desc64_hi12", VK_LoongArch_TLS_DESC64_HI12)
       .Case("desc_ld", VK_LoongArch_TLS_DESC_LD)
       .Case("desc_call", VK_LoongArch_TLS_DESC_CALL)
+      .Case("le_hi20_r", VK_LoongArch_TLS_LE_HI20_R)
+      .Case("le_add_r", VK_LoongArch_TLS_LE_ADD_R)
+      .Case("le_lo12_r", VK_LoongArch_TLS_LE_LO12_R)
       .Default(VK_LoongArch_Invalid);
 }
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h
index 148fea309d6fc..1546d894d56ab 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h
@@ -72,6 +72,9 @@ class LoongArchMCExpr : public MCTargetExpr {
     VK_LoongArch_TLS_DESC64_HI12,
     VK_LoongArch_TLS_DESC_LD,
     VK_LoongArch_TLS_DESC_CALL,
+    VK_LoongArch_TLS_LE_HI20_R,
+    VK_LoongArch_TLS_LE_ADD_R,
+    VK_LoongArch_TLS_LE_LO12_R,
     VK_LoongArch_Invalid // Must be the last item.
   };
 
diff --git a/llvm/test/MC/LoongArch/Relocations/relocations.s b/llvm/test/MC/LoongArch/Relocations/relocations.s
index 87df59978c6ea..e83b67199e656 100644
--- a/llvm/test/MC/LoongArch/Relocations/relocations.s
+++ b/llvm/test/MC/LoongArch/Relocations/relocations.s
@@ -273,3 +273,18 @@ lu52i.d $t1, $t1, %desc64_hi12(foo)
 # RELOC: R_LARCH_TLS_DESC64_HI12 foo 0x0
 # INSTR: lu52i.d $t1, $t1, %desc64_hi12(foo)
 # FIXUP: fixup A - offset: 0, value: %desc64_hi12(foo), kind: FK_NONE
+
+lu12i.w $t1, %le_hi20_r(foo)
+# RELOC: R_LARCH_TLS_LE_HI20_R foo 0x0
+# INSTR: lu12i.w $t1, %le_hi20_r(foo)
+# FIXUP: fixup A - offset: 0, value: %le_hi20_r(foo), kind: FK_NONE
+
+add.d $t1, $t2, $tp, %le_add_r(foo)
+# RELOC: R_LARCH_TLS_LE_ADD_R foo 0x0
+# INSTR: add.d $t1, $t2, $tp, %le_add_r(foo)
+# FIXUP: fixup A - offset: 0, value: %le_add_r(foo), kind: FK_NONE
+
+addi.d $t1, $a2, %le_lo12_r(foo)
+# RELOC: R_LARCH_TLS_LE_LO12_R foo 0x0
+# INSTR: addi.d $t1, $a2, %le_lo12_r(foo)
+# FIXUP: fixup A - offset: 0, value: %le_lo12_r(foo), kind: FK_NONE

From 6b98ab95f0d36705b5a1fc1e755c992ba2329c89 Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Fri, 19 Jul 2024 17:48:47 +0800
Subject: [PATCH 174/486] [lld][ELF][LoongArch] Add support for
 R_LARCH_LE_{HI20,ADD,LO12}_R relocations

Reviewed By: SixWeining

Pull Request: https://github.com/llvm/llvm-project/pull/99486
---
 lld/ELF/Arch/LoongArch.cpp      |  9 +++++++
 lld/test/ELF/loongarch-tls-le.s | 47 ++++++++++++++++++++++++++++++---
 2 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp
index c6ee73f23d471..9466e8b1ce54d 100644
--- a/lld/ELF/Arch/LoongArch.cpp
+++ b/lld/ELF/Arch/LoongArch.cpp
@@ -11,6 +11,7 @@
 #include "Symbols.h"
 #include "SyntheticSections.h"
 #include "Target.h"
+#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Support/LEB128.h"
 
 using namespace llvm;
@@ -407,7 +408,9 @@ RelExpr LoongArch::getRelExpr(const RelType type, const Symbol &s,
   case R_LARCH_TLS_TPREL32:
   case R_LARCH_TLS_TPREL64:
   case R_LARCH_TLS_LE_HI20:
+  case R_LARCH_TLS_LE_HI20_R:
   case R_LARCH_TLS_LE_LO12:
+  case R_LARCH_TLS_LE_LO12_R:
   case R_LARCH_TLS_LE64_LO20:
   case R_LARCH_TLS_LE64_HI12:
     return R_TPREL;
@@ -490,6 +493,7 @@ RelExpr LoongArch::getRelExpr(const RelType type, const Symbol &s,
     return R_TLSLD_GOT;
   case R_LARCH_TLS_GD_HI20:
     return R_TLSGD_GOT;
+  case R_LARCH_TLS_LE_ADD_R:
   case R_LARCH_RELAX:
     return config->relax ? R_RELAX_HINT : R_NONE;
   case R_LARCH_ALIGN:
@@ -617,6 +621,7 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel,
   case R_LARCH_TLS_LE_LO12:
   case R_LARCH_TLS_IE_PC_LO12:
   case R_LARCH_TLS_IE_LO12:
+  case R_LARCH_TLS_LE_LO12_R:
   case R_LARCH_TLS_DESC_PC_LO12:
   case R_LARCH_TLS_DESC_LO12:
     write32le(loc, setK12(read32le(loc), extractBits(val, 11, 0)));
@@ -638,6 +643,9 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel,
   case R_LARCH_TLS_DESC_HI20:
     write32le(loc, setJ20(read32le(loc), extractBits(val, 31, 12)));
     return;
+  case R_LARCH_TLS_LE_HI20_R:
+    write32le(loc, setJ20(read32le(loc), extractBits(val + 0x800, 31, 12)));
+    return;
 
   // Relocs intended for `lu32i.d`.
   case R_LARCH_ABS64_LO20:
@@ -707,6 +715,7 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel,
     // no-op
     return;
 
+  case R_LARCH_TLS_LE_ADD_R:
   case R_LARCH_RELAX:
     return; // Ignored (for now)
 
diff --git a/lld/test/ELF/loongarch-tls-le.s b/lld/test/ELF/loongarch-tls-le.s
index a20d7d83bae3f..394c60f67bce8 100644
--- a/lld/test/ELF/loongarch-tls-le.s
+++ b/lld/test/ELF/loongarch-tls-le.s
@@ -1,14 +1,14 @@
 # REQUIRES: loongarch
 
-# RUN: llvm-mc --filetype=obj --triple=loongarch32 %s -o %t.32.o
+# RUN: llvm-mc --filetype=obj --triple=loongarch32 --defsym ELF32=1 %s -o %t.32.o
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 %s -o %t.64.o
 
 # RUN: ld.lld %t.32.o -o %t.32
 # RUN: llvm-nm -p %t.32 | FileCheck --check-prefixes=NM %s
-# RUN: llvm-objdump -d --no-show-raw-insn %t.32 | FileCheck --check-prefixes=LE %s
+# RUN: llvm-objdump -d --no-show-raw-insn %t.32 | FileCheck --check-prefixes=LE,LE32 %s
 
 # RUN: ld.lld %t.64.o -o %t.64
-# RUN: llvm-objdump -d --no-show-raw-insn %t.64 | FileCheck --check-prefixes=LE %s
+# RUN: llvm-objdump -d --no-show-raw-insn %t.64 | FileCheck --check-prefixes=LE,LE64 %s
 
 # RUN: not ld.lld -shared %t.32.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR --implicit-check-not=error:
 
@@ -16,6 +16,10 @@
 # ERR: error: relocation R_LARCH_TLS_LE_LO12 against .LANCHOR0 cannot be used with -shared
 # ERR: error: relocation R_LARCH_TLS_LE_HI20 against a cannot be used with -shared
 # ERR: error: relocation R_LARCH_TLS_LE_LO12 against a cannot be used with -shared
+# ERR: error: relocation R_LARCH_TLS_LE_HI20_R against .LANCHOR0 cannot be used with -shared
+# ERR: error: relocation R_LARCH_TLS_LE_LO12_R against .LANCHOR0 cannot be used with -shared
+# ERR: error: relocation R_LARCH_TLS_LE_HI20_R against a cannot be used with -shared
+# ERR: error: relocation R_LARCH_TLS_LE_LO12_R against a cannot be used with -shared
 
 # NM: {{0*}}00000008 b .LANCHOR0
 # NM: {{0*}}00000800 B a
@@ -26,13 +30,50 @@
 # LE-NEXT: ori $a0, $a0, 8
 # LE-NEXT: lu12i.w $a1, 0
 # LE-NEXT: ori $a1, $a1, 2048
+
+# LE32:      add.w   $a0, $a0, $tp
+# LE32-NEXT: addi.w  $a0, $a0, 8
+# LE32-NEXT: lu12i.w $a0, 1
+# LE32-NEXT: add.w   $a0, $a0, $tp
+# LE32-NEXT: addi.w  $a0, $a0, -2048
+
+# LE64:      add.d   $a0, $a0, $tp
+# LE64-NEXT: addi.d  $a0, $a0, 8
+# LE64-NEXT: lu12i.w $a0, 1
+# LE64-NEXT: add.d   $a0, $a0, $tp
+# LE64-NEXT: addi.d  $a0, $a0, -2048
+
 # LE-EMPTY:
 
+.macro add dst, src1, src2, src3
+.ifdef ELF32
+add.w \dst, \src1, \src2, \src3
+.else
+add.d \dst, \src1, \src2, \src3
+.endif
+.endm
+.macro addi dst, src1, src2
+.ifdef ELF32
+addi.w \dst, \src1, \src2
+.else
+addi.d \dst, \src1, \src2
+.endif
+.endm
+
 .text
+
 _start:
 la.tls.le $a0, .LANCHOR0
 la.tls.le $a1, a
 
+lu12i.w $a0, %le_hi20_r(.LANCHOR0)
+add $a0, $a0, $tp, %le_add_r(.LANCHOR0)
+addi $a0, $a0, %le_lo12_r(.LANCHOR0)
+
+lu12i.w $a0, %le_hi20_r(a)
+add $a0, $a0, $tp, %le_add_r(a)
+addi $a0, $a0, %le_lo12_r(a)
+
 .section .tbss,"awT",@nobits
 .space 8
 .LANCHOR0:

From b8741cc185e89a95c9161218084f7c8b5578cf96 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 19 Jul 2024 10:33:07 +0100
Subject: [PATCH 175/486] [VPlan] Relax assertion retrieving a scalar from
 VPTransformState::get.

The current assertion VPTransformState::get when retrieving a single
scalar only does not account for cases where a def has multiple users,
some demanding all scalar lanes, some demanding only a single scalar.

For an example, see the modified test case. Relax the assertion by also
allowing requesting scalar lanes only when the Def doesn't have only its
first lane used.

Fixes https://github.com/llvm/llvm-project/issues/88849.
---
 llvm/lib/Transforms/Vectorize/VPlan.cpp       |  1 +
 ...demanding-all-lanes-and-first-lane-only.ll | 98 ++++++++++++++++++-
 2 files changed, 97 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 83a035fb4df86..58de6256900f0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -254,6 +254,7 @@ Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) {
 Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
   if (NeedsScalar) {
     assert((VF.isScalar() || Def->isLiveIn() || hasVectorValue(Def, Part) ||
+            !vputils::onlyFirstLaneUsed(Def) ||
             (hasScalarValue(Def, VPIteration(Part, 0)) &&
              Data.PerPartScalars[Def][Part].size() == 1)) &&
            "Trying to access a single scalar per part but has multiple scalars "
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll
index c673e9690a667..fcf1ba072a62c 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll
@@ -1,5 +1,5 @@
-; REQUIRES: asserts
-; RUN: not --crash opt -p loop-vectorize -mtriple=s390x-unknown-linux -mcpu=z16 %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -mtriple=s390x-unknown-linux -mcpu=z16 -S %s | FileCheck %s
 
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 
@@ -9,6 +9,94 @@ target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 ; all scalar lanes) and a VPInstruction that only demands the first lane.
 ; Test case for https://github.com/llvm/llvm-project/issues/88849.
 define void @test_scalar_iv_steps_used_by_replicate_and_first_lane_only_vpinst(ptr noalias %dst, ptr noalias %src.1) {
+; CHECK-LABEL: define void @test_scalar_iv_steps_used_by_replicate_and_first_lane_only_vpinst(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC_1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nsw i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nsw i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nsw i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[SRC_1]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[SRC_1]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[SRC_1]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[SRC_1]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP8]], align 1
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr [[TMP10]], align 1
+; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr [[TMP11]], align 1
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <4 x i8> poison, i8 [[TMP12]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x i8> [[TMP16]], i8 [[TMP13]], i32 1
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <4 x i8> [[TMP17]], i8 [[TMP14]], i32 2
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x i8> [[TMP18]], i8 [[TMP15]], i32 3
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq <4 x i8> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr [8 x i32], ptr @src, i64 0, i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr i32, ptr [[TMP22]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i1> [[TMP20]], i32 0
+; CHECK-NEXT:    br i1 [[TMP24]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 0
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[DST]], align 4
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; CHECK:       [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x i1> [[TMP20]], i32 1
+; CHECK-NEXT:    br i1 [[TMP26]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; CHECK:       [[PRED_STORE_IF1]]:
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT:    store i32 [[TMP27]], ptr [[DST]], align 4
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x i1> [[TMP20]], i32 2
+; CHECK-NEXT:    br i1 [[TMP28]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; CHECK:       [[PRED_STORE_IF3]]:
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 2
+; CHECK-NEXT:    store i32 [[TMP29]], ptr [[DST]], align 4
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
+; CHECK:       [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <4 x i1> [[TMP20]], i32 3
+; CHECK-NEXT:    br i1 [[TMP30]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; CHECK:       [[PRED_STORE_IF5]]:
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3
+; CHECK-NEXT:    store i32 [[TMP31]], ptr [[DST]], align 4
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
+; CHECK:       [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[MUL_IV:%.*]] = mul nsw i64 [[IV]], 4
+; CHECK-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds i8, ptr [[SRC_1]], i64 [[MUL_IV]]
+; CHECK-NEXT:    [[L_1:%.*]] = load i8, ptr [[GEP_SRC_1]], align 1
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[L_1]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[IV_OR:%.*]] = or disjoint i64 [[IV]], 4
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds [8 x i32], ptr @src, i64 0, i64 [[IV_OR]]
+; CHECK-NEXT:    [[L_2:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    store i32 [[L_2]], ptr [[DST]], align 4
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 4
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop.header
 
@@ -35,3 +123,9 @@ loop.latch:
 exit:
   ret void
 }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.

From 812694c449b222e960a39ccb517a074c08db4a71 Mon Sep 17 00:00:00 2001
From: Bjorn Pettersson <bjorn.a.pettersson@ericsson.com>
Date: Thu, 4 Jul 2024 17:00:53 +0200
Subject: [PATCH 176/486] [ValueTracking] Pre-commit ComputeNumSignBits test
 for (shl (zext X), C)

Adding a test case for potential simplifications of
  (shl (zext X), C)
based on number of known sign bits in X.
---
 .../Analysis/ValueTracking/numsignbits-shl.ll | 168 ++++++++++++++++++
 1 file changed, 168 insertions(+)
 create mode 100644 llvm/test/Analysis/ValueTracking/numsignbits-shl.ll

diff --git a/llvm/test/Analysis/ValueTracking/numsignbits-shl.ll b/llvm/test/Analysis/ValueTracking/numsignbits-shl.ll
new file mode 100644
index 0000000000000..c32b6ff12b426
--- /dev/null
+++ b/llvm/test/Analysis/ValueTracking/numsignbits-shl.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s
+
+declare void @escape(i16 %add)
+declare void @escape2(<2 x i16> %add)
+
+define void @numsignbits_shl_zext(i8 %x) {
+; CHECK-LABEL: define void @numsignbits_shl_zext(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i8 [[X]], 5
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[ASHR]] to i16
+; CHECK-NEXT:    [[NSB4:%.*]] = shl i16 [[ZEXT]], 10
+; CHECK-NEXT:    [[AND14:%.*]] = and i16 [[NSB4]], 16384
+; CHECK-NEXT:    [[ADD14:%.*]] = add i16 [[AND14]], [[NSB4]]
+; CHECK-NEXT:    call void @escape(i16 [[ADD14]])
+; CHECK-NEXT:    [[AND13:%.*]] = and i16 [[NSB4]], 8192
+; CHECK-NEXT:    [[ADD13:%.*]] = add i16 [[AND13]], [[NSB4]]
+; CHECK-NEXT:    call void @escape(i16 [[ADD13]])
+; CHECK-NEXT:    [[AND12:%.*]] = and i16 [[NSB4]], 4096
+; CHECK-NEXT:    [[ADD12:%.*]] = add i16 [[AND12]], [[NSB4]]
+; CHECK-NEXT:    call void @escape(i16 [[ADD12]])
+; CHECK-NEXT:    [[AND11:%.*]] = and i16 [[NSB4]], 2048
+; CHECK-NEXT:    [[ADD11:%.*]] = add i16 [[AND11]], [[NSB4]]
+; CHECK-NEXT:    call void @escape(i16 [[ADD11]])
+; CHECK-NEXT:    ret void
+;
+  %ashr = ashr i8 %x, 5
+  %zext = zext i8 %ashr to i16
+  %nsb4 = shl i16 %zext, 10
+  ; Validate ComputeNumSignBits using this simplification:
+  ;   (A & 2^C1) + A => A & (2^C1 - 1) iff bit C1 in A is a sign bit
+  ; 4 sign bits: Goal is to fold away the add for bits 12-14.
+  %and14 = and i16 %nsb4, 16384
+  %add14 = add i16 %and14, %nsb4
+  call void @escape(i16 %add14)
+  %and13 = and i16 %nsb4, 8192
+  %add13 = add i16 %and13, %nsb4
+  call void @escape(i16 %add13)
+  %and12 = and i16 %nsb4, 4096
+  %add12 = add i16 %and12, %nsb4
+  call void @escape(i16 %add12)
+  %and11 = and i16 %nsb4, 2048
+  %add11 = add i16 %and11, %nsb4
+  call void @escape(i16 %add11)
+  ret void
+}
+
+define void @numsignbits_shl_zext_shift_amounr_matches_extend(i8 %x) {
+; CHECK-LABEL: define void @numsignbits_shl_zext_shift_amounr_matches_extend(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i8 [[X]], 2
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[ASHR]] to i16
+; CHECK-NEXT:    [[NSB3:%.*]] = shl nuw i16 [[ZEXT]], 8
+; CHECK-NEXT:    [[AND14:%.*]] = and i16 [[NSB3]], 16384
+; CHECK-NEXT:    [[ADD14:%.*]] = add i16 [[AND14]], [[NSB3]]
+; CHECK-NEXT:    call void @escape(i16 [[ADD14]])
+; CHECK-NEXT:    [[AND13:%.*]] = and i16 [[NSB3]], 8192
+; CHECK-NEXT:    [[ADD13:%.*]] = add i16 [[AND13]], [[NSB3]]
+; CHECK-NEXT:    call void @escape(i16 [[ADD13]])
+; CHECK-NEXT:    [[AND12:%.*]] = and i16 [[NSB3]], 4096
+; CHECK-NEXT:    [[ADD12:%.*]] = add i16 [[AND12]], [[NSB3]]
+; CHECK-NEXT:    call void @escape(i16 [[ADD12]])
+; CHECK-NEXT:    ret void
+;
+  %ashr = ashr i8 %x, 2
+  %zext = zext i8 %ashr to i16
+  %nsb3 = shl i16 %zext, 8
+  ; Validate ComputeNumSignBits using this simplification:
+  ;   (A & 2^C1) + A => A & (2^C1 - 1) iff bit C1 in A is a sign bit
+  ; 3 sign bits: Goal is to fold away the add for bits 13-14.
+  %and14 = and i16 %nsb3, 16384
+  %add14 = add i16 %and14, %nsb3
+  call void @escape(i16 %add14)
+  %and13 = and i16 %nsb3, 8192
+  %add13 = add i16 %and13, %nsb3
+  call void @escape(i16 %add13)
+  %and12 = and i16 %nsb3, 4096
+  %add12 = add i16 %and12, %nsb3
+  call void @escape(i16 %add12)
+  ret void
+}
+
+define void @numsignbits_shl_zext_extended_bits_remains(i8 %x) {
+; CHECK-LABEL: define void @numsignbits_shl_zext_extended_bits_remains(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i8 [[X]], 5
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[ASHR]] to i16
+; CHECK-NEXT:    [[NSB1:%.*]] = shl nuw nsw i16 [[ZEXT]], 7
+; CHECK-NEXT:    [[AND14:%.*]] = and i16 [[NSB1]], 16384
+; CHECK-NEXT:    [[ADD14:%.*]] = add nuw i16 [[AND14]], [[NSB1]]
+; CHECK-NEXT:    call void @escape(i16 [[ADD14]])
+; CHECK-NEXT:    ret void
+;
+  %ashr = ashr i8 %x, 5
+  %zext = zext i8 %ashr to i16
+  %nsb1 = shl i16 %zext, 7
+  ; Validate ComputeNumSignBits using this simplification:
+  ;   (A & 2^C1) + A => A & (2^C1 - 1) iff bit C1 in A is a sign bit
+  ; 1 sign bit: The add can't be folded away here.
+  %and14 = and i16 %nsb1, 16384
+  %add14 = add i16 %and14, %nsb1
+  call void @escape(i16 %add14)
+  ret void
+}
+
+define void @numsignbits_shl_zext_all_bits_shifted_out(i8 %x) {
+; CHECK-LABEL: define void @numsignbits_shl_zext_all_bits_shifted_out(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ASHR:%.*]] = lshr i8 [[X]], 5
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext nneg i8 [[ASHR]] to i16
+; CHECK-NEXT:    [[NSB1:%.*]] = shl i16 [[ZEXT]], 14
+; CHECK-NEXT:    [[AND14:%.*]] = and i16 [[NSB1]], 16384
+; CHECK-NEXT:    [[ADD14:%.*]] = add i16 [[AND14]], [[NSB1]]
+; CHECK-NEXT:    call void @escape(i16 [[ADD14]])
+; CHECK-NEXT:    ret void
+;
+  %ashr = ashr i8 %x, 5
+  %zext = zext i8 %ashr to i16
+  %nsb1 = shl i16 %zext, 14
+  ; Validate ComputeNumSignBits using this simplification:
+  ;   (A & 2^C1) + A => A & (2^C1 - 1) iff bit C1 in A is a sign bit
+  ; 1 sign bit: The add can't be folded away here.
+  %and14 = and i16 %nsb1, 16384
+  %add14 = add i16 %and14, %nsb1
+  call void @escape(i16 %add14)
+  ret void
+}
+
+define void @numsignbits_shl_zext_vector(<2 x i8> %x) {
+; CHECK-LABEL: define void @numsignbits_shl_zext_vector(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i8> [[X]], <i8 5, i8 5>
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext <2 x i8> [[ASHR]] to <2 x i16>
+; CHECK-NEXT:    [[NSB4:%.*]] = shl <2 x i16> [[ZEXT]], <i16 10, i16 10>
+; CHECK-NEXT:    [[AND14:%.*]] = and <2 x i16> [[NSB4]], <i16 16384, i16 16384>
+; CHECK-NEXT:    [[ADD14:%.*]] = add <2 x i16> [[AND14]], [[NSB4]]
+; CHECK-NEXT:    call void @escape2(<2 x i16> [[ADD14]])
+; CHECK-NEXT:    [[AND13:%.*]] = and <2 x i16> [[NSB4]], <i16 8192, i16 8192>
+; CHECK-NEXT:    [[ADD13:%.*]] = add <2 x i16> [[AND13]], [[NSB4]]
+; CHECK-NEXT:    call void @escape2(<2 x i16> [[ADD13]])
+; CHECK-NEXT:    [[AND12:%.*]] = and <2 x i16> [[NSB4]], <i16 4096, i16 4096>
+; CHECK-NEXT:    [[ADD12:%.*]] = add <2 x i16> [[AND12]], [[NSB4]]
+; CHECK-NEXT:    call void @escape2(<2 x i16> [[ADD12]])
+; CHECK-NEXT:    [[AND11:%.*]] = and <2 x i16> [[NSB4]], <i16 2048, i16 2048>
+; CHECK-NEXT:    [[ADD11:%.*]] = add <2 x i16> [[AND11]], [[NSB4]]
+; CHECK-NEXT:    call void @escape2(<2 x i16> [[ADD11]])
+; CHECK-NEXT:    ret void
+;
+  %ashr = ashr <2 x i8> %x, <i8 5, i8 5>
+  %zext = zext <2 x i8> %ashr to <2 x i16>
+  %nsb4 = shl <2 x i16> %zext, <i16 10, i16 10>
+  ; Validate ComputeNumSignBits using this simplification:
+  ;   (A & 2^C1) + A => A & (2^C1 - 1) iff bit C1 in A is a sign bit
+  ; 4 sign bits: Goal is to fold away the add for bits 12-14.
+  %and14 = and <2 x i16> %nsb4, <i16 16384, i16 16384>
+  %add14 = add <2 x i16> %and14, %nsb4
+  call void @escape2(<2 x i16> %add14)
+  %and13 = and <2 x i16> %nsb4, <i16 8192, i16 8192>
+  %add13 = add <2 x i16> %and13, %nsb4
+  call void @escape2(<2 x i16> %add13)
+  %and12 = and <2 x i16> %nsb4, <i16 4096, i16 4096>
+  %add12 = add <2 x i16> %and12, %nsb4
+  call void @escape2(<2 x i16> %add12)
+  %and11 = and <2 x i16> %nsb4, <i16 2048, i16 2048>
+  %add11 = add <2 x i16> %and11, %nsb4
+  call void @escape2(<2 x i16> %add11)
+  ret void
+}

From 098bd842a7e50853fa231f8b73c24ec5006fe063 Mon Sep 17 00:00:00 2001
From: Bjorn Pettersson <bjorn.a.pettersson@ericsson.com>
Date: Thu, 18 Jul 2024 11:54:22 +0200
Subject: [PATCH 177/486] [ValueTracking] Let ComputeKnownSignBits handle (shl
 (zext X), C) (#97693)

Add simple support for looking through a zext when doing
ComputeKnownSignBits for shl. This is valid for the case when
all extended bits are shifted out, because then the number of sign
bits can be found by analysing the zext operand.

The solution here is simple as it only handle a single zext (not
passing remaining left shift amount during recursion). It could be
possible to generalize this in the future by for example passing an
'OffsetFromMSB' parameter to ComputeNumSignBitsImpl, telling it to
calculate number of sign bits starting at some offset from the most
significant bit.
---
 llvm/lib/Analysis/ValueTracking.cpp           | 17 +++++++++--
 .../Analysis/ValueTracking/numsignbits-shl.ll | 30 +++++++------------
 2 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 535a248a5f1a2..03eb6ef42b0ff 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -3866,11 +3866,22 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
     }
     case Instruction::Shl: {
       const APInt *ShAmt;
+      Value *X = nullptr;
       if (match(U->getOperand(1), m_APInt(ShAmt))) {
         // shl destroys sign bits.
-        Tmp = ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q);
-        if (ShAmt->uge(TyBits) ||   // Bad shift.
-            ShAmt->uge(Tmp)) break; // Shifted all sign bits out.
+        if (ShAmt->uge(TyBits))
+          break; // Bad shift.
+        // We can look through a zext (more or less treating it as a sext) if
+        // all extended bits are shifted out.
+        if (match(U->getOperand(0), m_ZExt(m_Value(X))) &&
+            ShAmt->uge(TyBits - X->getType()->getScalarSizeInBits())) {
+          Tmp = ComputeNumSignBits(X, DemandedElts, Depth + 1, Q);
+          Tmp += TyBits - X->getType()->getScalarSizeInBits();
+        } else
+          Tmp =
+              ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q);
+        if (ShAmt->uge(Tmp))
+          break; // Shifted all sign bits out.
         Tmp2 = ShAmt->getZExtValue();
         return Tmp - Tmp2;
       }
diff --git a/llvm/test/Analysis/ValueTracking/numsignbits-shl.ll b/llvm/test/Analysis/ValueTracking/numsignbits-shl.ll
index c32b6ff12b426..e86d28ebbc1d2 100644
--- a/llvm/test/Analysis/ValueTracking/numsignbits-shl.ll
+++ b/llvm/test/Analysis/ValueTracking/numsignbits-shl.ll
@@ -10,17 +10,14 @@ define void @numsignbits_shl_zext(i8 %x) {
 ; CHECK-NEXT:    [[ASHR:%.*]] = ashr i8 [[X]], 5
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[ASHR]] to i16
 ; CHECK-NEXT:    [[NSB4:%.*]] = shl i16 [[ZEXT]], 10
-; CHECK-NEXT:    [[AND14:%.*]] = and i16 [[NSB4]], 16384
-; CHECK-NEXT:    [[ADD14:%.*]] = add i16 [[AND14]], [[NSB4]]
+; CHECK-NEXT:    [[ADD14:%.*]] = and i16 [[NSB4]], 15360
 ; CHECK-NEXT:    call void @escape(i16 [[ADD14]])
-; CHECK-NEXT:    [[AND13:%.*]] = and i16 [[NSB4]], 8192
-; CHECK-NEXT:    [[ADD13:%.*]] = add i16 [[AND13]], [[NSB4]]
+; CHECK-NEXT:    [[ADD13:%.*]] = and i16 [[NSB4]], 7168
 ; CHECK-NEXT:    call void @escape(i16 [[ADD13]])
-; CHECK-NEXT:    [[AND12:%.*]] = and i16 [[NSB4]], 4096
-; CHECK-NEXT:    [[ADD12:%.*]] = add i16 [[AND12]], [[NSB4]]
+; CHECK-NEXT:    [[ADD12:%.*]] = and i16 [[NSB4]], 3072
 ; CHECK-NEXT:    call void @escape(i16 [[ADD12]])
 ; CHECK-NEXT:    [[AND11:%.*]] = and i16 [[NSB4]], 2048
-; CHECK-NEXT:    [[ADD11:%.*]] = add i16 [[AND11]], [[NSB4]]
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i16 [[AND11]], [[NSB4]]
 ; CHECK-NEXT:    call void @escape(i16 [[ADD11]])
 ; CHECK-NEXT:    ret void
 ;
@@ -51,14 +48,12 @@ define void @numsignbits_shl_zext_shift_amounr_matches_extend(i8 %x) {
 ; CHECK-NEXT:    [[ASHR:%.*]] = ashr i8 [[X]], 2
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[ASHR]] to i16
 ; CHECK-NEXT:    [[NSB3:%.*]] = shl nuw i16 [[ZEXT]], 8
-; CHECK-NEXT:    [[AND14:%.*]] = and i16 [[NSB3]], 16384
-; CHECK-NEXT:    [[ADD14:%.*]] = add i16 [[AND14]], [[NSB3]]
+; CHECK-NEXT:    [[ADD14:%.*]] = and i16 [[NSB3]], 16128
 ; CHECK-NEXT:    call void @escape(i16 [[ADD14]])
-; CHECK-NEXT:    [[AND13:%.*]] = and i16 [[NSB3]], 8192
-; CHECK-NEXT:    [[ADD13:%.*]] = add i16 [[AND13]], [[NSB3]]
+; CHECK-NEXT:    [[ADD13:%.*]] = and i16 [[NSB3]], 7936
 ; CHECK-NEXT:    call void @escape(i16 [[ADD13]])
 ; CHECK-NEXT:    [[AND12:%.*]] = and i16 [[NSB3]], 4096
-; CHECK-NEXT:    [[ADD12:%.*]] = add i16 [[AND12]], [[NSB3]]
+; CHECK-NEXT:    [[ADD12:%.*]] = add nsw i16 [[AND12]], [[NSB3]]
 ; CHECK-NEXT:    call void @escape(i16 [[ADD12]])
 ; CHECK-NEXT:    ret void
 ;
@@ -132,17 +127,14 @@ define void @numsignbits_shl_zext_vector(<2 x i8> %x) {
 ; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i8> [[X]], <i8 5, i8 5>
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext <2 x i8> [[ASHR]] to <2 x i16>
 ; CHECK-NEXT:    [[NSB4:%.*]] = shl <2 x i16> [[ZEXT]], <i16 10, i16 10>
-; CHECK-NEXT:    [[AND14:%.*]] = and <2 x i16> [[NSB4]], <i16 16384, i16 16384>
-; CHECK-NEXT:    [[ADD14:%.*]] = add <2 x i16> [[AND14]], [[NSB4]]
+; CHECK-NEXT:    [[ADD14:%.*]] = and <2 x i16> [[NSB4]], <i16 15360, i16 15360>
 ; CHECK-NEXT:    call void @escape2(<2 x i16> [[ADD14]])
-; CHECK-NEXT:    [[AND13:%.*]] = and <2 x i16> [[NSB4]], <i16 8192, i16 8192>
-; CHECK-NEXT:    [[ADD13:%.*]] = add <2 x i16> [[AND13]], [[NSB4]]
+; CHECK-NEXT:    [[ADD13:%.*]] = and <2 x i16> [[NSB4]], <i16 7168, i16 7168>
 ; CHECK-NEXT:    call void @escape2(<2 x i16> [[ADD13]])
-; CHECK-NEXT:    [[AND12:%.*]] = and <2 x i16> [[NSB4]], <i16 4096, i16 4096>
-; CHECK-NEXT:    [[ADD12:%.*]] = add <2 x i16> [[AND12]], [[NSB4]]
+; CHECK-NEXT:    [[ADD12:%.*]] = and <2 x i16> [[NSB4]], <i16 3072, i16 3072>
 ; CHECK-NEXT:    call void @escape2(<2 x i16> [[ADD12]])
 ; CHECK-NEXT:    [[AND11:%.*]] = and <2 x i16> [[NSB4]], <i16 2048, i16 2048>
-; CHECK-NEXT:    [[ADD11:%.*]] = add <2 x i16> [[AND11]], [[NSB4]]
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw <2 x i16> [[AND11]], [[NSB4]]
 ; CHECK-NEXT:    call void @escape2(<2 x i16> [[ADD11]])
 ; CHECK-NEXT:    ret void
 ;

From 008df3cf85e9bb1532c079bfd7a7a00e90e0a3c6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 19 Jul 2024 12:02:25 +0100
Subject: [PATCH 178/486] [LV] Check isPredInst instead of isScalarWithPred in
 uniform analysis. (#98892)

Any instruction marked as uniform will result in a uniform
VPReplicateRecipe. If it requires predication, it will be placed in a
replicate region, even if isScalarWithPredication returns false.

Check isPredicatedInst instead of isScalarWithPredication to avoid
generating uniform VPReplicateRecipes placed inside a replicate region.
This fixes an assertion when using scalable VFs.

Fixes https://github.com/llvm/llvm-project/issues/80416.
Fixes https://github.com/llvm/llvm-project/issues/94328.
Fixes https://github.com/llvm/llvm-project/issues/99625.

PR: https://github.com/llvm/llvm-project/pull/98892
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |  12 +-
 .../AArch64/divs-with-scalable-vfs.ll         | 303 +++++++++++++++++-
 .../X86/consecutive-ptr-uniforms.ll           |   2 +-
 3 files changed, 309 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index fbca4cdcbcfcd..cceed75aa50b0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3914,19 +3914,19 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
   SetVector<Instruction *> Worklist;
 
   // Add uniform instructions demanding lane 0 to the worklist. Instructions
-  // that are scalar with predication must not be considered uniform after
+  // that require predication must not be considered uniform after
   // vectorization, because that would create an erroneous replicating region
   // where only a single instance out of VF should be formed.
-  // TODO: optimize such seldom cases if found important, see PR40816.
   auto addToWorklistIfAllowed = [&](Instruction *I) -> void {
     if (isOutOfScope(I)) {
       LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
                         << *I << "\n");
       return;
     }
-    if (isScalarWithPredication(I, VF)) {
-      LLVM_DEBUG(dbgs() << "LV: Found not uniform being ScalarWithPredication: "
-                        << *I << "\n");
+    if (isPredicatedInst(I)) {
+      LLVM_DEBUG(
+          dbgs() << "LV: Found not uniform due to requiring predication: " << *I
+                 << "\n");
       return;
     }
     LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
@@ -9516,6 +9516,8 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
 void VPReplicateRecipe::execute(VPTransformState &State) {
   Instruction *UI = getUnderlyingInstr();
   if (State.Instance) { // Generate a single instance.
+    assert((State.VF.isScalar() || !isUniform()) &&
+           "uniform recipe shouldn't be predicated");
     assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
     State.ILV->scalarizeInstruction(UI, this, *State.Instance, State);
     // Insert scalar instance packing it into a vector.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
index a6a53b927696a..a6c07aaaabc64 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
@@ -1,8 +1,91 @@
-; REQUIRES: asserts
-; RUN: not --crash opt -p loop-vectorize -mtriple aarch64 -mcpu=neoverse-v1 -S %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -mtriple aarch64 -mcpu=neoverse-v1 -S %s | FileCheck %s
 
 ; Test case for https://github.com/llvm/llvm-project/issues/94328.
 define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
+; CHECK-LABEL: define void @sdiv_feeding_gep(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X:%.*]], i64 [[M:%.*]], i64 [[CONV6:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CONV61:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp slt i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ugt i64 [[TMP3]], 4294967295
+; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP9]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 2
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sdiv i64 [[M]], [[CONV6]]
+; CHECK-NEXT:    [[TMP19:%.*]] = sdiv i64 [[M]], [[CONV6]]
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP18]] to i32
+; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP19]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP18]], [[CONV61]]
+; CHECK-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP19]], [[CONV61]]
+; CHECK-NEXT:    [[TMP24:%.*]] = sub i64 [[TMP12]], [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = sub i64 [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP24]] to i32
+; CHECK-NEXT:    [[TMP27:%.*]] = trunc i64 [[TMP25]] to i32
+; CHECK-NEXT:    [[TMP28:%.*]] = mul i32 [[X]], [[TMP20]]
+; CHECK-NEXT:    [[TMP29:%.*]] = mul i32 [[X]], [[TMP21]]
+; CHECK-NEXT:    [[TMP30:%.*]] = add i32 [[TMP28]], [[TMP26]]
+; CHECK-NEXT:    [[TMP31:%.*]] = add i32 [[TMP29]], [[TMP27]]
+; CHECK-NEXT:    [[TMP32:%.*]] = sext i32 [[TMP30]] to i64
+; CHECK-NEXT:    [[TMP33:%.*]] = sext i32 [[TMP31]] to i64
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP32]]
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP33]]
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr double, ptr [[TMP34]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP38:%.*]] = mul i64 [[TMP37]], 2
+; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr double, ptr [[TMP34]], i64 [[TMP38]]
+; CHECK-NEXT:    store <vscale x 2 x double> zeroinitializer, ptr [[TMP36]], align 8
+; CHECK-NEXT:    store <vscale x 2 x double> zeroinitializer, ptr [[TMP39]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT:    [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[DIV18:%.*]] = sdiv i64 [[M]], [[CONV6]]
+; CHECK-NEXT:    [[CONV20:%.*]] = trunc i64 [[DIV18]] to i32
+; CHECK-NEXT:    [[MUL30:%.*]] = mul i64 [[DIV18]], [[CONV61]]
+; CHECK-NEXT:    [[SUB31:%.*]] = sub i64 [[IV]], [[MUL30]]
+; CHECK-NEXT:    [[CONV34:%.*]] = trunc i64 [[SUB31]] to i32
+; CHECK-NEXT:    [[MUL35:%.*]] = mul i32 [[X]], [[CONV20]]
+; CHECK-NEXT:    [[ADD36:%.*]] = add i32 [[MUL35]], [[CONV34]]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[ADD36]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr double, ptr [[DST]], i64 [[IDXPROM]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   %conv61 = zext i32 %x to i64
   br label %loop
@@ -28,6 +111,101 @@ exit:
 }
 
 define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
+; CHECK-LABEL: define void @sdiv_feeding_gep_predicated(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X:%.*]], i64 [[M:%.*]], i64 [[CONV6:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CONV61:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP0]], 4294967295
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT:    [[TMP15:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 2 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = mul <vscale x 2 x i64> [[TMP16]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 2
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 1, [[TMP19]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP20]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[M]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[CONV6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp ule <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP22]], <vscale x 2 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = select <vscale x 2 x i1> [[TMP23]], <vscale x 2 x i64> [[BROADCAST_SPLAT2]], <vscale x 2 x i64> shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP25:%.*]] = sdiv <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <vscale x 2 x i64> [[TMP25]], i32 0
+; CHECK-NEXT:    [[TMP27:%.*]] = trunc i64 [[TMP26]] to i32
+; CHECK-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP26]], [[CONV61]]
+; CHECK-NEXT:    [[TMP29:%.*]] = sub i64 [[TMP21]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP29]] to i32
+; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[X]], [[TMP27]]
+; CHECK-NEXT:    [[TMP32:%.*]] = add i32 [[TMP31]], [[TMP30]]
+; CHECK-NEXT:    [[TMP33:%.*]] = sext i32 [[TMP32]] to i64
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr double, ptr [[TMP34]], i32 0
+; CHECK-NEXT:    call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> zeroinitializer, ptr [[TMP35]], i32 8, <vscale x 2 x i1> [[TMP23]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP14]])
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <vscale x 2 x i1> [[TMP36]], i32 0
+; CHECK-NEXT:    br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[C:%.*]] = icmp ule i64 [[IV]], [[M]]
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[DIV18:%.*]] = sdiv i64 [[M]], [[CONV6]]
+; CHECK-NEXT:    [[CONV20:%.*]] = trunc i64 [[DIV18]] to i32
+; CHECK-NEXT:    [[MUL30:%.*]] = mul i64 [[DIV18]], [[CONV61]]
+; CHECK-NEXT:    [[SUB31:%.*]] = sub i64 [[IV]], [[MUL30]]
+; CHECK-NEXT:    [[CONV34:%.*]] = trunc i64 [[SUB31]] to i32
+; CHECK-NEXT:    [[MUL35:%.*]] = mul i32 [[X]], [[CONV20]]
+; CHECK-NEXT:    [[ADD36:%.*]] = add i32 [[MUL35]], [[CONV34]]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[ADD36]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr double, ptr [[DST]], i64 [[IDXPROM]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP]], align 8
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   %conv61 = zext i32 %x to i64
   br label %loop
@@ -61,6 +239,116 @@ exit:
 
 ; Test case for https://github.com/llvm/llvm-project/issues/80416.
 define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
+; CHECK-LABEL: define void @udiv_urem_feeding_gep(
+; CHECK-SAME: i64 [[X:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MUL_1_I:%.*]] = mul i64 [[X]], [[X]]
+; CHECK-NEXT:    [[MUL_2_I:%.*]] = mul i64 [[MUL_1_I]], [[X]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[N]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[N]], 4294967295
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP7]]
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[TMP0]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[TMP0]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP15:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 2 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = mul <vscale x 2 x i64> [[TMP16]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 2
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 1, [[TMP19]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP20]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[MUL_2_I]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[MUL_1_I]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i64> shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP22:%.*]] = udiv <vscale x 2 x i64> [[VEC_IND]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i64> shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP24:%.*]] = urem <vscale x 2 x i64> [[VEC_IND]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> [[BROADCAST_SPLAT2]], <vscale x 2 x i64> shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP26:%.*]] = udiv <vscale x 2 x i64> [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> [[BROADCAST_SPLAT2]], <vscale x 2 x i64> shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP28:%.*]] = urem <vscale x 2 x i64> [[TMP24]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> [[BROADCAST_SPLAT4]], <vscale x 2 x i64> shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP30:%.*]] = udiv <vscale x 2 x i64> [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> [[BROADCAST_SPLAT4]], <vscale x 2 x i64> shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP32:%.*]] = urem <vscale x 2 x i64> [[TMP28]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <vscale x 2 x i64> [[TMP22]], i32 0
+; CHECK-NEXT:    [[TMP34:%.*]] = mul i64 [[X]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <vscale x 2 x i64> [[TMP26]], i32 0
+; CHECK-NEXT:    [[TMP36:%.*]] = add i64 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = mul i64 [[TMP36]], [[X]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <vscale x 2 x i64> [[TMP30]], i32 0
+; CHECK-NEXT:    [[TMP39:%.*]] = add i64 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = mul i64 [[TMP39]], [[X]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <vscale x 2 x i64> [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP42:%.*]] = add i64 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = shl i64 [[TMP42]], 32
+; CHECK-NEXT:    [[TMP44:%.*]] = ashr i64 [[TMP43]], 32
+; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr i64, ptr [[TMP45]], i32 0
+; CHECK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP22]], ptr [[TMP46]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP14]])
+; CHECK-NEXT:    [[TMP47:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <vscale x 2 x i1> [[TMP47]], i32 0
+; CHECK-NEXT:    br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[DIV_I:%.*]] = udiv i64 [[IV]], [[MUL_2_I]]
+; CHECK-NEXT:    [[REM_I:%.*]] = urem i64 [[IV]], [[MUL_2_I]]
+; CHECK-NEXT:    [[DIV_1_I:%.*]] = udiv i64 [[REM_I]], [[MUL_1_I]]
+; CHECK-NEXT:    [[REM_1_I:%.*]] = urem i64 [[REM_I]], [[MUL_1_I]]
+; CHECK-NEXT:    [[DIV_2_I:%.*]] = udiv i64 [[REM_1_I]], [[X]]
+; CHECK-NEXT:    [[REM_2_I:%.*]] = urem i64 [[REM_1_I]], [[X]]
+; CHECK-NEXT:    [[MUL_I:%.*]] = mul i64 [[X]], [[DIV_I]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add i64 [[MUL_I]], [[DIV_1_I]]
+; CHECK-NEXT:    [[MUL_1_I9:%.*]] = mul i64 [[ADD_I]], [[X]]
+; CHECK-NEXT:    [[ADD_1_I:%.*]] = add i64 [[MUL_1_I9]], [[DIV_2_I]]
+; CHECK-NEXT:    [[MUL_2_I11:%.*]] = mul i64 [[ADD_1_I]], [[X]]
+; CHECK-NEXT:    [[ADD_2_I:%.*]] = add i64 [[MUL_2_I11]], [[REM_2_I]]
+; CHECK-NEXT:    [[SEXT_I:%.*]] = shl i64 [[ADD_2_I]], 32
+; CHECK-NEXT:    [[CONV6_I:%.*]] = ashr i64 [[SEXT_I]], 32
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[DST]], i64 [[CONV6_I]]
+; CHECK-NEXT:    store i64 [[DIV_I]], ptr [[GEP]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   %mul.1.i = mul i64 %x, %x
   %mul.2.i = mul i64 %mul.1.i, %x
@@ -91,3 +379,14 @@ loop:
 exit:
   ret void
 }
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
index bee5b397ecb47..bb35806cea20a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
@@ -76,7 +76,7 @@ attributes #0 = { "target-cpu"="knl" }
 ; CHECK:     LV: Found trip count: 3
 ; CHECK:     LV: Found uniform instruction:   {{%.*}} = icmp eq i32 {{%.*}}, 0
 ; CHECK-NOT: LV: Found uniform instruction:   {{%.*}} = load i32, ptr {{%.*}}, align 1
-; CHECK:     LV: Found not uniform being ScalarWithPredication:  {{%.*}} = load i32, ptr {{%.*}}, align 1
+; CHECK:     LV: Found not uniform due to requiring predication:  {{%.*}} = load i32, ptr {{%.*}}, align 1
 ; CHECK:     LV: Found scalar instruction:   {{%.*}} = getelementptr inbounds [3 x i32], ptr @a, i32 0, i32 {{%.*}}
 ;
 ; FORCE-LABEL: @PR40816(

From 42a7c424b65050c5522e7055ab1486b572d86b69 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Fri, 19 Jul 2024 13:03:40 +0200
Subject: [PATCH 179/486] [clang] Add `std::span` to the default gsl pointer
 annotation list. (#99622)

---
 clang/lib/Sema/SemaAttr.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index aaabd989c5c9f..5a7f12c7689d4 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -193,6 +193,7 @@ void Sema::inferGslOwnerPointerAttribute(CXXRecordDecl *Record) {
       "basic_string_view",
       "reference_wrapper",
       "regex_iterator",
+      "span",
   };
 
   if (!Record->getIdentifier())

From 243af2ff217f65ef291232faa60779b86e01a967 Mon Sep 17 00:00:00 2001
From: Vladislav Dzhidzhoev <vdzhidzhoev@accesssoftek.com>
Date: Fri, 19 Jul 2024 13:08:07 +0200
Subject: [PATCH 180/486] [LLDB][test] Improve SHELL detection on Windows in
 Makefile.rules (#99532)

In MinGW make, the `%windir%` variable has a lowercase name `$(windir)`
when launched from cmd.exe, and `$(WINDIR)` name when launched from
MSYS2, since MSYS2 represents standard Windows environment variables
names in upper case.

This commit makes Makefile.rules consider both run variants.
---
 lldb/packages/Python/lldbsuite/test/make/Makefile.rules | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
index 597aa94566c24..be3ad684dd736 100644
--- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
+++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
@@ -55,8 +55,10 @@ LLDB_BASE_DIR := $(THIS_FILE_DIR)/../../../../../
 # Also reset BUILDDIR value because "pwd" returns cygwin or msys path
 # which needs to be converted to windows path.
 #----------------------------------------------------------------------
-ifeq "$(OS)" "Windows_NT"
-	SHELL = $(WINDIR)\system32\cmd.exe
+ifeq "$(HOST_OS)" "Windows_NT"
+	# MinGW make gets $(windir) variable if launched from cmd.exe
+	# and $(WINDIR) if launched from MSYS2.
+	SHELL := $(or $(windir),$(WINDIR),C:\WINDOWS)\system32\cmd.exe
 	BUILDDIR := $(shell echo %cd%)
 endif
 

From e404eed24bebd5e3e04fc153eb330bae7d92107f Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Fri, 19 Jul 2024 13:04:43 +0200
Subject: [PATCH 181/486] [clang] Add the `const` to all default lists in
 SemaAttr.cpp, NFC

Address the comment in https://github.com/llvm/llvm-project/pull/99622#issuecomment-2238800532
---
 clang/lib/Sema/SemaAttr.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index 5a7f12c7689d4..b0c239678d0b0 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -117,7 +117,7 @@ void Sema::inferGslPointerAttribute(NamedDecl *ND,
   if (!Parent)
     return;
 
-  static llvm::StringSet<> Containers{
+  static const llvm::StringSet<> Containers{
       "array",
       "basic_string",
       "deque",
@@ -137,9 +137,9 @@ void Sema::inferGslPointerAttribute(NamedDecl *ND,
       "unordered_multimap",
   };
 
-  static llvm::StringSet<> Iterators{"iterator", "const_iterator",
-                                     "reverse_iterator",
-                                     "const_reverse_iterator"};
+  static const llvm::StringSet<> Iterators{"iterator", "const_iterator",
+                                           "reverse_iterator",
+                                           "const_reverse_iterator"};
 
   if (Parent->isInStdNamespace() && Iterators.count(ND->getName()) &&
       Containers.count(Parent->getName()))
@@ -165,7 +165,7 @@ void Sema::inferGslPointerAttribute(TypedefNameDecl *TD) {
 }
 
 void Sema::inferGslOwnerPointerAttribute(CXXRecordDecl *Record) {
-  static llvm::StringSet<> StdOwners{
+  static const llvm::StringSet<> StdOwners{
       "any",
       "array",
       "basic_regex",
@@ -189,7 +189,7 @@ void Sema::inferGslOwnerPointerAttribute(CXXRecordDecl *Record) {
       "unordered_multimap",
       "variant",
   };
-  static llvm::StringSet<> StdPointers{
+  static const llvm::StringSet<> StdPointers{
       "basic_string_view",
       "reference_wrapper",
       "regex_iterator",
@@ -217,7 +217,7 @@ void Sema::inferGslOwnerPointerAttribute(CXXRecordDecl *Record) {
 }
 
 void Sema::inferNullableClassAttribute(CXXRecordDecl *CRD) {
-  static llvm::StringSet<> Nullable{
+  static const llvm::StringSet<> Nullable{
       "auto_ptr",         "shared_ptr", "unique_ptr",         "exception_ptr",
       "coroutine_handle", "function",   "move_only_function",
   };

From fdfc49186318727653cf6b13686bb77cfed60e33 Mon Sep 17 00:00:00 2001
From: dlav-sc <daniil.avdeev@syntacore.com>
Date: Fri, 19 Jul 2024 14:22:18 +0300
Subject: [PATCH 182/486] [lldb] SHT_NOBITS sections type (#99044)

.sbss section was recognized as eSectionTypeOther, but has to be
eSectionTypeZeroFill.

Fixed tests for RISCV target:

TestClassLoadingViaMemberTypedef.TestCase

TestClassTemplateNonTypeParameterPack.TestCaseClassTemplateNonTypeParameterPack
TestThreadSelectionBug.TestThreadSelectionBug
lldbsuite.test.lldbtest.TestOffsetof
lldbsuite.test.lldbtest.TestOffsetofCpp
TestTargetDumpTypeSystem.TestCase
TestCPP11EnumTypes.CPP11EnumTypesTestCase
TestCppIsTypeComplete.TestCase
TestExprCrash.ExprCrashTestCase
TestDebugIndexCache.DebugIndexCacheTestcase
---
 lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index 51bd34e95c77d..890db5c274814 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -1696,7 +1696,6 @@ static SectionType GetSectionTypeFromName(llvm::StringRef Name) {
   return llvm::StringSwitch<SectionType>(Name)
       .Case(".ARM.exidx", eSectionTypeARMexidx)
       .Case(".ARM.extab", eSectionTypeARMextab)
-      .Cases(".bss", ".tbss", eSectionTypeZeroFill)
       .Case(".ctf", eSectionTypeDebug)
       .Cases(".data", ".tdata", eSectionTypeData)
       .Case(".eh_frame", eSectionTypeEHFrame)
@@ -1713,6 +1712,10 @@ SectionType ObjectFileELF::GetSectionType(const ELFSectionHeaderInfo &H) const {
     if (H.sh_flags & SHF_EXECINSTR)
       return eSectionTypeCode;
     break;
+  case SHT_NOBITS:
+    if (H.sh_flags & SHF_ALLOC)
+      return eSectionTypeZeroFill;
+    break;
   case SHT_SYMTAB:
     return eSectionTypeELFSymbolTable;
   case SHT_DYNSYM:

From 6da23b647ea0c7961a31742e3daac80f4c08e99c Mon Sep 17 00:00:00 2001
From: Dan Liew <dan@su-root.co.uk>
Date: Fri, 19 Jul 2024 12:35:01 +0100
Subject: [PATCH 183/486] [BoundsSafety] Add `-fexperimental-bounds-safety` CC1
 and language option and use it to tweak `counted_by`'s semantics (#92623)

This adds the `-fexperimental-bounds-safety` cc1 and corresponding
language option. This language option enables "-fbounds-safety" which is
a bounds-safety extension for C that is being incrementally upstreamed.

This cc1 flag is not exposed as a driver flag yet because most of the
implementation isn't upstream yet.

The language option is used to make a small semantic change to how the
`counted_by` attribute is treated. Without
`-fexperimental-bounds-safety` the attribute is allowed (but emits a
warning) on a flexible array member where the element type is a struct
with a flexible array member. With the flag this situation is an error.

E.g.

```
struct has_unannotated_FAM {
  int count;
  char buffer[];
};

struct buffer_of_structs_with_unnannotated_FAM {
  int count;
  // Forbidden with `-fexperimental-bounds-safety`
  struct has_unannotated_FAM Arr[] __counted_by(count);
};
```

rdar://125400392
---
 clang/include/clang/Basic/LangOptions.def     |  3 ++
 clang/include/clang/Driver/Options.td         |  8 ++++
 clang/lib/Sema/SemaDeclAttr.cpp               |  2 +-
 .../Sema/attr-counted-by-bounds-safety-vlas.c | 37 +++++++++++++++++++
 4 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Sema/attr-counted-by-bounds-safety-vlas.c

diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 5ad9c1f24b7c5..a6f36b23f07dc 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -520,6 +520,9 @@ BENIGN_LANGOPT(CheckNew, 1, 0, "Do not assume C++ operator new may not return NU
 BENIGN_LANGOPT(CheckConstexprFunctionBodies, 1, 1,
                "Emit diagnostics for a constexpr function body that can never "
                "be used in a constant expression.")
+
+LANGOPT(BoundsSafety, 1, 0, "Bounds safety extension for C")
+
 #undef LANGOPT
 #undef COMPATIBLE_LANGOPT
 #undef BENIGN_LANGOPT
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index fecfa2f3c7dba..5229940ec8258 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1911,6 +1911,14 @@ def fapinotes_swift_version : Joined<["-"], "fapinotes-swift-version=">,
   MetaVarName<"<version>">,
   HelpText<"Specify the Swift version to use when filtering API notes">;
 
+defm bounds_safety : BoolFOption<
+  "experimental-bounds-safety",
+  LangOpts<"BoundsSafety">, DefaultFalse,
+  PosFlag<SetTrue, [], [CC1Option], "Enable">,
+  NegFlag<SetFalse, [], [CC1Option], "Disable">,
+  BothFlags<[], [CC1Option],
+          " experimental bounds safety extension for C">>;
+
 defm addrsig : BoolFOption<"addrsig",
   CodeGenOpts<"Addrsig">, DefaultFalse,
   PosFlag<SetTrue, [], [ClangOption, CC1Option], "Emit">,
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 41295bfb3b94f..f2f3bfcd09035 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -5948,7 +5948,7 @@ CheckCountedByAttrOnField(Sema &S, FieldDecl *FD, Expr *E,
   } else if (PointeeTy->isFunctionType()) {
     InvalidTypeKind = CountedByInvalidPointeeTypeKind::FUNCTION;
   } else if (PointeeTy->isStructureTypeWithFlexibleArrayMember()) {
-    if (FieldTy->isArrayType()) {
+    if (FieldTy->isArrayType() && !S.getLangOpts().BoundsSafety) {
       // This is a workaround for the Linux kernel that has already adopted
       // `counted_by` on a FAM where the pointee is a struct with a FAM. This
       // should be an error because computing the bounds of the array cannot be
diff --git a/clang/test/Sema/attr-counted-by-bounds-safety-vlas.c b/clang/test/Sema/attr-counted-by-bounds-safety-vlas.c
new file mode 100644
index 0000000000000..7d9c9a90880ff
--- /dev/null
+++ b/clang/test/Sema/attr-counted-by-bounds-safety-vlas.c
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -fsyntax-only -fexperimental-bounds-safety -verify %s
+//
+// This is a portion of the `attr-counted-by-vla.c` test but is checked
+// under the semantics of `-fexperimental-bounds-safety` which has different
+// behavior.
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+struct has_unannotated_VLA {
+  int count;
+  char buffer[];
+};
+
+struct has_annotated_VLA {
+  int count;
+  char buffer[] __counted_by(count);
+};
+
+struct buffer_of_structs_with_unnannotated_vla {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to an array with element of unknown size because 'struct has_unannotated_VLA' is a struct type with a flexible array member}}
+  struct has_unannotated_VLA Arr[] __counted_by(count);
+};
+
+
+struct buffer_of_structs_with_annotated_vla {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to an array with element of unknown size because 'struct has_annotated_VLA' is a struct type with a flexible array member}}
+  struct has_annotated_VLA Arr[] __counted_by(count);
+};
+
+struct buffer_of_const_structs_with_annotated_vla {
+  int count;
+  // Make sure the `const` qualifier is printed when printing the element type.
+  // expected-error@+1{{'counted_by' cannot be applied to an array with element of unknown size because 'const struct has_annotated_VLA' is a struct type with a flexible array member}}
+  const struct has_annotated_VLA Arr[] __counted_by(count);
+};

From ef47bbb4715de5790308cac166d8cd05020e8e18 Mon Sep 17 00:00:00 2001
From: jameshu15869 <55058507+jameshu15869@users.noreply.github.com>
Date: Fri, 19 Jul 2024 06:57:55 -0500
Subject: [PATCH 184/486] [libc] Add AMDGPU Timing to CMake (#99603)

`libc/benchmarks/gpu/timing/CMakeLists.txt` did not correctly build
`amdgpu` utils. This PR fixes that issue by adding `amdgpu` to the loop
that adds the correct sub directories.
---
 libc/benchmarks/gpu/timing/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/benchmarks/gpu/timing/CMakeLists.txt b/libc/benchmarks/gpu/timing/CMakeLists.txt
index 8bbc7e33f122a..b6d84607aa607 100644
--- a/libc/benchmarks/gpu/timing/CMakeLists.txt
+++ b/libc/benchmarks/gpu/timing/CMakeLists.txt
@@ -1,4 +1,4 @@
-foreach(target nvptx)
+foreach(target nvptx amdgpu)
   add_subdirectory(${target})
   list(APPEND target_gpu_timing libc.benchmarks.gpu.timing.${target}.${target}_timing)
 endforeach()

From 123c036bd361de9ed6baa0090e5942105764e8db Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 19 Jul 2024 13:03:21 +0100
Subject: [PATCH 185/486] Recommit "[TBAA] Emit distinct TBAA tags for pointers
 with different depths,types. (#76612)"

This reverts the revert commit bee240367cc48bbc93fe5eb57d537968dfe4419f.

This version includes updates to the tests to use patterns when matching
the pointer argument.

Original commit message:

    This patch extends Clang's TBAA generation code to emit distinct tags
    for incompatible pointer types.

    Pointers with different element types are incompatible if the pointee
    types are also incompatible (modulo sugar/modifiers).

    Express this in TBAA by generating different tags for pointers based on
    the pointer depth and pointee type. To get the TBAA tag for the pointee
    type it uses getTypeInfoHelper on the pointee type.

    (Moved from https://reviews.llvm.org/D122573)

    PR: https://github.com/llvm/llvm-project/pull/76612
---
 clang/docs/ReleaseNotes.rst                   |   7 +
 clang/include/clang/Basic/CodeGenOptions.def  |   1 +
 clang/include/clang/Driver/Options.td         |   5 +
 clang/lib/CodeGen/CodeGenTBAA.cpp             |  54 +-
 clang/lib/Driver/ToolChains/Clang.cpp         |   3 +
 .../CodeGen/sanitize-metadata-nosanitize.c    |  46 +-
 clang/test/CodeGen/tbaa-pointers.c            | 176 +++--
 .../attr-likelihood-iteration-stmt.cpp        | 132 ++--
 ...arallel_reduction_codegen_tbaa_PR46146.cpp | 620 +++++++++---------
 9 files changed, 572 insertions(+), 472 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 4efdb076ba768..aa91663260755 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -435,6 +435,10 @@ Non-comprehensive list of changes in this release
 - Added support for ``TypeLoc::dump()`` for easier debugging, and improved
   textual and JSON dumping for various ``TypeLoc``-related nodes.
 
+- Clang can now emit distinct type-based alias analysis tags for incompatible
+  pointers, enabling more powerful alias analysis when accessing pointer types.
+  The new behavior can be enabled using ``-fpointer-tbaa``.
+
 New Compiler Flags
 ------------------
 - ``-fsanitize=implicit-bitfield-conversion`` checks implicit truncation and
@@ -477,6 +481,9 @@ New Compiler Flags
 - For the ARM target, added ``-Warm-interrupt-vfp-clobber`` that will emit a
   diagnostic when an interrupt handler is declared and VFP is enabled.
 
+- ``-fpointer-tbaa`` enables emission of distinct type-based alias
+  analysis tags for incompatible pointers.
+
 Deprecated Compiler Flags
 -------------------------
 
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index f671b780bcbeb..12808eb275fa4 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -234,6 +234,7 @@ ENUM_CODEGENOPT(StructReturnConvention, StructReturnConventionKind, 2, SRCK_Defa
 
 CODEGENOPT(RelaxAll          , 1, 0) ///< Relax all machine code instructions.
 CODEGENOPT(RelaxedAliasing   , 1, 0) ///< Set when -fno-strict-aliasing is enabled.
+CODEGENOPT(PointerTBAA, 1, 0)        ///< Whether or not to use distinct TBAA tags for pointers.
 CODEGENOPT(StructPathTBAA    , 1, 0) ///< Whether or not to use struct-path TBAA.
 CODEGENOPT(NewStructPathTBAA , 1, 0) ///< Whether or not to use enhanced struct-path TBAA.
 CODEGENOPT(SaveTempLabels    , 1, 0) ///< Save temporary labels.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 5229940ec8258..8707e71f2a319 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3389,6 +3389,7 @@ def fstruct_path_tbaa : Flag<["-"], "fstruct-path-tbaa">, Group<f_Group>;
 def fno_struct_path_tbaa : Flag<["-"], "fno-struct-path-tbaa">, Group<f_Group>;
 def fno_strict_enums : Flag<["-"], "fno-strict-enums">, Group<f_Group>;
 def fno_strict_overflow : Flag<["-"], "fno-strict-overflow">, Group<f_Group>;
+def fno_pointer_tbaa : Flag<["-"], "fno-pointer-tbaa">, Group<f_Group>;
 def fno_temp_file : Flag<["-"], "fno-temp-file">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>, HelpText<
   "Directly create compilation output files. This may lead to incorrect incremental builds if the compiler crashes">,
@@ -3893,6 +3894,7 @@ defm strict_vtable_pointers : BoolFOption<"strict-vtable-pointers",
             " overwriting polymorphic C++ objects">,
   NegFlag<SetFalse>>;
 def fstrict_overflow : Flag<["-"], "fstrict-overflow">, Group<f_Group>;
+def fpointer_tbaa : Flag<["-"], "fpointer-tbaa">, Group<f_Group>;
 def fdriver_only : Flag<["-"], "fdriver-only">, Flags<[NoXarchOption]>,
   Visibility<[ClangOption, CLOption, DXCOption]>,
   Group<Action_Group>, HelpText<"Only run the driver.">;
@@ -7173,6 +7175,9 @@ def fuse_register_sized_bitfield_access: Flag<["-"], "fuse-register-sized-bitfie
 def relaxed_aliasing : Flag<["-"], "relaxed-aliasing">,
   HelpText<"Turn off Type Based Alias Analysis">,
   MarshallingInfoFlag<CodeGenOpts<"RelaxedAliasing">>;
+def pointer_tbaa: Flag<["-"], "pointer-tbaa">,
+  HelpText<"Turn on Type Based Alias Analysis for pointer accesses">,
+  MarshallingInfoFlag<CodeGenOpts<"PointerTBAA">>;
 def no_struct_path_tbaa : Flag<["-"], "no-struct-path-tbaa">,
   HelpText<"Turn off struct-path aware Type Based Alias Analysis">,
   MarshallingInfoNegativeFlag<CodeGenOpts<"StructPathTBAA">>;
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index b889d1bfe003f..b66b6234c9d3d 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -186,10 +186,56 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
     return getChar();
 
   // Handle pointers and references.
-  // TODO: Implement C++'s type "similarity" and consider dis-"similar"
-  // pointers distinct.
-  if (Ty->isPointerType() || Ty->isReferenceType())
-    return createScalarTypeNode("any pointer", getChar(), Size);
+  //
+  // C has a very strict rule for pointer aliasing. C23 6.7.6.1p2:
+  //     For two pointer types to be compatible, both shall be identically
+  //     qualified and both shall be pointers to compatible types.
+  //
+  // This rule is impractically strict; we want to at least ignore CVR
+  // qualifiers. Distinguishing by CVR qualifiers would make it UB to
+  // e.g. cast a `char **` to `const char * const *` and dereference it,
+  // which is too common and useful to invalidate. C++'s similar types
+  // rule permits qualifier differences in these nested positions; in fact,
+  // C++ even allows that cast as an implicit conversion.
+  //
+  // Other qualifiers could theoretically be distinguished, especially if
+  // they involve a significant representation difference.  We don't
+  // currently do so, however.
+  //
+  // Computing the pointee type string recursively is implicitly more
+  // forgiving than the standards require.  Effectively, we are turning
+  // the question "are these types compatible/similar" into "are
+  // accesses to these types allowed to alias".  In both C and C++,
+  // the latter question has special carve-outs for signedness
+  // mismatches that only apply at the top level.  As a result, we are
+  // allowing e.g. `int *` l-values to access `unsigned *` objects.
+  if (Ty->isPointerType() || Ty->isReferenceType()) {
+    llvm::MDNode *AnyPtr = createScalarTypeNode("any pointer", getChar(), Size);
+    if (!CodeGenOpts.PointerTBAA)
+      return AnyPtr;
+    // Compute the depth of the pointer and generate a tag of the form "p<depth>
+    // <base type tag>".
+    unsigned PtrDepth = 0;
+    do {
+      PtrDepth++;
+      Ty = Ty->getPointeeType().getTypePtr();
+    } while (Ty->isPointerType());
+    // TODO: Implement C++'s type "similarity" and consider dis-"similar"
+    // pointers distinct for non-builtin types.
+    if (isa<BuiltinType>(Ty)) {
+      llvm::MDNode *ScalarMD = getTypeInfoHelper(Ty);
+      StringRef Name =
+          cast<llvm::MDString>(
+              ScalarMD->getOperand(CodeGenOpts.NewStructPathTBAA ? 2 : 0))
+              ->getString();
+      SmallString<256> OutName("p");
+      OutName += std::to_string(PtrDepth);
+      OutName += " ";
+      OutName += Name;
+      return createScalarTypeNode(OutName, AnyPtr, Size);
+    }
+    return AnyPtr;
+  }
 
   // Accesses to arrays are accesses to objects of their element types.
   if (CodeGenOpts.NewStructPathTBAA && Ty->isArrayType())
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 1fd6fba210042..c9f0fe0a3d60f 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -5736,6 +5736,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Args.hasFlag(options::OPT_fstrict_aliasing, StrictAliasingAliasOption,
                     options::OPT_fno_strict_aliasing, !IsWindowsMSVC))
     CmdArgs.push_back("-relaxed-aliasing");
+  if (Args.hasFlag(options::OPT_fpointer_tbaa, options::OPT_fno_pointer_tbaa,
+                   false))
+    CmdArgs.push_back("-pointer-tbaa");
   if (!Args.hasFlag(options::OPT_fstruct_path_tbaa,
                     options::OPT_fno_struct_path_tbaa, true))
     CmdArgs.push_back("-no-struct-path-tbaa");
diff --git a/clang/test/CodeGen/sanitize-metadata-nosanitize.c b/clang/test/CodeGen/sanitize-metadata-nosanitize.c
index 388a3df547a73..da0c809148018 100644
--- a/clang/test/CodeGen/sanitize-metadata-nosanitize.c
+++ b/clang/test/CodeGen/sanitize-metadata-nosanitize.c
@@ -12,7 +12,7 @@
 //.
 // CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(write, argmem: none, inaccessiblemem: none)
 // CHECK-LABEL: define dso_local void @escape
-// CHECK-SAME: (ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !pcsections !2 {
+// CHECK-SAME: (ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !pcsections [[META2:![0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret void
 //
@@ -23,13 +23,13 @@ __attribute__((noinline, not_tail_called)) void escape(const volatile void *p) {
 
 // CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none)
 // CHECK-LABEL: define dso_local i32 @normal_function
-// CHECK-SAME: (ptr noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] !pcsections !4 {
+// CHECK-SAME: (ptr noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] !pcsections [[META4:![0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[X]], ptr [[X_ADDR]], align 8, !tbaa [[TBAA6:![0-9]+]]
-// CHECK-NEXT:    store atomic i32 1, ptr [[X]] monotonic, align 4, !pcsections !10
+// CHECK-NEXT:    store atomic i32 1, ptr [[X]] monotonic, align 4, !pcsections [[META11:![0-9]+]]
 // CHECK-NEXT:    notail call void @escape(ptr noundef nonnull [[X_ADDR]])
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA11:![0-9]+]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA12:![0-9]+]]
 // CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int normal_function(int *x, int *y) {
@@ -46,7 +46,7 @@ int normal_function(int *x, int *y) {
 // CHECK-NEXT:    store ptr [[X]], ptr [[X_ADDR]], align 8, !tbaa [[TBAA6]]
 // CHECK-NEXT:    store atomic i32 1, ptr [[X]] monotonic, align 4
 // CHECK-NEXT:    notail call void @escape(ptr noundef nonnull [[X_ADDR]])
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA11]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA12]]
 // CHECK-NEXT:    ret i32 [[TMP0]]
 //
 __attribute__((disable_sanitizer_instrumentation)) int test_disable_sanitize_instrumentation(int *x, int *y) {
@@ -57,13 +57,13 @@ __attribute__((disable_sanitizer_instrumentation)) int test_disable_sanitize_ins
 
 // CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none)
 // CHECK-LABEL: define dso_local i32 @test_no_sanitize_thread
-// CHECK-SAME: (ptr noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] !pcsections !13 {
+// CHECK-SAME: (ptr noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] !pcsections [[META14:![0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[X]], ptr [[X_ADDR]], align 8, !tbaa [[TBAA6]]
-// CHECK-NEXT:    store atomic i32 1, ptr [[X]] monotonic, align 4, !pcsections !10
+// CHECK-NEXT:    store atomic i32 1, ptr [[X]] monotonic, align 4, !pcsections [[META11]]
 // CHECK-NEXT:    notail call void @escape(ptr noundef nonnull [[X_ADDR]])
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA11]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA12]]
 // CHECK-NEXT:    ret i32 [[TMP0]]
 //
 __attribute__((no_sanitize("thread"))) int test_no_sanitize_thread(int *x, int *y) {
@@ -74,13 +74,13 @@ __attribute__((no_sanitize("thread"))) int test_no_sanitize_thread(int *x, int *
 
 // CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none)
 // CHECK-LABEL: define dso_local i32 @test_no_sanitize_all
-// CHECK-SAME: (ptr noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR3]] !pcsections !13 {
+// CHECK-SAME: (ptr noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR3]] !pcsections [[META14]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[X]], ptr [[X_ADDR]], align 8, !tbaa [[TBAA6]]
-// CHECK-NEXT:    store atomic i32 1, ptr [[X]] monotonic, align 4, !pcsections !10
+// CHECK-NEXT:    store atomic i32 1, ptr [[X]] monotonic, align 4, !pcsections [[META11]]
 // CHECK-NEXT:    notail call void @escape(ptr noundef nonnull [[X_ADDR]])
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA11]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA12]]
 // CHECK-NEXT:    ret i32 [[TMP0]]
 //
 __attribute__((no_sanitize("all"))) int test_no_sanitize_all(int *x, int *y) {
@@ -89,23 +89,9 @@ __attribute__((no_sanitize("all"))) int test_no_sanitize_all(int *x, int *y) {
   return *y;
 }
 //.
-// CHECK: attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind willreturn memory(write, argmem: none, inaccessiblemem: none) "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
-// CHECK: attributes #1 = { mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none) "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
-// CHECK: attributes #2 = { disable_sanitizer_instrumentation mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none) "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
-// CHECK: attributes #3 = { mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none) "min-legal-vector-width"="0" "no-trapping-math"="true" "no_sanitize_thread" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
-// CHECK: attributes #4 = { nounwind "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
-//.
-// CHECK: !2 = !{!"sanmd_covered2!C", !3}
-// CHECK: !3 = !{i64 0}
-// CHECK: !4 = !{!"sanmd_covered2!C", !5}
-// CHECK: !5 = !{i64 3}
-// CHECK: !6 = !{!7, !7, i64 0}
-// CHECK: !7 = !{!"any pointer", !8, i64 0}
-// CHECK: !8 = !{!"omnipotent char", !9, i64 0}
-// CHECK: !9 = !{!"Simple C/C++ TBAA"}
-// CHECK: !10 = !{!"sanmd_atomics2!C"}
-// CHECK: !11 = !{!12, !12, i64 0}
-// CHECK: !12 = !{!"int", !8, i64 0}
-// CHECK: !13 = !{!"sanmd_covered2!C", !14}
-// CHECK: !14 = !{i64 2}
+// CHECK: attributes #[[ATTR0]] = { mustprogress nofree noinline norecurse nosync nounwind willreturn memory(write, argmem: none, inaccessiblemem: none) "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+// CHECK: attributes #[[ATTR1]] = { mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none) "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+// CHECK: attributes #[[ATTR2]] = { disable_sanitizer_instrumentation mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none) "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+// CHECK: attributes #[[ATTR3]] = { mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none) "min-legal-vector-width"="0" "no-trapping-math"="true" "no_sanitize_thread" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+// CHECK: attributes #[[ATTR4:[0-9]+]] = { nounwind "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
 //.
diff --git a/clang/test/CodeGen/tbaa-pointers.c b/clang/test/CodeGen/tbaa-pointers.c
index b9ebe87982001..f85fca5e34fc0 100644
--- a/clang/test/CodeGen/tbaa-pointers.c
+++ b/clang/test/CodeGen/tbaa-pointers.c
@@ -1,80 +1,115 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -disable-llvm-passes %s -emit-llvm -o - | FileCheck --check-prefixes=COMMON,DEFAULT %s
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -disable-llvm-passes -pointer-tbaa %s -emit-llvm -o - | FileCheck --check-prefixes=COMMON,ENABLED %s
+// RUN: %clang --target=x86_64-apple-darwin -O1 %s -emit-llvm -S -mllvm -disable-llvm-optzns -o - | FileCheck --check-prefixes=COMMON,DEFAULT %s
+// RUN: %clang --target=x86_64-apple-darwin -O1 -fpointer-tbaa %s -emit-llvm -S -mllvm -disable-llvm-optzns -o - | FileCheck --check-prefixes=COMMON,ENABLED %s
 
 void p2unsigned(unsigned **ptr) {
-  // CHECK-LABEL: define void @p2unsigned(ptr noundef %ptr)
-  // CHECK-NEXT: entry:
-  // CHECK-NEXT:  %ptr.addr = alloca ptr, align 8
-  // CHECK-NEXT:  store ptr %ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0:!.+]]
-  // CHECK-NEXT:  [[BASE:%.+]] = load ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:  store ptr null, ptr [[BASE]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:  ret void
+  // COMMON-LABEL: define void @p2unsigned(
+  // COMMON-SAME:    ptr noundef [[PTR:%.+]])
+  // COMMON-NEXT:  entry:
+  // COMMON-NEXT:   [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // ENABLED-NEXT:  store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0:!.+]]
+  // ENABLED-NEXT:  [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
+  // ENABLED-NEXT:  store ptr null, ptr [[BASE]], align 8, !tbaa [[P1INT_0:!.+]]
+  // DEFAULT-NEXT:  store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR:!.+]]
+  // DEFAULT-NEXT:  [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:  store ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
+  // COMMON-NEXT:  ret void
   //
   *ptr = 0;
 }
 
 void p2unsigned_volatile(unsigned *volatile *ptr) {
-  // CHECK-LABEL: define void @p2unsigned_volatile(ptr noundef %ptr)
-  // CHECK-NEXT: entry:
-  // CHECK-NEXT:   %ptr.addr = alloca ptr, align 8
-  // CHECK-NEXT:   store ptr %ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE:%.+]] = load ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   store volatile ptr null, ptr [[BASE]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   ret void
+  // COMMON-LABEL: define void @p2unsigned_volatile(
+  // COMMON-SAME:    ptr noundef [[PTR:%.+]])
+  // COMMON-NEXT:  entry:
+  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
+  // ENABLED-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
+  // ENABLED-NEXT:   store volatile ptr null, ptr [[BASE]], align 8, !tbaa [[P1INT_0]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store volatile ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
+  // COMMON-NEXT:   ret void
   //
   *ptr = 0;
 }
 
 void p3int(int ***ptr) {
-  // CHECK-LABEL: define void @p3int(ptr noundef %ptr)
-  // CHECK-NEXT: entry:
-  // CHECK-NEXT:   %ptr.addr = alloca ptr, align 8
-  // CHECK-NEXT:   store ptr %ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_0:%.+]] = load ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   store ptr null, ptr [[BASE_1]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   ret void
+  // COMMON-LABEL: define void @p3int(
+  // COMMON-SAME:    ptr noundef [[PTR:%.+]])
+  // COMMON-NEXT:  entry:
+  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P3INT_0:!.+]]
+  // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P3INT_0]]
+  // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P2INT_0]]
+  // ENABLED-NEXT:   store ptr null, ptr [[BASE_1]], align 8, !tbaa [[P1INT_0]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
+
+  // COMMON-NEXT:   ret void
   //
   **ptr = 0;
 }
 
 void p4char(char ****ptr) {
-  // CHECK-LABEL: define void @p4char(ptr noundef %ptr)
-  // CHECK-NEXT: entry:
-  // CHECK-NEXT:   %ptr.addr = alloca ptr, align 8
-  // CHECK-NEXT:   store ptr %ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_0:%.+]] = load ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   ret void
+  // COMMON-LABEL: define void @p4char(
+  // COMMON-SAME:    ptr noundef [[PTR:%.+]])
+  // COMMON-NEXT:  entry:
+  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0:!.+]]
+  // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
+  // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0:!.+]]
+  // ENABLED-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[P2CHAR_0:!.+]]
+  // ENABLED-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[P1CHAR_0:!.+]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
+  // COMMON-NEXT:   ret void
   //
   ***ptr = 0;
 }
 
 void p4char_const1(const char ****ptr) {
-  // CHECK-LABEL: define void @p4char_const1(ptr noundef %ptr)
-  // CHECK-NEXT: entry:
-  // CHECK-NEXT:   %ptr.addr = alloca ptr, align 8
-  // CHECK-NEXT:   store ptr %ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_0:%.+]] = load ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   ret void
+  // COMMON-LABEL: define void @p4char_const1(
+  // COMMON-SAME:    ptr noundef [[PTR:%.+]])
+  // COMMON-NEXT:  entry:
+  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
+  // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
+  // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0]]
+  // ENABLED-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[P2CHAR_0]]
+  // ENABLED-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[P1CHAR_0]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
+  // COMMON-NEXT:   ret void
   //
   ***ptr = 0;
 }
 
 void p4char_const2(const char **const **ptr) {
-  // CHECK-LABEL: define void @p4char_const2(ptr noundef %ptr)
-  // CHECK-NEXT: entry:
-  // CHECK-NEXT:   %ptr.addr = alloca ptr, align 8
-  // CHECK-NEXT:   store ptr %ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_0:%.+]] = load ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   ret void
+  // COMMON-LABEL: define void @p4char_const2(
+  // COMMON-SAME:    ptr noundef [[PTR:%.+]])
+  // COMMON-NEXT:  entry:
+  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
+  // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
+  // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0]]
+  // ENABLED-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[P2CHAR_0]]
+  // ENABLED-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[P1CHAR_0]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
+  // COMMON-NEXT:   ret void
   //
   ***ptr = 0;
 }
@@ -85,19 +120,36 @@ struct S1 {
 };
 
 void p2struct(struct S1 **ptr) {
-  // CHECK-LABEL: define void @p2struct(ptr noundef %ptr)
-  // CHECK-NEXT: entry:
-  // CHECK-NEXT:   %ptr.addr = alloca ptr, align 8
-  // CHECK-NEXT:   store ptr %ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   [[BASE:%.+]] = load ptr, ptr %ptr.addr, align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   store ptr null, ptr [[BASE]], align 8, !tbaa [[ANY_POINTER_0]]
-  // CHECK-NEXT:   ret void
+  // COMMON-LABEL: define void @p2struct(
+  // COMMON-SAME:    ptr noundef [[PTR:%.+]])
+  // COMMON-NEXT:  entry:
+  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2S1_0:!.+]]
+  // ENABLED-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2S1_0]]
+  // ENABLED-NEXT:   store ptr null, ptr [[BASE]], align 8, !tbaa [[P1S1_:!.+]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
+  // COMMON-NEXT:   ret void
   //
   *ptr = 0;
 }
 
-// CHECK: [[ANY_POINTER_0]] = !{[[ANY_POINTER:!.+]], [[ANY_POINTER]], i64 0}
-// CHECK: [[ANY_POINTER]] = !{!"any pointer", [[CHAR:!.+]], i64 0}
-// CHECK: [[CHAR]] = !{!"omnipotent char", [[TBAA_ROOT:!.+]], i64 0}
-// CHECK: [[TBAA_ROOT]] = !{!"Simple C/C++ TBAA"}
-//
+// ENABLED: [[P2INT_0]] = !{[[P2INT:!.+]], [[P2INT]], i64 0}
+// ENABLED: [[P2INT]] = !{!"p2 int", [[ANY_POINTER:!.+]], i64 0}
+// DEFAULT: [[ANYPTR]] = !{[[ANY_POINTER:!.+]], [[ANY_POINTER]], i64 0}
+// COMMON: [[ANY_POINTER]] = !{!"any pointer", [[CHAR:!.+]], i64 0}
+// COMMON: [[CHAR]] = !{!"omnipotent char", [[TBAA_ROOT:!.+]], i64 0}
+// COMMON: [[TBAA_ROOT]] = !{!"Simple C/C++ TBAA"}
+// ENABLED: [[P1INT_0]] = !{[[P1INT:!.+]], [[P1INT]], i64 0}
+// ENABLED: [[P1INT]] = !{!"p1 int", [[ANY_POINTER]], i64 0}
+// ENABLED: [[P3INT_0]] = !{[[P3INT:!.+]], [[P3INT]], i64 0}
+// ENABLED: [[P3INT]] = !{!"p3 int", [[ANY_POINTER]], i64 0}
+// ENABLED: [[P4CHAR_0]] = !{[[P4CHAR:!.+]], [[P4CHAR]], i64 0}
+// ENABLED: [[P4CHAR]] = !{!"p4 omnipotent char", [[ANY_POINTER]], i64 0}
+// ENABLED: [[P3CHAR_0]] = !{[[P3CHAR:!.+]], [[P3CHAR]], i64 0}
+// ENABLED: [[P3CHAR]] = !{!"p3 omnipotent char", [[ANY_POINTER]], i64 0}
+// ENABLED: [[P2CHAR_0]] = !{[[P2CHAR:!.+]], [[P2CHAR]], i64 0}
+// ENABLED: [[P2CHAR]] = !{!"p2 omnipotent char", [[ANY_POINTER]], i64 0}
+// ENABLED: [[P1CHAR_0]] = !{[[P1CHAR:!.+]], [[P1CHAR]], i64 0}
+// ENABLED: [[P1CHAR]] = !{!"p1 omnipotent char", [[ANY_POINTER]], i64 0}
diff --git a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
index 412edd6d3edc0..e842de6335046 100644
--- a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
+++ b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
@@ -68,23 +68,23 @@ void w_branch_elided(unsigned e){
 // CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store i32 [[E:%.*]], ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4:[0-9]+]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR3:[0-9]+]]
 // CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK:       for.cond:
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[I]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP0]], [[TMP1]]
 // CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
 // CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 // CHECK:       for.cond.cleanup:
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_END:%.*]]
 // CHECK:       for.body:
 // CHECK-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK:       for.inc:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    [[INC:%.*]] = add i32 [[TMP4]], 1
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add i32 [[TMP2]], 1
 // CHECK-NEXT:    store i32 [[INC]], ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
 // CHECK:       for.end:
@@ -100,23 +100,23 @@ void fl(unsigned e)
 // CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store i32 [[E:%.*]], ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK:       for.cond:
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[I]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP0]], [[TMP1]]
 // CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
 // CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 // CHECK:       for.cond.cleanup:
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_END:%.*]]
 // CHECK:       for.body:
 // CHECK-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK:       for.inc:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK-NEXT:    store i32 [[INC]], ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK:       for.end:
@@ -146,42 +146,42 @@ void f_branch_elided()
 // CHECK-NEXT:    [[__END1:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store ptr [[E:%.*]], ptr [[E_ADDR]], align 8, !tbaa [[TBAA14:![0-9]+]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    store ptr [[TMP1]], ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP3]], i64 0, i64 0
-// CHECK-NEXT:    store ptr [[ARRAYDECAY]], ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__END1]]) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP5]], i64 0, i64 0
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR3]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR3]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP1]], i64 0, i64 0
+// CHECK-NEXT:    store ptr [[ARRAYDECAY]], ptr [[__BEGIN1]], align 8, !tbaa [[TBAA16:![0-9]+]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__END1]]) #[[ATTR3]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP2]], i64 0, i64 0
 // CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, ptr [[ARRAYDECAY1]], i64 4
-// CHECK-NEXT:    store ptr [[ADD_PTR]], ptr [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    store ptr [[ADD_PTR]], ptr [[__END1]], align 8, !tbaa [[TBAA16]]
 // CHECK-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK:       for.cond:
-// CHECK-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__END1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[CMP:%.*]] = icmp ne ptr [[TMP6]], [[TMP7]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[__END1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne ptr [[TMP3]], [[TMP4]]
 // CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
 // CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 // CHECK:       for.cond.cleanup:
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__END1]]) #[[ATTR4]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR4]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR4]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__END1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_END:%.*]]
 // CHECK:       for.body:
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    store i32 [[TMP13]], ptr [[I]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i32 [[TMP6]], ptr [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK:       for.inc:
-// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i32 1
-// CHECK-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
+// CHECK-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 1
+// CHECK-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
 // CHECK:       for.end:
 // CHECK-NEXT:    ret void
 //
@@ -198,42 +198,42 @@ void frl(int (&&e) [4])
 // CHECK-NEXT:    [[__END1:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store ptr [[E:%.*]], ptr [[E_ADDR]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    store ptr [[TMP1]], ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP3]], i64 0, i64 0
-// CHECK-NEXT:    store ptr [[ARRAYDECAY]], ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__END1]]) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP5]], i64 0, i64 0
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR3]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR3]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP1]], i64 0, i64 0
+// CHECK-NEXT:    store ptr [[ARRAYDECAY]], ptr [[__BEGIN1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__END1]]) #[[ATTR3]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP2]], i64 0, i64 0
 // CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, ptr [[ARRAYDECAY1]], i64 4
-// CHECK-NEXT:    store ptr [[ADD_PTR]], ptr [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    store ptr [[ADD_PTR]], ptr [[__END1]], align 8, !tbaa [[TBAA16]]
 // CHECK-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK:       for.cond:
-// CHECK-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__END1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[CMP:%.*]] = icmp ne ptr [[TMP6]], [[TMP7]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[__END1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne ptr [[TMP3]], [[TMP4]]
 // CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
 // CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 // CHECK:       for.cond.cleanup:
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__END1]]) #[[ATTR4]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR4]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR4]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__END1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_END:%.*]]
 // CHECK:       for.body:
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    store i32 [[TMP13]], ptr [[I]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i32 [[TMP6]], ptr [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK:       for.inc:
-// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i32 1
-// CHECK-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
+// CHECK-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 1
+// CHECK-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN1]], align 8, !tbaa [[TBAA16]]
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK:       for.end:
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp b/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
index 7b37480856ca2..4c2d40e21bef0 100644
--- a/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
+++ b/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
@@ -36,14 +36,14 @@ void test() {
 // CHECK1-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8, !tbaa [[TBAA10:![0-9]+]]
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_kernel_environment, ptr [[DYN_PTR]])
 // CHECK1-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 // CHECK1-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
 // CHECK1:       user_code.entry:
 // CHECK1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1:[0-9]+]])
 // CHECK1-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA15:![0-9]+]]
 // CHECK1-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4:[0-9]+]]
 // CHECK1-NEXT:    call void @__kmpc_target_deinit()
 // CHECK1-NEXT:    ret void
@@ -66,78 +66,78 @@ void test() {
 // CHECK1-NEXT:    [[REF_TMP:%.*]] = alloca float, align 4
 // CHECK1-NEXT:    [[REF_TMP2:%.*]] = alloca float, align 4
 // CHECK1-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [3 x ptr], align 8
-// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA17:![0-9]+]]
+// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA17]]
 // CHECK1-NEXT:    [[ISTART:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[IEND:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[PARTIAL_SUM:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 8)
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 99, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 99, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[IB]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_distribute_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP2]], 99
 // CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
 // CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 99, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP1:%.*]] = icmp sle i32 [[TMP5]], [[TMP6]]
 // CHECK1-NEXT:    br i1 [[CMP1]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
 // CHECK1:       omp.inner.for.cond.cleanup:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 1
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[IB]], align 4
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP]], align 4
+// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP]], align 4, !tbaa [[TBAA19:![0-9]+]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP2]]) #[[ATTR4]]
-// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP2]], align 4
-// CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP2]]) #[[ATTR12:[0-9]+]]
+// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP2]], align 4, !tbaa [[TBAA19]]
+// CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP2]]) #[[ATTR11:[0-9]+]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP2]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IB]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL3:%.*]] = mul nsw i32 [[TMP8]], 4
-// CHECK1-NEXT:    store i32 [[MUL3]], ptr [[ISTART]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[IB]], align 4
+// CHECK1-NEXT:    store i32 [[MUL3]], ptr [[ISTART]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[ADD4]], 4
-// CHECK1-NEXT:    store i32 [[MUL5]], ptr [[IEND]], align 4
+// CHECK1-NEXT:    store i32 [[MUL5]], ptr [[IEND]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[TMP10]], align 8
+// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[TMP10]], align 8, !tbaa [[TBAA21:![0-9]+]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 1
-// CHECK1-NEXT:    store ptr [[IEND]], ptr [[TMP11]], align 8
+// CHECK1-NEXT:    store ptr [[IEND]], ptr [[TMP11]], align 8, !tbaa [[TBAA21]]
 // CHECK1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 2
-// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8
+// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8, !tbaa [[TBAA21]]
 // CHECK1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined_omp_outlined, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined_omp_outlined_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 3)
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP13]], 1
-// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -161,13 +161,13 @@ void test() {
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__RE_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23:![0-9]+]]
+// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA24:![0-9]+]]
+// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA24]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8
-// CHECK1-NEXT:    call void @_ZNSt7complexIfEC2ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[THIS1]], ptr nonnull align 4 dereferenceable(4) [[TMP0]], ptr nonnull align 4 dereferenceable(4) [[TMP1]]) #[[ATTR12]]
+// CHECK1-NEXT:    call void @_ZNSt7complexIfEC2ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[THIS1]], ptr nonnull align 4 dereferenceable(4) [[TMP0]], ptr nonnull align 4 dereferenceable(4) [[TMP1]]) #[[ATTR11]]
 // CHECK1-NEXT:    ret void
 //
 //
@@ -197,79 +197,79 @@ void test() {
 // CHECK1-NEXT:    [[REF_TMP15:%.*]] = alloca float, align 4
 // CHECK1-NEXT:    [[REF_TMP16:%.*]] = alloca float, align 4
 // CHECK1-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8
-// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[IEND]], ptr [[IEND_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[IEND]], ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4
-// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4
-// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
 // CHECK1-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
 // CHECK1-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], 1
 // CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], 1
 // CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[I]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
 // CHECK1-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP]], align 4
+// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP]], align 4, !tbaa [[TBAA19]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP6]]) #[[ATTR4]]
-// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP6]], align 4
-// CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP6]]) #[[ATTR12]]
+// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP6]], align 4, !tbaa [[TBAA19]]
+// CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP6]]) #[[ATTR11]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP6]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB3:[0-9]+]], i32 [[TMP12]], i32 33, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_COND:%.*]]
 // CHECK1:       omp.dispatch.cond:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP8:%.*]] = icmp ugt i32 [[TMP13]], [[TMP14]]
 // CHECK1-NEXT:    br i1 [[CMP8]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
 // CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP15]], [[COND_TRUE]] ], [ [[TMP16]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP19]], 1
 // CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[TMP18]], [[ADD9]]
 // CHECK1-NEXT:    br i1 [[CMP10]], label [[OMP_DISPATCH_BODY:%.*]], label [[OMP_DISPATCH_CLEANUP:%.*]]
@@ -278,30 +278,30 @@ void test() {
 // CHECK1:       omp.dispatch.body:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP21]], 1
 // CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP20]], [[ADD11]]
 // CHECK1-NEXT:    br i1 [[CMP12]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
 // CHECK1:       omp.inner.for.cond.cleanup:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP23]], 1
 // CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP22]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I7]], align 4
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP14]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP15]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I7]], align 4
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP24]] to float
-// CHECK1-NEXT:    store float [[CONV]], ptr [[REF_TMP15]], align 4
+// CHECK1-NEXT:    store float [[CONV]], ptr [[REF_TMP15]], align 4, !tbaa [[TBAA19]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP16]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I7]], align 4
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV17:%.*]] = sitofp i32 [[TMP25]] to float
-// CHECK1-NEXT:    store float [[CONV17]], ptr [[REF_TMP16]], align 4
-// CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[REF_TMP14]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP15]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP16]]) #[[ATTR12]]
-// CHECK1-NEXT:    [[CALL:%.*]] = call nonnull align 4 dereferenceable(8) ptr @_ZNSt7complexIfEpLIfEERS0_RKS_IT_E(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]], ptr nonnull align 4 dereferenceable(8) [[REF_TMP14]]) #[[ATTR12]]
+// CHECK1-NEXT:    store float [[CONV17]], ptr [[REF_TMP16]], align 4, !tbaa [[TBAA19]]
+// CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[REF_TMP14]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP15]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP16]]) #[[ATTR11]]
+// CHECK1-NEXT:    [[CALL:%.*]] = call nonnull align 4 dereferenceable(8) ptr @_ZNSt7complexIfEpLIfEERS0_RKS_IT_E(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]], ptr nonnull align 4 dereferenceable(8) [[REF_TMP14]]) #[[ATTR11]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP16]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP15]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP14]]) #[[ATTR4]]
@@ -309,25 +309,25 @@ void test() {
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP26]], 1
-// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_INC:%.*]]
 // CHECK1:       omp.dispatch.inc:
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[TMP27]], [[TMP28]]
-// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_COND]]
 // CHECK1:       omp.dispatch.end:
 // CHECK1-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB3]], i32 [[TMP32]])
 // CHECK1-NEXT:    [[TMP33:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOMP_REDUCTION_RED_LIST]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[PARTIAL_SUM5]], ptr [[TMP33]], align 8
@@ -335,7 +335,7 @@ void test() {
 // CHECK1-NEXT:    [[TMP35:%.*]] = icmp eq i32 [[TMP34]], 1
 // CHECK1-NEXT:    br i1 [[TMP35]], label [[DOTOMP_REDUCTION_THEN:%.*]], label [[DOTOMP_REDUCTION_DONE:%.*]]
 // CHECK1:       .omp.reduction.then:
-// CHECK1-NEXT:    [[CALL21:%.*]] = call nonnull align 4 dereferenceable(8) ptr @_ZNSt7complexIfEpLIfEERS0_RKS_IT_E(ptr nonnull align 4 dereferenceable(8) [[TMP2]], ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]]) #[[ATTR12]]
+// CHECK1-NEXT:    [[CALL21:%.*]] = call nonnull align 4 dereferenceable(8) ptr @_ZNSt7complexIfEpLIfEERS0_RKS_IT_E(ptr nonnull align 4 dereferenceable(8) [[TMP2]], ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]]) #[[ATTR11]]
 // CHECK1-NEXT:    br label [[DOTOMP_REDUCTION_DONE]]
 // CHECK1:       .omp.reduction.done:
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
@@ -354,30 +354,30 @@ void test() {
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_ZNSt7complexIfEpLIfEERS0_RKS_IT_E
-// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(8) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(8) [[__C:%.*]]) #[[ATTR6:[0-9]+]] comdat align 2 {
+// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(8) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(8) [[__C:%.*]]) #[[ATTR5]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__C_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[__C]], ptr [[__C_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store ptr [[__C]], ptr [[__C_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8
-// CHECK1-NEXT:    [[CALL:%.*]] = call float @_ZNKSt7complexIfE4realEv(ptr nonnull align 4 dereferenceable(8) [[TMP0]]) #[[ATTR12]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    [[CALL:%.*]] = call float @_ZNKSt7complexIfE4realEv(ptr nonnull align 4 dereferenceable(8) [[TMP0]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[__RE_]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[__RE_]], align 4, !tbaa [[TBAA26:![0-9]+]]
 // CHECK1-NEXT:    [[ADD:%.*]] = fadd float [[TMP1]], [[CALL]]
-// CHECK1-NEXT:    store float [[ADD]], ptr [[__RE_]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8
-// CHECK1-NEXT:    [[CALL2:%.*]] = call float @_ZNKSt7complexIfE4imagEv(ptr nonnull align 4 dereferenceable(8) [[TMP2]]) #[[ATTR12]]
+// CHECK1-NEXT:    store float [[ADD]], ptr [[__RE_]], align 4, !tbaa [[TBAA26]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    [[CALL2:%.*]] = call float @_ZNKSt7complexIfE4imagEv(ptr nonnull align 4 dereferenceable(8) [[TMP2]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP3:%.*]] = load float, ptr [[__IM_]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load float, ptr [[__IM_]], align 4, !tbaa [[TBAA28:![0-9]+]]
 // CHECK1-NEXT:    [[ADD3:%.*]] = fadd float [[TMP3]], [[CALL2]]
-// CHECK1-NEXT:    store float [[ADD3]], ptr [[__IM_]], align 4
+// CHECK1-NEXT:    store float [[ADD3]], ptr [[__IM_]], align 4, !tbaa [[TBAA28]]
 // CHECK1-NEXT:    ret ptr [[THIS1]]
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_omp_reduction_shuffle_and_reduce_func
-// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i16 noundef signext [[TMP1:%.*]], i16 noundef signext [[TMP2:%.*]], i16 noundef signext [[TMP3:%.*]]) #[[ATTR7:[0-9]+]] {
+// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i16 noundef signext [[TMP1:%.*]], i16 noundef signext [[TMP2:%.*]], i16 noundef signext [[TMP3:%.*]]) #[[ATTR1]] {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTADDR1:%.*]] = alloca i16, align 2
@@ -442,59 +442,59 @@ void test() {
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_omp_reduction_inter_warp_copy_func
-// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i32 noundef [[TMP1:%.*]]) #[[ATTR7]] {
+// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i32 noundef [[TMP1:%.*]]) #[[ATTR1]] {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCNT_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTADDR1]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
 // CHECK1-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+// CHECK1-NEXT:    [[NVPTX_LANE_ID:%.*]] = and i32 [[TMP3]], 31
 // CHECK1-NEXT:    [[TMP4:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
-// CHECK1-NEXT:    [[NVPTX_LANE_ID:%.*]] = and i32 [[TMP4]], 31
-// CHECK1-NEXT:    [[TMP5:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
-// CHECK1-NEXT:    [[NVPTX_WARP_ID:%.*]] = ashr i32 [[TMP5]], 5
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[DOTADDR]], align 8
+// CHECK1-NEXT:    [[NVPTX_WARP_ID:%.*]] = ashr i32 [[TMP4]], 5
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[DOTADDR]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTCNT_ADDR]], align 4
 // CHECK1-NEXT:    br label [[PRECOND:%.*]]
 // CHECK1:       precond:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCNT_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = icmp ult i32 [[TMP7]], 2
-// CHECK1-NEXT:    br i1 [[TMP8]], label [[BODY:%.*]], label [[EXIT:%.*]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCNT_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 2
+// CHECK1-NEXT:    br i1 [[TMP7]], label [[BODY:%.*]], label [[EXIT:%.*]]
 // CHECK1:       body:
-// CHECK1-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
-// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4:[0-9]+]], i32 [[TMP2]])
+// CHECK1-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
+// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4:[0-9]+]], i32 [[OMP_GLOBAL_THREAD_NUM]])
 // CHECK1-NEXT:    [[WARP_MASTER:%.*]] = icmp eq i32 [[NVPTX_LANE_ID]], 0
 // CHECK1-NEXT:    br i1 [[WARP_MASTER]], label [[THEN:%.*]], label [[ELSE:%.*]]
 // CHECK1:       then:
-// CHECK1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP6]], i64 0, i64 0
-// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP9]], align 8
-// CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 [[TMP7]]
-// CHECK1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[NVPTX_WARP_ID]]
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4
-// CHECK1-NEXT:    store volatile i32 [[TMP13]], ptr addrspace(3) [[TMP12]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP5]], i64 0, i64 0
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP9]], i32 [[TMP6]]
+// CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[NVPTX_WARP_ID]]
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4
+// CHECK1-NEXT:    store volatile i32 [[TMP12]], ptr addrspace(3) [[TMP11]], align 4
 // CHECK1-NEXT:    br label [[IFCONT:%.*]]
 // CHECK1:       else:
 // CHECK1-NEXT:    br label [[IFCONT]]
 // CHECK1:       ifcont:
-// CHECK1-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
-// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4]], i32 [[TMP2]])
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTADDR1]], align 4
-// CHECK1-NEXT:    [[IS_ACTIVE_THREAD:%.*]] = icmp ult i32 [[TMP3]], [[TMP14]]
-// CHECK1-NEXT:    br i1 [[IS_ACTIVE_THREAD]], label [[THEN2:%.*]], label [[ELSE3:%.*]]
+// CHECK1-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
+// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTADDR1]], align 4
+// CHECK1-NEXT:    [[IS_ACTIVE_THREAD:%.*]] = icmp ult i32 [[TMP2]], [[TMP13]]
+// CHECK1-NEXT:    br i1 [[IS_ACTIVE_THREAD]], label [[THEN3:%.*]], label [[ELSE4:%.*]]
 // CHECK1:       then3:
-// CHECK1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[TMP3]]
-// CHECK1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP6]], i64 0, i64 0
-// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[TMP16]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = getelementptr i32, ptr [[TMP17]], i32 [[TMP7]]
-// CHECK1-NEXT:    [[TMP19:%.*]] = load volatile i32, ptr addrspace(3) [[TMP15]], align 4
-// CHECK1-NEXT:    store i32 [[TMP19]], ptr [[TMP18]], align 4
-// CHECK1-NEXT:    br label [[IFCONT4:%.*]]
+// CHECK1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[TMP2]]
+// CHECK1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP5]], i64 0, i64 0
+// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[TMP16]], i32 [[TMP6]]
+// CHECK1-NEXT:    [[TMP18:%.*]] = load volatile i32, ptr addrspace(3) [[TMP14]], align 4
+// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[TMP17]], align 4
+// CHECK1-NEXT:    br label [[IFCONT5:%.*]]
 // CHECK1:       else4:
-// CHECK1-NEXT:    br label [[IFCONT4]]
+// CHECK1-NEXT:    br label [[IFCONT5]]
 // CHECK1:       ifcont5:
-// CHECK1-NEXT:    [[TMP20:%.*]] = add nsw i32 [[TMP7]], 1
-// CHECK1-NEXT:    store i32 [[TMP20]], ptr [[DOTCNT_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = add nsw i32 [[TMP6]], 1
+// CHECK1-NEXT:    store i32 [[TMP19]], ptr [[DOTCNT_ADDR]], align 4
 // CHECK1-NEXT:    br label [[PRECOND]]
 // CHECK1:       exit:
 // CHECK1-NEXT:    ret void
@@ -507,17 +507,17 @@ void test() {
 // CHECK1-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store i16 [[TMP0]], ptr [[DOTADDR]], align 2
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTADDR1]], align 4
+// CHECK1-NEXT:    store i16 [[TMP0]], ptr [[DOTADDR]], align 2, !tbaa [[TBAA29:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTADDR1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
 // CHECK1-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 // CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 0
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 1
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 2
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined_omp_outlined(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP4]], ptr [[TMP6]], ptr [[TMP8]]) #[[ATTR4]]
 // CHECK1-NEXT:    ret void
 //
@@ -528,14 +528,14 @@ void test() {
 // CHECK1-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8, !tbaa [[TBAA10]]
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_kernel_environment, ptr [[DYN_PTR]])
 // CHECK1-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 // CHECK1-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
 // CHECK1:       user_code.entry:
 // CHECK1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 // CHECK1-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @__kmpc_target_deinit()
 // CHECK1-NEXT:    ret void
@@ -558,78 +558,78 @@ void test() {
 // CHECK1-NEXT:    [[REF_TMP:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[REF_TMP2:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [3 x ptr], align 8
-// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA17]]
 // CHECK1-NEXT:    [[ISTART:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[IEND:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[PARTIAL_SUM:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 16)
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 99, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 99, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[IB]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_distribute_static_init_4(ptr @[[GLOB2]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP2]], 99
 // CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
 // CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 99, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP1:%.*]] = icmp sle i32 [[TMP5]], [[TMP6]]
 // CHECK1-NEXT:    br i1 [[CMP1]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
 // CHECK1:       omp.inner.for.cond.cleanup:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 1
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[IB]], align 4
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP]], align 8
+// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP]], align 8, !tbaa [[TBAA31:![0-9]+]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP2]]) #[[ATTR4]]
-// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP2]], align 8
-// CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP2]]) #[[ATTR12]]
+// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP2]], align 8, !tbaa [[TBAA31]]
+// CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP2]]) #[[ATTR11]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP2]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IB]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL3:%.*]] = mul nsw i32 [[TMP8]], 4
-// CHECK1-NEXT:    store i32 [[MUL3]], ptr [[ISTART]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[IB]], align 4
+// CHECK1-NEXT:    store i32 [[MUL3]], ptr [[ISTART]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[ADD4]], 4
-// CHECK1-NEXT:    store i32 [[MUL5]], ptr [[IEND]], align 4
+// CHECK1-NEXT:    store i32 [[MUL5]], ptr [[IEND]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[TMP10]], align 8
+// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[TMP10]], align 8, !tbaa [[TBAA21]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 1
-// CHECK1-NEXT:    store ptr [[IEND]], ptr [[TMP11]], align 8
+// CHECK1-NEXT:    store ptr [[IEND]], ptr [[TMP11]], align 8, !tbaa [[TBAA21]]
 // CHECK1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 2
-// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8
+// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8, !tbaa [[TBAA21]]
 // CHECK1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined_omp_outlined, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined_omp_outlined_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 3)
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP13]], 1
-// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -653,13 +653,13 @@ void test() {
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__RE_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA33:![0-9]+]]
+// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA33]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8
-// CHECK1-NEXT:    call void @_ZNSt7complexIdEC2ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[THIS1]], ptr nonnull align 8 dereferenceable(8) [[TMP0]], ptr nonnull align 8 dereferenceable(8) [[TMP1]]) #[[ATTR12]]
+// CHECK1-NEXT:    call void @_ZNSt7complexIdEC2ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[THIS1]], ptr nonnull align 8 dereferenceable(8) [[TMP0]], ptr nonnull align 8 dereferenceable(8) [[TMP1]]) #[[ATTR11]]
 // CHECK1-NEXT:    ret void
 //
 //
@@ -689,79 +689,79 @@ void test() {
 // CHECK1-NEXT:    [[REF_TMP15:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[REF_TMP16:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8
-// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[IEND]], ptr [[IEND_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[IEND]], ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4
-// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4
-// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
 // CHECK1-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
 // CHECK1-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], 1
 // CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], 1
 // CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[I]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
 // CHECK1-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP]], align 8
+// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP]], align 8, !tbaa [[TBAA31]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP6]]) #[[ATTR4]]
-// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP6]], align 8
-// CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP6]]) #[[ATTR12]]
+// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP6]], align 8, !tbaa [[TBAA31]]
+// CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP6]]) #[[ATTR11]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP6]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB3]], i32 [[TMP12]], i32 33, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_COND:%.*]]
 // CHECK1:       omp.dispatch.cond:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP8:%.*]] = icmp ugt i32 [[TMP13]], [[TMP14]]
 // CHECK1-NEXT:    br i1 [[CMP8]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
 // CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP15]], [[COND_TRUE]] ], [ [[TMP16]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP19]], 1
 // CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[TMP18]], [[ADD9]]
 // CHECK1-NEXT:    br i1 [[CMP10]], label [[OMP_DISPATCH_BODY:%.*]], label [[OMP_DISPATCH_CLEANUP:%.*]]
@@ -770,30 +770,30 @@ void test() {
 // CHECK1:       omp.dispatch.body:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP21]], 1
 // CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP20]], [[ADD11]]
 // CHECK1-NEXT:    br i1 [[CMP12]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
 // CHECK1:       omp.inner.for.cond.cleanup:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP23]], 1
 // CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP22]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I7]], align 4
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[REF_TMP14]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP15]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I7]], align 4
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP24]] to double
-// CHECK1-NEXT:    store double [[CONV]], ptr [[REF_TMP15]], align 8
+// CHECK1-NEXT:    store double [[CONV]], ptr [[REF_TMP15]], align 8, !tbaa [[TBAA31]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP16]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I7]], align 4
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV17:%.*]] = sitofp i32 [[TMP25]] to double
-// CHECK1-NEXT:    store double [[CONV17]], ptr [[REF_TMP16]], align 8
-// CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[REF_TMP14]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP15]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP16]]) #[[ATTR12]]
-// CHECK1-NEXT:    [[CALL:%.*]] = call nonnull align 8 dereferenceable(16) ptr @_ZNSt7complexIdEpLIdEERS0_RKS_IT_E(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]], ptr nonnull align 8 dereferenceable(16) [[REF_TMP14]]) #[[ATTR12]]
+// CHECK1-NEXT:    store double [[CONV17]], ptr [[REF_TMP16]], align 8, !tbaa [[TBAA31]]
+// CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[REF_TMP14]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP15]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP16]]) #[[ATTR11]]
+// CHECK1-NEXT:    [[CALL:%.*]] = call nonnull align 8 dereferenceable(16) ptr @_ZNSt7complexIdEpLIdEERS0_RKS_IT_E(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]], ptr nonnull align 8 dereferenceable(16) [[REF_TMP14]]) #[[ATTR11]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP16]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP15]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[REF_TMP14]]) #[[ATTR4]]
@@ -801,25 +801,25 @@ void test() {
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP26]], 1
-// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_INC:%.*]]
 // CHECK1:       omp.dispatch.inc:
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[TMP27]], [[TMP28]]
-// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_COND]]
 // CHECK1:       omp.dispatch.end:
 // CHECK1-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB3]], i32 [[TMP32]])
 // CHECK1-NEXT:    [[TMP33:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOMP_REDUCTION_RED_LIST]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[PARTIAL_SUM5]], ptr [[TMP33]], align 8
@@ -827,7 +827,7 @@ void test() {
 // CHECK1-NEXT:    [[TMP35:%.*]] = icmp eq i32 [[TMP34]], 1
 // CHECK1-NEXT:    br i1 [[TMP35]], label [[DOTOMP_REDUCTION_THEN:%.*]], label [[DOTOMP_REDUCTION_DONE:%.*]]
 // CHECK1:       .omp.reduction.then:
-// CHECK1-NEXT:    [[CALL21:%.*]] = call nonnull align 8 dereferenceable(16) ptr @_ZNSt7complexIdEpLIdEERS0_RKS_IT_E(ptr nonnull align 8 dereferenceable(16) [[TMP2]], ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]]) #[[ATTR12]]
+// CHECK1-NEXT:    [[CALL21:%.*]] = call nonnull align 8 dereferenceable(16) ptr @_ZNSt7complexIdEpLIdEERS0_RKS_IT_E(ptr nonnull align 8 dereferenceable(16) [[TMP2]], ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]]) #[[ATTR11]]
 // CHECK1-NEXT:    br label [[DOTOMP_REDUCTION_DONE]]
 // CHECK1:       .omp.reduction.done:
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
@@ -846,30 +846,30 @@ void test() {
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_ZNSt7complexIdEpLIdEERS0_RKS_IT_E
-// CHECK1-SAME: (ptr nonnull align 8 dereferenceable(16) [[THIS:%.*]], ptr nonnull align 8 dereferenceable(16) [[__C:%.*]]) #[[ATTR6]] comdat align 2 {
+// CHECK1-SAME: (ptr nonnull align 8 dereferenceable(16) [[THIS:%.*]], ptr nonnull align 8 dereferenceable(16) [[__C:%.*]]) #[[ATTR5]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__C_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[__C]], ptr [[__C_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store ptr [[__C]], ptr [[__C_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8
-// CHECK1-NEXT:    [[CALL:%.*]] = call double @_ZNKSt7complexIdE4realEv(ptr nonnull align 8 dereferenceable(16) [[TMP0]]) #[[ATTR12]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    [[CALL:%.*]] = call double @_ZNKSt7complexIdE4realEv(ptr nonnull align 8 dereferenceable(16) [[TMP0]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP1:%.*]] = load double, ptr [[__RE_]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load double, ptr [[__RE_]], align 8, !tbaa [[TBAA35:![0-9]+]]
 // CHECK1-NEXT:    [[ADD:%.*]] = fadd double [[TMP1]], [[CALL]]
-// CHECK1-NEXT:    store double [[ADD]], ptr [[__RE_]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8
-// CHECK1-NEXT:    [[CALL2:%.*]] = call double @_ZNKSt7complexIdE4imagEv(ptr nonnull align 8 dereferenceable(16) [[TMP2]]) #[[ATTR12]]
+// CHECK1-NEXT:    store double [[ADD]], ptr [[__RE_]], align 8, !tbaa [[TBAA35]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    [[CALL2:%.*]] = call double @_ZNKSt7complexIdE4imagEv(ptr nonnull align 8 dereferenceable(16) [[TMP2]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP3:%.*]] = load double, ptr [[__IM_]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load double, ptr [[__IM_]], align 8, !tbaa [[TBAA37:![0-9]+]]
 // CHECK1-NEXT:    [[ADD3:%.*]] = fadd double [[TMP3]], [[CALL2]]
-// CHECK1-NEXT:    store double [[ADD3]], ptr [[__IM_]], align 8
+// CHECK1-NEXT:    store double [[ADD3]], ptr [[__IM_]], align 8, !tbaa [[TBAA37]]
 // CHECK1-NEXT:    ret ptr [[THIS1]]
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_omp_reduction_shuffle_and_reduce_func1
-// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i16 noundef signext [[TMP1:%.*]], i16 noundef signext [[TMP2:%.*]], i16 noundef signext [[TMP3:%.*]]) #[[ATTR7]] {
+// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i16 noundef signext [[TMP1:%.*]], i16 noundef signext [[TMP2:%.*]], i16 noundef signext [[TMP3:%.*]]) #[[ATTR1]] {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTADDR1:%.*]] = alloca i16, align 2
@@ -947,59 +947,59 @@ void test() {
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_omp_reduction_inter_warp_copy_func2
-// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i32 noundef [[TMP1:%.*]]) #[[ATTR7]] {
+// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i32 noundef [[TMP1:%.*]]) #[[ATTR1]] {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCNT_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTADDR1]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
 // CHECK1-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+// CHECK1-NEXT:    [[NVPTX_LANE_ID:%.*]] = and i32 [[TMP3]], 31
 // CHECK1-NEXT:    [[TMP4:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
-// CHECK1-NEXT:    [[NVPTX_LANE_ID:%.*]] = and i32 [[TMP4]], 31
-// CHECK1-NEXT:    [[TMP5:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
-// CHECK1-NEXT:    [[NVPTX_WARP_ID:%.*]] = ashr i32 [[TMP5]], 5
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[DOTADDR]], align 8
+// CHECK1-NEXT:    [[NVPTX_WARP_ID:%.*]] = ashr i32 [[TMP4]], 5
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[DOTADDR]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTCNT_ADDR]], align 4
 // CHECK1-NEXT:    br label [[PRECOND:%.*]]
 // CHECK1:       precond:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCNT_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = icmp ult i32 [[TMP7]], 4
-// CHECK1-NEXT:    br i1 [[TMP8]], label [[BODY:%.*]], label [[EXIT:%.*]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCNT_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 4
+// CHECK1-NEXT:    br i1 [[TMP7]], label [[BODY:%.*]], label [[EXIT:%.*]]
 // CHECK1:       body:
-// CHECK1-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
-// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4]], i32 [[TMP2]])
+// CHECK1-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
+// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4]], i32 [[OMP_GLOBAL_THREAD_NUM]])
 // CHECK1-NEXT:    [[WARP_MASTER:%.*]] = icmp eq i32 [[NVPTX_LANE_ID]], 0
 // CHECK1-NEXT:    br i1 [[WARP_MASTER]], label [[THEN:%.*]], label [[ELSE:%.*]]
 // CHECK1:       then:
-// CHECK1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP6]], i64 0, i64 0
-// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP9]], align 8
-// CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 [[TMP7]]
-// CHECK1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[NVPTX_WARP_ID]]
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4
-// CHECK1-NEXT:    store volatile i32 [[TMP13]], ptr addrspace(3) [[TMP12]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP5]], i64 0, i64 0
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP9]], i32 [[TMP6]]
+// CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[NVPTX_WARP_ID]]
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4
+// CHECK1-NEXT:    store volatile i32 [[TMP12]], ptr addrspace(3) [[TMP11]], align 4
 // CHECK1-NEXT:    br label [[IFCONT:%.*]]
 // CHECK1:       else:
 // CHECK1-NEXT:    br label [[IFCONT]]
 // CHECK1:       ifcont:
-// CHECK1-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
-// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4]], i32 [[TMP2]])
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTADDR1]], align 4
-// CHECK1-NEXT:    [[IS_ACTIVE_THREAD:%.*]] = icmp ult i32 [[TMP3]], [[TMP14]]
-// CHECK1-NEXT:    br i1 [[IS_ACTIVE_THREAD]], label [[THEN2:%.*]], label [[ELSE3:%.*]]
+// CHECK1-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
+// CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTADDR1]], align 4
+// CHECK1-NEXT:    [[IS_ACTIVE_THREAD:%.*]] = icmp ult i32 [[TMP2]], [[TMP13]]
+// CHECK1-NEXT:    br i1 [[IS_ACTIVE_THREAD]], label [[THEN3:%.*]], label [[ELSE4:%.*]]
 // CHECK1:       then3:
-// CHECK1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[TMP3]]
-// CHECK1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP6]], i64 0, i64 0
-// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[TMP16]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = getelementptr i32, ptr [[TMP17]], i32 [[TMP7]]
-// CHECK1-NEXT:    [[TMP19:%.*]] = load volatile i32, ptr addrspace(3) [[TMP15]], align 4
-// CHECK1-NEXT:    store i32 [[TMP19]], ptr [[TMP18]], align 4
-// CHECK1-NEXT:    br label [[IFCONT4:%.*]]
+// CHECK1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[TMP2]]
+// CHECK1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP5]], i64 0, i64 0
+// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[TMP16]], i32 [[TMP6]]
+// CHECK1-NEXT:    [[TMP18:%.*]] = load volatile i32, ptr addrspace(3) [[TMP14]], align 4
+// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[TMP17]], align 4
+// CHECK1-NEXT:    br label [[IFCONT5:%.*]]
 // CHECK1:       else4:
-// CHECK1-NEXT:    br label [[IFCONT4]]
+// CHECK1-NEXT:    br label [[IFCONT5]]
 // CHECK1:       ifcont5:
-// CHECK1-NEXT:    [[TMP20:%.*]] = add nsw i32 [[TMP7]], 1
-// CHECK1-NEXT:    store i32 [[TMP20]], ptr [[DOTCNT_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = add nsw i32 [[TMP6]], 1
+// CHECK1-NEXT:    store i32 [[TMP19]], ptr [[DOTCNT_ADDR]], align 4
 // CHECK1-NEXT:    br label [[PRECOND]]
 // CHECK1:       exit:
 // CHECK1-NEXT:    ret void
@@ -1012,17 +1012,17 @@ void test() {
 // CHECK1-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store i16 [[TMP0]], ptr [[DOTADDR]], align 2
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTADDR1]], align 4
+// CHECK1-NEXT:    store i16 [[TMP0]], ptr [[DOTADDR]], align 2, !tbaa [[TBAA29]]
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTADDR1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
 // CHECK1-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 // CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 0
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 1
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 2
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined_omp_outlined(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP4]], ptr [[TMP6]], ptr [[TMP8]]) #[[ATTR4]]
 // CHECK1-NEXT:    ret void
 //
@@ -1033,40 +1033,40 @@ void test() {
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__RE_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA24]]
+// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA24]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[TMP0]], align 4
-// CHECK1-NEXT:    store float [[TMP1]], ptr [[__RE_]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA24]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[TMP0]], align 4, !tbaa [[TBAA19]]
+// CHECK1-NEXT:    store float [[TMP1]], ptr [[__RE_]], align 4, !tbaa [[TBAA26]]
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4
-// CHECK1-NEXT:    store float [[TMP3]], ptr [[__IM_]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA24]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4, !tbaa [[TBAA19]]
+// CHECK1-NEXT:    store float [[TMP3]], ptr [[__IM_]], align 4, !tbaa [[TBAA28]]
 // CHECK1-NEXT:    ret void
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_ZNKSt7complexIfE4realEv
-// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(8) [[THIS:%.*]]) #[[ATTR6]] comdat align 2 {
+// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(8) [[THIS:%.*]]) #[[ATTR5]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load float, ptr [[__RE_]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load float, ptr [[__RE_]], align 4, !tbaa [[TBAA26]]
 // CHECK1-NEXT:    ret float [[TMP0]]
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_ZNKSt7complexIfE4imagEv
-// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(8) [[THIS:%.*]]) #[[ATTR6]] comdat align 2 {
+// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(8) [[THIS:%.*]]) #[[ATTR5]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP0:%.*]] = load float, ptr [[__IM_]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load float, ptr [[__IM_]], align 4, !tbaa [[TBAA28]]
 // CHECK1-NEXT:    ret float [[TMP0]]
 //
 //
@@ -1076,39 +1076,39 @@ void test() {
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__RE_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8
-// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA33]]
+// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA33]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load double, ptr [[TMP0]], align 8
-// CHECK1-NEXT:    store double [[TMP1]], ptr [[__RE_]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA33]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load double, ptr [[TMP0]], align 8, !tbaa [[TBAA31]]
+// CHECK1-NEXT:    store double [[TMP1]], ptr [[__RE_]], align 8, !tbaa [[TBAA35]]
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load double, ptr [[TMP2]], align 8
-// CHECK1-NEXT:    store double [[TMP3]], ptr [[__IM_]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA33]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load double, ptr [[TMP2]], align 8, !tbaa [[TBAA31]]
+// CHECK1-NEXT:    store double [[TMP3]], ptr [[__IM_]], align 8, !tbaa [[TBAA37]]
 // CHECK1-NEXT:    ret void
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_ZNKSt7complexIdE4realEv
-// CHECK1-SAME: (ptr nonnull align 8 dereferenceable(16) [[THIS:%.*]]) #[[ATTR6]] comdat align 2 {
+// CHECK1-SAME: (ptr nonnull align 8 dereferenceable(16) [[THIS:%.*]]) #[[ATTR5]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load double, ptr [[__RE_]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load double, ptr [[__RE_]], align 8, !tbaa [[TBAA35]]
 // CHECK1-NEXT:    ret double [[TMP0]]
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_ZNKSt7complexIdE4imagEv
-// CHECK1-SAME: (ptr nonnull align 8 dereferenceable(16) [[THIS:%.*]]) #[[ATTR6]] comdat align 2 {
+// CHECK1-SAME: (ptr nonnull align 8 dereferenceable(16) [[THIS:%.*]]) #[[ATTR5]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP0:%.*]] = load double, ptr [[__IM_]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load double, ptr [[__IM_]], align 8, !tbaa [[TBAA37]]
 // CHECK1-NEXT:    ret double [[TMP0]]
 //

From 176bf50cd244505e38f8838a55568060dd3913e8 Mon Sep 17 00:00:00 2001
From: Dan Liew <dan@su-root.co.uk>
Date: Fri, 19 Jul 2024 13:06:26 +0100
Subject: [PATCH 186/486] [clang][NFC] Move Bounds Safety Sema code to
 `SemaBoundsSafety.cpp` (#99330)

This patch adds a new `SemaBoundsSafety.cpp` source file and moves the
existing `CheckCountedByAttrOnField` function and related helper
functions and types from `SemaDeclAttr.cpp` into the new source file.
The `CheckCountedByAttrOnField` function is now a method on the Sema
class and now has doxygen comments.

The goal behind this refactor is to clearly separate the
`-fbounds-safety` Sema code from everything else.

Although `counted_by(_or_null)` and `sized_by(_or_null)` attributes have
a meaning outside of `-fbounds-safety` it seems reasonable to also have
the Sema logic live in `SemaBoundsSafety.cpp` since the intention is
that the attributes will have the same semantics (but not necessarily
the same enforcement).

As `-fbounds-safety` is upstreamed additional Sema checks will be added
to `SemaBoundsSafety.cpp`.

rdar://131777237
---
 clang/include/clang/Sema/Sema.h     |  38 ++++++
 clang/lib/Sema/CMakeLists.txt       |   1 +
 clang/lib/Sema/SemaBoundsSafety.cpp | 193 ++++++++++++++++++++++++++++
 clang/lib/Sema/SemaDeclAttr.cpp     | 177 +------------------------
 4 files changed, 233 insertions(+), 176 deletions(-)
 create mode 100644 clang/lib/Sema/SemaBoundsSafety.cpp

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 3cb1aa935fe46..d638d31e050dc 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -15051,6 +15051,44 @@ class Sema final : public SemaBase {
   void ProcessAPINotes(Decl *D);
 
   ///@}
+
+  //
+  //
+  // -------------------------------------------------------------------------
+  //
+  //
+
+  /// \name Bounds Safety
+  /// Implementations are in SemaBoundsSafety.cpp
+  ///@{
+public:
+  /// Check if applying the specified attribute variant from the "counted by"
+  /// family of attributes to FieldDecl \p FD is semantically valid. If
+  /// semantically invalid diagnostics will be emitted explaining the problems.
+  ///
+  /// \param FD The FieldDecl to apply the attribute to
+  /// \param E The count expression on the attribute
+  /// \param[out] Decls If the attribute is semantically valid \p Decls
+  ///             is populated with TypeCoupledDeclRefInfo objects, each
+  ///             describing Decls referred to in \p E.
+  /// \param CountInBytes If true the attribute is from the "sized_by" family of
+  ///                     attributes. If the false the attribute is from
+  ///                     "counted_by" family of attributes.
+  /// \param OrNull If true the attribute is from the "_or_null" suffixed family
+  ///               of attributes. If false the attribute does not have the
+  ///               suffix.
+  ///
+  /// Together \p CountInBytes and \p OrNull decide the attribute variant. E.g.
+  /// \p CountInBytes and \p OrNull both being true indicates the
+  /// `counted_by_or_null` attribute.
+  ///
+  /// \returns false iff semantically valid.
+  bool CheckCountedByAttrOnField(
+      FieldDecl *FD, Expr *E,
+      llvm::SmallVectorImpl<TypeCoupledDeclRefInfo> &Decls, bool CountInBytes,
+      bool OrNull);
+
+  ///@}
 };
 
 DeductionFailureInfo
diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index 5934c8c30daf9..2cee4f5ef6e99 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -36,6 +36,7 @@ add_clang_library(clangSema
   SemaAvailability.cpp
   SemaBPF.cpp
   SemaBase.cpp
+  SemaBoundsSafety.cpp
   SemaCXXScopeSpec.cpp
   SemaCast.cpp
   SemaChecking.cpp
diff --git a/clang/lib/Sema/SemaBoundsSafety.cpp b/clang/lib/Sema/SemaBoundsSafety.cpp
new file mode 100644
index 0000000000000..290c820938898
--- /dev/null
+++ b/clang/lib/Sema/SemaBoundsSafety.cpp
@@ -0,0 +1,193 @@
+//===-- SemaBoundsSafety.cpp - Bounds Safety specific routines-*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file declares semantic analysis functions specific to `-fbounds-safety`
+/// (Bounds Safety) and also its attributes when used without `-fbounds-safety`
+/// (e.g. `counted_by`)
+///
+//===----------------------------------------------------------------------===//
+#include "clang/Sema/Sema.h"
+
+namespace clang {
+
+static CountAttributedType::DynamicCountPointerKind
+getCountAttrKind(bool CountInBytes, bool OrNull) {
+  if (CountInBytes)
+    return OrNull ? CountAttributedType::SizedByOrNull
+                  : CountAttributedType::SizedBy;
+  return OrNull ? CountAttributedType::CountedByOrNull
+                : CountAttributedType::CountedBy;
+}
+
+static const RecordDecl *GetEnclosingNamedOrTopAnonRecord(const FieldDecl *FD) {
+  const auto *RD = FD->getParent();
+  // An unnamed struct is anonymous struct only if it's not instantiated.
+  // However, the struct may not be fully processed yet to determine
+  // whether it's anonymous or not. In that case, this function treats it as
+  // an anonymous struct and tries to find a named parent.
+  while (RD && (RD->isAnonymousStructOrUnion() ||
+                (!RD->isCompleteDefinition() && RD->getName().empty()))) {
+    const auto *Parent = dyn_cast<RecordDecl>(RD->getParent());
+    if (!Parent)
+      break;
+    RD = Parent;
+  }
+  return RD;
+}
+
+enum class CountedByInvalidPointeeTypeKind {
+  INCOMPLETE,
+  SIZELESS,
+  FUNCTION,
+  FLEXIBLE_ARRAY_MEMBER,
+  VALID,
+};
+
+bool Sema::CheckCountedByAttrOnField(
+    FieldDecl *FD, Expr *E,
+    llvm::SmallVectorImpl<TypeCoupledDeclRefInfo> &Decls, bool CountInBytes,
+    bool OrNull) {
+  // Check the context the attribute is used in
+
+  unsigned Kind = getCountAttrKind(CountInBytes, OrNull);
+
+  if (FD->getParent()->isUnion()) {
+    Diag(FD->getBeginLoc(), diag::err_count_attr_in_union)
+        << Kind << FD->getSourceRange();
+    return true;
+  }
+
+  const auto FieldTy = FD->getType();
+  if (FieldTy->isArrayType() && (CountInBytes || OrNull)) {
+    Diag(FD->getBeginLoc(),
+         diag::err_count_attr_not_on_ptr_or_flexible_array_member)
+        << Kind << FD->getLocation() << /* suggest counted_by */ 1;
+    return true;
+  }
+  if (!FieldTy->isArrayType() && !FieldTy->isPointerType()) {
+    Diag(FD->getBeginLoc(),
+         diag::err_count_attr_not_on_ptr_or_flexible_array_member)
+        << Kind << FD->getLocation() << /* do not suggest counted_by */ 0;
+    return true;
+  }
+
+  LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
+      LangOptions::StrictFlexArraysLevelKind::IncompleteOnly;
+  if (FieldTy->isArrayType() &&
+      !Decl::isFlexibleArrayMemberLike(getASTContext(), FD, FieldTy,
+                                       StrictFlexArraysLevel, true)) {
+    Diag(FD->getBeginLoc(),
+         diag::err_counted_by_attr_on_array_not_flexible_array_member)
+        << Kind << FD->getLocation();
+    return true;
+  }
+
+  CountedByInvalidPointeeTypeKind InvalidTypeKind =
+      CountedByInvalidPointeeTypeKind::VALID;
+  QualType PointeeTy;
+  int SelectPtrOrArr = 0;
+  if (FieldTy->isPointerType()) {
+    PointeeTy = FieldTy->getPointeeType();
+    SelectPtrOrArr = 0;
+  } else {
+    assert(FieldTy->isArrayType());
+    const ArrayType *AT = getASTContext().getAsArrayType(FieldTy);
+    PointeeTy = AT->getElementType();
+    SelectPtrOrArr = 1;
+  }
+  // Note: The `Decl::isFlexibleArrayMemberLike` check earlier on means
+  // only `PointeeTy->isStructureTypeWithFlexibleArrayMember()` is reachable
+  // when `FieldTy->isArrayType()`.
+  bool ShouldWarn = false;
+  if (PointeeTy->isIncompleteType() && !CountInBytes) {
+    InvalidTypeKind = CountedByInvalidPointeeTypeKind::INCOMPLETE;
+  } else if (PointeeTy->isSizelessType()) {
+    InvalidTypeKind = CountedByInvalidPointeeTypeKind::SIZELESS;
+  } else if (PointeeTy->isFunctionType()) {
+    InvalidTypeKind = CountedByInvalidPointeeTypeKind::FUNCTION;
+  } else if (PointeeTy->isStructureTypeWithFlexibleArrayMember()) {
+    if (FieldTy->isArrayType() && !getLangOpts().BoundsSafety) {
+      // This is a workaround for the Linux kernel that has already adopted
+      // `counted_by` on a FAM where the pointee is a struct with a FAM. This
+      // should be an error because computing the bounds of the array cannot be
+      // done correctly without manually traversing every struct object in the
+      // array at runtime. To allow the code to be built this error is
+      // downgraded to a warning.
+      ShouldWarn = true;
+    }
+    InvalidTypeKind = CountedByInvalidPointeeTypeKind::FLEXIBLE_ARRAY_MEMBER;
+  }
+
+  if (InvalidTypeKind != CountedByInvalidPointeeTypeKind::VALID) {
+    unsigned DiagID = ShouldWarn
+                          ? diag::warn_counted_by_attr_elt_type_unknown_size
+                          : diag::err_counted_by_attr_pointee_unknown_size;
+    Diag(FD->getBeginLoc(), DiagID)
+        << SelectPtrOrArr << PointeeTy << (int)InvalidTypeKind
+        << (ShouldWarn ? 1 : 0) << Kind << FD->getSourceRange();
+    return true;
+  }
+
+  // Check the expression
+
+  if (!E->getType()->isIntegerType() || E->getType()->isBooleanType()) {
+    Diag(E->getBeginLoc(), diag::err_count_attr_argument_not_integer)
+        << Kind << E->getSourceRange();
+    return true;
+  }
+
+  auto *DRE = dyn_cast<DeclRefExpr>(E);
+  if (!DRE) {
+    Diag(E->getBeginLoc(),
+         diag::err_count_attr_only_support_simple_decl_reference)
+        << Kind << E->getSourceRange();
+    return true;
+  }
+
+  auto *CountDecl = DRE->getDecl();
+  FieldDecl *CountFD = dyn_cast<FieldDecl>(CountDecl);
+  if (auto *IFD = dyn_cast<IndirectFieldDecl>(CountDecl)) {
+    CountFD = IFD->getAnonField();
+  }
+  if (!CountFD) {
+    Diag(E->getBeginLoc(), diag::err_count_attr_must_be_in_structure)
+        << CountDecl << Kind << E->getSourceRange();
+
+    Diag(CountDecl->getBeginLoc(),
+         diag::note_flexible_array_counted_by_attr_field)
+        << CountDecl << CountDecl->getSourceRange();
+    return true;
+  }
+
+  if (FD->getParent() != CountFD->getParent()) {
+    if (CountFD->getParent()->isUnion()) {
+      Diag(CountFD->getBeginLoc(), diag::err_count_attr_refer_to_union)
+          << Kind << CountFD->getSourceRange();
+      return true;
+    }
+    // Whether CountRD is an anonymous struct is not determined at this
+    // point. Thus, an additional diagnostic in case it's not anonymous struct
+    // is done later in `Parser::ParseStructDeclaration`.
+    auto *RD = GetEnclosingNamedOrTopAnonRecord(FD);
+    auto *CountRD = GetEnclosingNamedOrTopAnonRecord(CountFD);
+
+    if (RD != CountRD) {
+      Diag(E->getBeginLoc(), diag::err_count_attr_param_not_in_same_struct)
+          << CountFD << Kind << FieldTy->isArrayType() << E->getSourceRange();
+      Diag(CountFD->getBeginLoc(),
+           diag::note_flexible_array_counted_by_attr_field)
+          << CountFD << CountFD->getSourceRange();
+      return true;
+    }
+  }
+
+  Decls.push_back(TypeCoupledDeclRefInfo(CountFD, /*IsDref*/ false));
+  return false;
+}
+
+} // namespace clang
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index f2f3bfcd09035..50e7be9ea6020 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -5852,181 +5852,6 @@ static void handleZeroCallUsedRegsAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   D->addAttr(ZeroCallUsedRegsAttr::Create(S.Context, Kind, AL));
 }
 
-static const RecordDecl *GetEnclosingNamedOrTopAnonRecord(const FieldDecl *FD) {
-  const auto *RD = FD->getParent();
-  // An unnamed struct is anonymous struct only if it's not instantiated.
-  // However, the struct may not be fully processed yet to determine
-  // whether it's anonymous or not. In that case, this function treats it as
-  // an anonymous struct and tries to find a named parent.
-  while (RD && (RD->isAnonymousStructOrUnion() ||
-                (!RD->isCompleteDefinition() && RD->getName().empty()))) {
-    const auto *Parent = dyn_cast<RecordDecl>(RD->getParent());
-    if (!Parent)
-      break;
-    RD = Parent;
-  }
-  return RD;
-}
-
-static CountAttributedType::DynamicCountPointerKind
-getCountAttrKind(bool CountInBytes, bool OrNull) {
-  if (CountInBytes)
-    return OrNull ? CountAttributedType::SizedByOrNull
-                  : CountAttributedType::SizedBy;
-  return OrNull ? CountAttributedType::CountedByOrNull
-                : CountAttributedType::CountedBy;
-}
-
-enum class CountedByInvalidPointeeTypeKind {
-  INCOMPLETE,
-  SIZELESS,
-  FUNCTION,
-  FLEXIBLE_ARRAY_MEMBER,
-  VALID,
-};
-
-static bool
-CheckCountedByAttrOnField(Sema &S, FieldDecl *FD, Expr *E,
-                          llvm::SmallVectorImpl<TypeCoupledDeclRefInfo> &Decls,
-                          bool CountInBytes, bool OrNull) {
-  // Check the context the attribute is used in
-
-  unsigned Kind = getCountAttrKind(CountInBytes, OrNull);
-
-  if (FD->getParent()->isUnion()) {
-    S.Diag(FD->getBeginLoc(), diag::err_count_attr_in_union)
-        << Kind << FD->getSourceRange();
-    return true;
-  }
-
-  const auto FieldTy = FD->getType();
-  if (FieldTy->isArrayType() && (CountInBytes || OrNull)) {
-    S.Diag(FD->getBeginLoc(),
-           diag::err_count_attr_not_on_ptr_or_flexible_array_member)
-        << Kind << FD->getLocation() << /* suggest counted_by */ 1;
-    return true;
-  }
-  if (!FieldTy->isArrayType() && !FieldTy->isPointerType()) {
-    S.Diag(FD->getBeginLoc(),
-           diag::err_count_attr_not_on_ptr_or_flexible_array_member)
-        << Kind << FD->getLocation() << /* do not suggest counted_by */ 0;
-    return true;
-  }
-
-  LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
-      LangOptions::StrictFlexArraysLevelKind::IncompleteOnly;
-  if (FieldTy->isArrayType() &&
-      !Decl::isFlexibleArrayMemberLike(S.getASTContext(), FD, FieldTy,
-                                       StrictFlexArraysLevel, true)) {
-    S.Diag(FD->getBeginLoc(),
-           diag::err_counted_by_attr_on_array_not_flexible_array_member)
-        << Kind << FD->getLocation();
-    return true;
-  }
-
-  CountedByInvalidPointeeTypeKind InvalidTypeKind =
-      CountedByInvalidPointeeTypeKind::VALID;
-  QualType PointeeTy;
-  int SelectPtrOrArr = 0;
-  if (FieldTy->isPointerType()) {
-    PointeeTy = FieldTy->getPointeeType();
-    SelectPtrOrArr = 0;
-  } else {
-    assert(FieldTy->isArrayType());
-    const ArrayType *AT = S.getASTContext().getAsArrayType(FieldTy);
-    PointeeTy = AT->getElementType();
-    SelectPtrOrArr = 1;
-  }
-  // Note: The `Decl::isFlexibleArrayMemberLike` check earlier on means
-  // only `PointeeTy->isStructureTypeWithFlexibleArrayMember()` is reachable
-  // when `FieldTy->isArrayType()`.
-  bool ShouldWarn = false;
-  if (PointeeTy->isIncompleteType() && !CountInBytes) {
-    InvalidTypeKind = CountedByInvalidPointeeTypeKind::INCOMPLETE;
-  } else if (PointeeTy->isSizelessType()) {
-    InvalidTypeKind = CountedByInvalidPointeeTypeKind::SIZELESS;
-  } else if (PointeeTy->isFunctionType()) {
-    InvalidTypeKind = CountedByInvalidPointeeTypeKind::FUNCTION;
-  } else if (PointeeTy->isStructureTypeWithFlexibleArrayMember()) {
-    if (FieldTy->isArrayType() && !S.getLangOpts().BoundsSafety) {
-      // This is a workaround for the Linux kernel that has already adopted
-      // `counted_by` on a FAM where the pointee is a struct with a FAM. This
-      // should be an error because computing the bounds of the array cannot be
-      // done correctly without manually traversing every struct object in the
-      // array at runtime. To allow the code to be built this error is
-      // downgraded to a warning.
-      ShouldWarn = true;
-    }
-    InvalidTypeKind = CountedByInvalidPointeeTypeKind::FLEXIBLE_ARRAY_MEMBER;
-  }
-
-  if (InvalidTypeKind != CountedByInvalidPointeeTypeKind::VALID) {
-    unsigned DiagID = ShouldWarn
-                          ? diag::warn_counted_by_attr_elt_type_unknown_size
-                          : diag::err_counted_by_attr_pointee_unknown_size;
-    S.Diag(FD->getBeginLoc(), DiagID)
-        << SelectPtrOrArr << PointeeTy << (int)InvalidTypeKind
-        << (ShouldWarn ? 1 : 0) << Kind << FD->getSourceRange();
-    return true;
-  }
-
-  // Check the expression
-
-  if (!E->getType()->isIntegerType() || E->getType()->isBooleanType()) {
-    S.Diag(E->getBeginLoc(), diag::err_count_attr_argument_not_integer)
-        << Kind << E->getSourceRange();
-    return true;
-  }
-
-  auto *DRE = dyn_cast<DeclRefExpr>(E);
-  if (!DRE) {
-    S.Diag(E->getBeginLoc(),
-           diag::err_count_attr_only_support_simple_decl_reference)
-        << Kind << E->getSourceRange();
-    return true;
-  }
-
-  auto *CountDecl = DRE->getDecl();
-  FieldDecl *CountFD = dyn_cast<FieldDecl>(CountDecl);
-  if (auto *IFD = dyn_cast<IndirectFieldDecl>(CountDecl)) {
-    CountFD = IFD->getAnonField();
-  }
-  if (!CountFD) {
-    S.Diag(E->getBeginLoc(), diag::err_count_attr_must_be_in_structure)
-        << CountDecl << Kind << E->getSourceRange();
-
-    S.Diag(CountDecl->getBeginLoc(),
-           diag::note_flexible_array_counted_by_attr_field)
-        << CountDecl << CountDecl->getSourceRange();
-    return true;
-  }
-
-  if (FD->getParent() != CountFD->getParent()) {
-    if (CountFD->getParent()->isUnion()) {
-      S.Diag(CountFD->getBeginLoc(), diag::err_count_attr_refer_to_union)
-          << Kind << CountFD->getSourceRange();
-      return true;
-    }
-    // Whether CountRD is an anonymous struct is not determined at this
-    // point. Thus, an additional diagnostic in case it's not anonymous struct
-    // is done later in `Parser::ParseStructDeclaration`.
-    auto *RD = GetEnclosingNamedOrTopAnonRecord(FD);
-    auto *CountRD = GetEnclosingNamedOrTopAnonRecord(CountFD);
-
-    if (RD != CountRD) {
-      S.Diag(E->getBeginLoc(), diag::err_count_attr_param_not_in_same_struct)
-          << CountFD << Kind << FieldTy->isArrayType() << E->getSourceRange();
-      S.Diag(CountFD->getBeginLoc(),
-             diag::note_flexible_array_counted_by_attr_field)
-          << CountFD << CountFD->getSourceRange();
-      return true;
-    }
-  }
-
-  Decls.push_back(TypeCoupledDeclRefInfo(CountFD, /*IsDref*/ false));
-  return false;
-}
-
 static void handleCountedByAttrField(Sema &S, Decl *D, const ParsedAttr &AL) {
   auto *FD = dyn_cast<FieldDecl>(D);
   assert(FD);
@@ -6059,7 +5884,7 @@ static void handleCountedByAttrField(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
 
   llvm::SmallVector<TypeCoupledDeclRefInfo, 1> Decls;
-  if (CheckCountedByAttrOnField(S, FD, CountExpr, Decls, CountInBytes, OrNull))
+  if (S.CheckCountedByAttrOnField(FD, CountExpr, Decls, CountInBytes, OrNull))
     return;
 
   QualType CAT = S.BuildCountAttributedArrayOrPointerType(

From 7b60f2dcfbe11ddb62f7a6f2d374c1a3ff556eb0 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 19 Jul 2024 12:13:38 +0000
Subject: [PATCH 187/486] [gn build] Port 176bf50cd244

---
 llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
index ed903559cac84..83a10d5b18bc8 100644
--- a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
@@ -60,6 +60,7 @@ static_library("Sema") {
     "SemaAvailability.cpp",
     "SemaBPF.cpp",
     "SemaBase.cpp",
+    "SemaBoundsSafety.cpp",
     "SemaCUDA.cpp",
     "SemaCXXScopeSpec.cpp",
     "SemaCast.cpp",

From 7122b70cfc8e23a069410215c363da76d842bda4 Mon Sep 17 00:00:00 2001
From: Mariya Podchishchaeva <mariya.podchishchaeva@intel.com>
Date: Fri, 19 Jul 2024 14:24:05 +0200
Subject: [PATCH 188/486] [clang] Fix underlying type of EmbedExpr (#99050)

This patch makes remaining cases of #embed to emit int type since there
is an agreement to do that for C. C++ is being discussed, but in general
we don't want to produce different types for C and C++.
---
 clang/lib/AST/Expr.cpp                      |  2 +-
 clang/lib/Sema/SemaInit.cpp                 |  2 +-
 clang/test/Preprocessor/Inputs/big_char.txt |  1 +
 clang/test/Preprocessor/embed_codegen.cpp   |  8 ++++----
 clang/test/Preprocessor/embed_weird.cpp     | 11 +++++++++++
 5 files changed, 18 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/Preprocessor/Inputs/big_char.txt

diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 6af1b5683e08b..9d5b8167d0ee6 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -2376,7 +2376,7 @@ APValue SourceLocExpr::EvaluateInContext(const ASTContext &Ctx,
 EmbedExpr::EmbedExpr(const ASTContext &Ctx, SourceLocation Loc,
                      EmbedDataStorage *Data, unsigned Begin,
                      unsigned NumOfElements)
-    : Expr(EmbedExprClass, Ctx.UnsignedCharTy, VK_PRValue, OK_Ordinary),
+    : Expr(EmbedExprClass, Ctx.IntTy, VK_PRValue, OK_Ordinary),
       EmbedKeywordLoc(Loc), Ctx(&Ctx), Data(Data), Begin(Begin),
       NumOfElements(NumOfElements) {
   setDependence(ExprDependence::None);
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 17435afab03f4..dc2ba039afe7f 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -2016,7 +2016,7 @@ canInitializeArrayWithEmbedDataString(ArrayRef<Expr *> ExprList,
   if (InitType->isArrayType()) {
     const ArrayType *InitArrayType = InitType->getAsArrayTypeUnsafe();
     QualType InitElementTy = InitArrayType->getElementType();
-    QualType EmbedExprElementTy = EE->getType();
+    QualType EmbedExprElementTy = EE->getDataStringLiteral()->getType();
     const bool TypesMatch =
         Context.typesAreCompatible(InitElementTy, EmbedExprElementTy) ||
         (InitElementTy->isCharType() && EmbedExprElementTy->isCharType());
diff --git a/clang/test/Preprocessor/Inputs/big_char.txt b/clang/test/Preprocessor/Inputs/big_char.txt
new file mode 100644
index 0000000000000..ce542efaa5124
--- /dev/null
+++ b/clang/test/Preprocessor/Inputs/big_char.txt
@@ -0,0 +1 @@
+ÿ
\ No newline at end of file
diff --git a/clang/test/Preprocessor/embed_codegen.cpp b/clang/test/Preprocessor/embed_codegen.cpp
index 2cf14d8d6a15d..5baab9b59a9c1 100644
--- a/clang/test/Preprocessor/embed_codegen.cpp
+++ b/clang/test/Preprocessor/embed_codegen.cpp
@@ -14,9 +14,9 @@ int ca[] = {
 };
 
 // CHECK: %arrayinit.element = getelementptr inbounds i32, ptr %notca, i64 1
-// CHECK: store i8 106, ptr %arrayinit.element, align 4
+// CHECK: store i32 106, ptr %arrayinit.element, align 4
 // CHECK: %arrayinit.element1 = getelementptr inbounds i32, ptr %notca, i64 2
-// CHECK: store i8 107, ptr %arrayinit.element1, align 4
+// CHECK: store i32 107, ptr %arrayinit.element1, align 4
 int notca[] = {
 a
 #embed <jk.txt> prefix(,)
@@ -75,9 +75,9 @@ constexpr struct T t[] = {
 // CHECK:  %arrayinit.element7 = getelementptr inbounds %struct.T, ptr %tnonc, i64 1
 // CHECK:  call void @llvm.memset.p0.i64(ptr align 4 %arrayinit.element7, i8 0, i64 20, i1 false)
 // CHECK:  %arr8 = getelementptr inbounds %struct.T, ptr %arrayinit.element7, i32 0, i32 0
-// CHECK:  store i8 106, ptr %arr8, align 4
+// CHECK:  store i32 106, ptr %arr8, align 4
 // CHECK:  %arrayinit.element9 = getelementptr inbounds i32, ptr %arr8, i64 1
-// CHECK:  store i8 107, ptr %arrayinit.element9, align 4
+// CHECK:  store i32 107, ptr %arrayinit.element9, align 4
 struct T tnonc[] = {
   a, 300, 1, 2, 3
 #embed <jk.txt> prefix(,)
diff --git a/clang/test/Preprocessor/embed_weird.cpp b/clang/test/Preprocessor/embed_weird.cpp
index cc73a88e5a657..6eb2923152f15 100644
--- a/clang/test/Preprocessor/embed_weird.cpp
+++ b/clang/test/Preprocessor/embed_weird.cpp
@@ -115,3 +115,14 @@ void f1() {
   };
 }
 #endif
+
+struct HasChar {
+  signed char ch;
+};
+
+constexpr struct HasChar c = {
+#embed "Inputs/big_char.txt" // cxx-error {{constant expression evaluates to 255 which cannot be narrowed to type 'signed char'}} \
+                                cxx-note {{insert an explicit cast to silence this issue}} \
+                                c-error {{constexpr initializer evaluates to 255 which is not exactly representable in type 'signed char'}}
+
+};

From 7d8375b86ee490acafb4254603ccd4edc2a58256 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Horv=C3=A1th?= <xazax.hun@gmail.com>
Date: Fri, 19 Jul 2024 05:24:38 -0700
Subject: [PATCH 189/486] [clang][driver] Fix -print-libgcc-file-name on Darwin
 platforms (#98325)

On Darwin, -print-libgcc-file-name was returning a nonsensical result.
It would return the name of the library that would be used by the
default toolchain implementation, but that was something that didn't
exist on Darwin.

Fixing this requires initializing the Darwin toolchain before processing
`-print-libgcc-file-name`. Previously, the Darwin toolchain would only
be initialized when building the jobs for this compilation, which is too
late since `-print-libgcc-file-name` requires the toolchain to be
initialized in order to provide the right results.

rdar://90633749

Co-authored-by: Gabor Horvath <gaborh@apple.com>
---
 clang/include/clang/Driver/Driver.h           |  5 +-
 clang/lib/Driver/Driver.cpp                   | 10 +-
 clang/lib/Driver/ToolChains/Darwin.cpp        | 65 ++++++++-----
 clang/lib/Driver/ToolChains/Darwin.h          | 13 +++
 .../Driver/darwin-print-libgcc-file-name.c    | 91 +++++++++++++++++++
 5 files changed, 160 insertions(+), 24 deletions(-)
 create mode 100644 clang/test/Driver/darwin-print-libgcc-file-name.c

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index cc1538372d5f8..04b46782467d6 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -628,8 +628,9 @@ class Driver {
   /// treated before building actions or binding tools.
   ///
   /// \return Whether any compilation should be built for this
-  /// invocation.
-  bool HandleImmediateArgs(const Compilation &C);
+  /// invocation. The compilation can only be modified when
+  /// this function returns false.
+  bool HandleImmediateArgs(Compilation &C);
 
   /// ConstructAction - Construct the appropriate action to do for
   /// \p Phase on the \p Input, taking in to account arguments
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 28c3b52483e51..8e44d5afa40e0 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -2128,7 +2128,7 @@ void Driver::HandleAutocompletions(StringRef PassedFlags) const {
   llvm::outs() << llvm::join(SuggestedCompletions, "\n") << '\n';
 }
 
-bool Driver::HandleImmediateArgs(const Compilation &C) {
+bool Driver::HandleImmediateArgs(Compilation &C) {
   // The order these options are handled in gcc is all over the place, but we
   // don't expect inconsistencies w.r.t. that to matter in practice.
 
@@ -2271,6 +2271,14 @@ bool Driver::HandleImmediateArgs(const Compilation &C) {
   if (C.getArgs().hasArg(options::OPT_print_libgcc_file_name)) {
     ToolChain::RuntimeLibType RLT = TC.GetRuntimeLibType(C.getArgs());
     const llvm::Triple Triple(TC.ComputeEffectiveClangTriple(C.getArgs()));
+    // The 'Darwin' toolchain is initialized only when its arguments are
+    // computed. Get the default arguments for OFK_None to ensure that
+    // initialization is performed before trying to access properties of
+    // the toolchain in the functions below.
+    // FIXME: Remove when darwin's toolchain is initialized during construction.
+    // FIXME: For some more esoteric targets the default toolchain is not the
+    //        correct one.
+    C.getArgsForToolChain(&TC, Triple.getArchName(), Action::OFK_None);
     RegisterEffectiveTriple TripleRAII(TC, Triple);
     switch (RLT) {
     case ToolChain::RLT_CompilerRT:
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index f354b0974d5f2..c6f9d7beffb1d 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -1272,23 +1272,8 @@ unsigned DarwinClang::GetDefaultDwarfVersion() const {
 void MachO::AddLinkRuntimeLib(const ArgList &Args, ArgStringList &CmdArgs,
                               StringRef Component, RuntimeLinkOptions Opts,
                               bool IsShared) const {
-  SmallString<64> DarwinLibName = StringRef("libclang_rt.");
-  // On Darwin the builtins component is not in the library name.
-  if (Component != "builtins") {
-    DarwinLibName += Component;
-    if (!(Opts & RLO_IsEmbedded))
-      DarwinLibName += "_";
-  }
-
-  DarwinLibName += getOSLibraryNameSuffix();
-  DarwinLibName += IsShared ? "_dynamic.dylib" : ".a";
-  SmallString<128> Dir(getDriver().ResourceDir);
-  llvm::sys::path::append(Dir, "lib", "darwin");
-  if (Opts & RLO_IsEmbedded)
-    llvm::sys::path::append(Dir, "macho_embedded");
-
-  SmallString<128> P(Dir);
-  llvm::sys::path::append(P, DarwinLibName);
+  std::string P = getCompilerRT(
+      Args, Component, IsShared ? ToolChain::FT_Shared : ToolChain::FT_Static);
 
   // For now, allow missing resource libraries to support developers who may
   // not have compiler-rt checked out or integrated into their build (unless
@@ -1303,18 +1288,56 @@ void MachO::AddLinkRuntimeLib(const ArgList &Args, ArgStringList &CmdArgs,
   // rpaths. This is currently true from this place, but we need to be
   // careful if this function is ever called before user's rpaths are emitted.
   if (Opts & RLO_AddRPath) {
-    assert(DarwinLibName.ends_with(".dylib") && "must be a dynamic library");
+    assert(StringRef(P).ends_with(".dylib") && "must be a dynamic library");
 
     // Add @executable_path to rpath to support having the dylib copied with
     // the executable.
     CmdArgs.push_back("-rpath");
     CmdArgs.push_back("@executable_path");
 
-    // Add the path to the resource dir to rpath to support using the dylib
-    // from the default location without copying.
+    // Add the compiler-rt library's directory to rpath to support using the
+    // dylib from the default location without copying.
     CmdArgs.push_back("-rpath");
-    CmdArgs.push_back(Args.MakeArgString(Dir));
+    CmdArgs.push_back(Args.MakeArgString(llvm::sys::path::parent_path(P)));
+  }
+}
+
+std::string MachO::getCompilerRT(const ArgList &, StringRef Component,
+                                 FileType Type) const {
+  assert(Type != ToolChain::FT_Object &&
+         "it doesn't make sense to ask for the compiler-rt library name as an "
+         "object file");
+  SmallString<64> MachOLibName = StringRef("libclang_rt");
+  // On MachO, the builtins component is not in the library name
+  if (Component != "builtins") {
+    MachOLibName += '.';
+    MachOLibName += Component;
+  }
+  MachOLibName += Type == ToolChain::FT_Shared ? "_dynamic.dylib" : ".a";
+
+  SmallString<128> FullPath(getDriver().ResourceDir);
+  llvm::sys::path::append(FullPath, "lib", "darwin", "macho_embedded",
+                          MachOLibName);
+  return std::string(FullPath);
+}
+
+std::string Darwin::getCompilerRT(const ArgList &, StringRef Component,
+                                  FileType Type) const {
+  assert(Type != ToolChain::FT_Object &&
+         "it doesn't make sense to ask for the compiler-rt library name as an "
+         "object file");
+  SmallString<64> DarwinLibName = StringRef("libclang_rt.");
+  // On Darwin, the builtins component is not in the library name
+  if (Component != "builtins") {
+    DarwinLibName += Component;
+    DarwinLibName += '_';
   }
+  DarwinLibName += getOSLibraryNameSuffix();
+  DarwinLibName += Type == ToolChain::FT_Shared ? "_dynamic.dylib" : ".a";
+
+  SmallString<128> FullPath(getDriver().ResourceDir);
+  llvm::sys::path::append(FullPath, "lib", "darwin", DarwinLibName);
+  return std::string(FullPath);
 }
 
 StringRef Darwin::getPlatformFamily() const {
diff --git a/clang/lib/Driver/ToolChains/Darwin.h b/clang/lib/Driver/ToolChains/Darwin.h
index b45279ecedeb2..2e55b49682a7e 100644
--- a/clang/lib/Driver/ToolChains/Darwin.h
+++ b/clang/lib/Driver/ToolChains/Darwin.h
@@ -223,6 +223,13 @@ class LLVM_LIBRARY_VISIBILITY MachO : public ToolChain {
     // There aren't any profiling libs for embedded targets currently.
   }
 
+  // Return the full path of the compiler-rt library on a non-Darwin MachO
+  // system. Those are under
+  // <resourcedir>/lib/darwin/macho_embedded/<...>(.dylib|.a).
+  std::string
+  getCompilerRT(const llvm::opt::ArgList &Args, StringRef Component,
+                FileType Type = ToolChain::FT_Static) const override;
+
   /// }
   /// @name ToolChain Implementation
   /// {
@@ -356,6 +363,12 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO {
   void addProfileRTLibs(const llvm::opt::ArgList &Args,
                         llvm::opt::ArgStringList &CmdArgs) const override;
 
+  // Return the full path of the compiler-rt library on a Darwin MachO system.
+  // Those are under <resourcedir>/lib/darwin/<...>(.dylib|.a).
+  std::string
+  getCompilerRT(const llvm::opt::ArgList &Args, StringRef Component,
+                FileType Type = ToolChain::FT_Static) const override;
+
 protected:
   /// }
   /// @name Darwin specific Toolchain functions
diff --git a/clang/test/Driver/darwin-print-libgcc-file-name.c b/clang/test/Driver/darwin-print-libgcc-file-name.c
new file mode 100644
index 0000000000000..73e89b6718642
--- /dev/null
+++ b/clang/test/Driver/darwin-print-libgcc-file-name.c
@@ -0,0 +1,91 @@
+// Test the output of -print-libgcc-file-name on Darwin.
+
+//
+// All platforms
+//
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     --target=x86_64-apple-macos \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-MACOS %s
+// CHECK-CLANGRT-MACOS: libclang_rt.osx.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     --target=arm64-apple-ios \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-IOS %s
+// CHECK-CLANGRT-IOS: libclang_rt.ios.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     --target=arm64-apple-watchos \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-WATCHOS %s
+// CHECK-CLANGRT-WATCHOS: libclang_rt.watchos.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     --target=arm64-apple-tvos \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-TVOS %s
+// CHECK-CLANGRT-TVOS: libclang_rt.tvos.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     --target=arm64-apple-driverkit \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-DRIVERKIT %s
+// CHECK-CLANGRT-DRIVERKIT: libclang_rt.driverkit.a
+
+//
+// Simulators
+//
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     --target=arm64-apple-ios-simulator \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-IOS-SIMULATOR %s
+// CHECK-CLANGRT-IOS-SIMULATOR: libclang_rt.iossim.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     --target=arm64-apple-watchos-simulator \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-WATCHOS-SIMULATOR %s
+// CHECK-CLANGRT-WATCHOS-SIMULATOR: libclang_rt.watchossim.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     --target=arm64-apple-tvos-simulator \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-TVOS-SIMULATOR %s
+// CHECK-CLANGRT-TVOS-SIMULATOR: libclang_rt.tvossim.a
+
+// Check the sanitizer and profile variants
+// While the driver also links in sanitizer-specific dylibs, the result of
+// -print-libgcc-file-name is the path of the basic compiler-rt library.
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     -fsanitize=address --target=x86_64-apple-macos \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-MACOS-SAN %s
+// CHECK-CLANGRT-MACOS-SAN: libclang_rt.osx.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     -fsanitize=address --target=arm64-apple-ios \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-IOS-SAN %s
+// CHECK-CLANGRT-IOS-SAN: libclang_rt.ios.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     -fsanitize=address --target=arm64-apple-watchos \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-WATCHOS-SAN %s
+// CHECK-CLANGRT-WATCHOS-SAN: libclang_rt.watchos.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     -fsanitize=address --target=arm64-apple-tvos \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-TVOS-SAN %s
+// CHECK-CLANGRT-TVOS-SAN: libclang_rt.tvos.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     -fsanitize=address --target=arm64-apple-driverkit \
+// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-DRIVERKIT-SAN %s
+// CHECK-CLANGRT-DRIVERKIT-SAN: libclang_rt.driverkit.a

From 007aa6d1b274baf0b4d1c02a8d7d56f219e5f127 Mon Sep 17 00:00:00 2001
From: Yangyu Chen <cyy@cyyself.name>
Date: Fri, 19 Jul 2024 20:32:28 +0800
Subject: [PATCH 190/486] [SLP] Increase UsesLimit to 64 (#99467)

Since commit 82b800ecb35fb46881aa52000fa40b1b99aa654e addressed the
issue #99327 , we see some performance regression (13%) on some
verilator generated C++ code. This is because the UsesLimit is set to 8,
which is too small for the verilator generated code. I have analyzed the
need for the UsesLimit from [1] and found that the UsesLimit should be
at least 64 to cover most of these cases. Thus, This patch increases the
UsesLimit to 64.

Link:
https://github.com/llvm/llvm-project/issues/99327#issuecomment-2236052879
[1]

Signed-off-by: Yangyu Chen <cyy@cyyself.name>
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d8c3bae06e932..bf2f2c334e4c1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -209,7 +209,7 @@ static const unsigned AliasedCheckLimit = 10;
 
 // Limit of the number of uses for potentially transformed instructions/values,
 // used in checks to avoid compile-time explode.
-static constexpr int UsesLimit = 8;
+static constexpr int UsesLimit = 64;
 
 // Another limit for the alias checks: The maximum distance between load/store
 // instructions where alias checks are done.

From 8a79dc7e6f765f3f49c5dd9330fc0826d3362858 Mon Sep 17 00:00:00 2001
From: Egor Zhdan <e_zhdan@apple.com>
Date: Fri, 19 Jul 2024 13:35:13 +0100
Subject: [PATCH 191/486] [APINotes] Support annotating C++ methods

This adds support for adding Clang attributes to C++ methods declared
within C++ records by using API Notes.

For instance:
```
Tags:
- Name: IntWrapper
  Methods:
  - Name: getIncremented
    Availability: none
```

This is the first instance of something within a C++ record being
annotated with API Notes, so it adds the necessary infra to make a C++
record an "API Notes context".

Notably this does not add support for nested C++ tags. That will be
added in a follow-up patch.

rdar://131387880
---
 clang/include/clang/APINotes/APINotesReader.h |  21 +++
 clang/include/clang/APINotes/APINotesWriter.h |   8 +
 clang/include/clang/APINotes/Types.h          |   7 +
 clang/lib/APINotes/APINotesFormat.h           |  29 ++++
 clang/lib/APINotes/APINotesReader.cpp         | 149 ++++++++++++++++
 clang/lib/APINotes/APINotesWriter.cpp         |  72 ++++++++
 clang/lib/APINotes/APINotesYAMLCompiler.cpp   | 164 ++++++++++--------
 clang/lib/Sema/SemaAPINotes.cpp               |  50 +++++-
 .../APINotes/Inputs/Headers/Methods.apinotes  |   8 +
 clang/test/APINotes/Inputs/Headers/Methods.h  |  14 ++
 .../Inputs/Headers/Namespaces.apinotes        |   3 +
 .../test/APINotes/Inputs/Headers/Namespaces.h |   1 +
 .../APINotes/Inputs/Headers/module.modulemap  |   4 +
 clang/test/APINotes/methods.cpp               |   9 +
 clang/test/APINotes/namespaces.cpp            |   5 +
 15 files changed, 465 insertions(+), 79 deletions(-)
 create mode 100644 clang/test/APINotes/Inputs/Headers/Methods.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/Methods.h
 create mode 100644 clang/test/APINotes/methods.cpp

diff --git a/clang/include/clang/APINotes/APINotesReader.h b/clang/include/clang/APINotes/APINotesReader.h
index 37a4ff7a69712..03657352c49a5 100644
--- a/clang/include/clang/APINotes/APINotesReader.h
+++ b/clang/include/clang/APINotes/APINotesReader.h
@@ -141,6 +141,16 @@ class APINotesReader {
                                                  ObjCSelectorRef Selector,
                                                  bool IsInstanceMethod);
 
+  /// Look for information regarding the given C++ method in the given C++ tag
+  /// context.
+  ///
+  /// \param CtxID The ID that references the parent context, i.e. a C++ tag.
+  /// \param Name The name of the C++ method we're looking for.
+  ///
+  /// \returns Information about the method, if known.
+  VersionedInfo<CXXMethodInfo> lookupCXXMethod(ContextID CtxID,
+                                               llvm::StringRef Name);
+
   /// Look for information regarding the given global variable.
   ///
   /// \param Name The name of the global variable.
@@ -166,6 +176,17 @@ class APINotesReader {
   /// \returns information about the enumerator, if known.
   VersionedInfo<EnumConstantInfo> lookupEnumConstant(llvm::StringRef Name);
 
+  /// Look for the context ID of the given C++ tag.
+  ///
+  /// \param Name The name of the tag we're looking for.
+  /// \param ParentCtx The context in which this tag is declared, e.g. a C++
+  /// namespace.
+  ///
+  /// \returns The ID, if known.
+  std::optional<ContextID>
+  lookupTagID(llvm::StringRef Name,
+              std::optional<Context> ParentCtx = std::nullopt);
+
   /// Look for information regarding the given tag
   /// (struct/union/enum/C++ class).
   ///
diff --git a/clang/include/clang/APINotes/APINotesWriter.h b/clang/include/clang/APINotes/APINotesWriter.h
index e82dbc7c9540e..e0fe5eacef725 100644
--- a/clang/include/clang/APINotes/APINotesWriter.h
+++ b/clang/include/clang/APINotes/APINotesWriter.h
@@ -78,6 +78,14 @@ class APINotesWriter {
                      bool IsInstanceMethod, const ObjCMethodInfo &Info,
                      llvm::VersionTuple SwiftVersion);
 
+  /// Add information about a specific C++ method.
+  ///
+  /// \param CtxID The context in which this method resides, i.e. a C++ tag.
+  /// \param Name The name of the method.
+  /// \param Info Information about this method.
+  void addCXXMethod(ContextID CtxID, llvm::StringRef Name,
+                    const CXXMethodInfo &Info, llvm::VersionTuple SwiftVersion);
+
   /// Add information about a global variable.
   ///
   /// \param Name The name of this global variable.
diff --git a/clang/include/clang/APINotes/Types.h b/clang/include/clang/APINotes/Types.h
index b389aa8d56f16..c8e5e4df25d17 100644
--- a/clang/include/clang/APINotes/Types.h
+++ b/clang/include/clang/APINotes/Types.h
@@ -656,6 +656,12 @@ class GlobalFunctionInfo : public FunctionInfo {
   GlobalFunctionInfo() {}
 };
 
+/// Describes API notes data for a C++ method.
+class CXXMethodInfo : public FunctionInfo {
+public:
+  CXXMethodInfo() {}
+};
+
 /// Describes API notes data for an enumerator.
 class EnumConstantInfo : public CommonEntityInfo {
 public:
@@ -789,6 +795,7 @@ enum class ContextKind : uint8_t {
   ObjCClass = 0,
   ObjCProtocol = 1,
   Namespace = 2,
+  Tag = 3,
 };
 
 struct Context {
diff --git a/clang/lib/APINotes/APINotesFormat.h b/clang/lib/APINotes/APINotesFormat.h
index 42dfe7a773a97..cd6456dbe37b2 100644
--- a/clang/lib/APINotes/APINotesFormat.h
+++ b/clang/lib/APINotes/APINotesFormat.h
@@ -63,6 +63,10 @@ enum BlockID {
   /// about the method.
   OBJC_METHOD_BLOCK_ID,
 
+  /// The C++ method data block, which maps C++ (context id, method name) pairs
+  /// to information about the method.
+  CXX_METHOD_BLOCK_ID,
+
   /// The Objective-C selector data block, which maps Objective-C
   /// selector names (# of pieces, identifier IDs) to the selector ID
   /// used in other tables.
@@ -181,6 +185,20 @@ using ObjCMethodDataLayout =
                          >;
 } // namespace objc_method_block
 
+namespace cxx_method_block {
+enum {
+  CXX_METHOD_DATA = 1,
+};
+
+using CXXMethodDataLayout =
+    llvm::BCRecordLayout<CXX_METHOD_DATA, // record ID
+                         llvm::BCVBR<16>, // table offset within the blob (see
+                                          // below)
+                         llvm::BCBlob     // map from C++ (context id, name)
+                                          // tuples to C++ method information
+                         >;
+} // namespace cxx_method_block
+
 namespace objc_selector_block {
 enum {
   OBJC_SELECTOR_DATA = 1,
@@ -269,6 +287,17 @@ struct ContextTableKey {
       : parentContextID(parentContextID), contextKind(contextKind),
         contextID(contextID) {}
 
+  ContextTableKey(std::optional<ContextID> ParentContextID, ContextKind Kind,
+                  uint32_t ContextID)
+      : parentContextID(ParentContextID ? ParentContextID->Value : -1),
+        contextKind(static_cast<uint8_t>(Kind)), contextID(ContextID) {}
+
+  ContextTableKey(std::optional<Context> ParentContext, ContextKind Kind,
+                  uint32_t ContextID)
+      : ContextTableKey(ParentContext ? std::make_optional(ParentContext->id)
+                                      : std::nullopt,
+                        Kind, ContextID) {}
+
   llvm::hash_code hashValue() const {
     return llvm::hash_value(
         std::tuple{parentContextID, contextKind, contextID});
diff --git a/clang/lib/APINotes/APINotesReader.cpp b/clang/lib/APINotes/APINotesReader.cpp
index 7600738374840..871f782511d5f 100644
--- a/clang/lib/APINotes/APINotesReader.cpp
+++ b/clang/lib/APINotes/APINotesReader.cpp
@@ -473,6 +473,29 @@ class GlobalFunctionTableInfo
   }
 };
 
+/// Used to deserialize the on-disk C++ method table.
+class CXXMethodTableInfo
+    : public VersionedTableInfo<CXXMethodTableInfo, SingleDeclTableKey,
+                                CXXMethodInfo> {
+public:
+  static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
+    auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
+    auto NameID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
+    return {CtxID, NameID};
+  }
+
+  hash_value_type ComputeHash(internal_key_type Key) {
+    return static_cast<size_t>(Key.hashValue());
+  }
+
+  static CXXMethodInfo readUnversioned(internal_key_type Key,
+                                       const uint8_t *&Data) {
+    CXXMethodInfo Info;
+    ReadFunctionInfo(Data, Info);
+    return Info;
+  }
+};
+
 /// Used to deserialize the on-disk enumerator table.
 class EnumConstantTableInfo
     : public VersionedTableInfo<EnumConstantTableInfo, uint32_t,
@@ -630,6 +653,12 @@ class APINotesReader::Implementation {
   /// The Objective-C method table.
   std::unique_ptr<SerializedObjCMethodTable> ObjCMethodTable;
 
+  using SerializedCXXMethodTable =
+      llvm::OnDiskIterableChainedHashTable<CXXMethodTableInfo>;
+
+  /// The C++ method table.
+  std::unique_ptr<SerializedCXXMethodTable> CXXMethodTable;
+
   using SerializedObjCSelectorTable =
       llvm::OnDiskIterableChainedHashTable<ObjCSelectorTableInfo>;
 
@@ -683,6 +712,8 @@ class APINotesReader::Implementation {
                              llvm::SmallVectorImpl<uint64_t> &Scratch);
   bool readObjCMethodBlock(llvm::BitstreamCursor &Cursor,
                            llvm::SmallVectorImpl<uint64_t> &Scratch);
+  bool readCXXMethodBlock(llvm::BitstreamCursor &Cursor,
+                          llvm::SmallVectorImpl<uint64_t> &Scratch);
   bool readObjCSelectorBlock(llvm::BitstreamCursor &Cursor,
                              llvm::SmallVectorImpl<uint64_t> &Scratch);
   bool readGlobalVariableBlock(llvm::BitstreamCursor &Cursor,
@@ -1140,6 +1171,81 @@ bool APINotesReader::Implementation::readObjCMethodBlock(
   return false;
 }
 
+bool APINotesReader::Implementation::readCXXMethodBlock(
+    llvm::BitstreamCursor &Cursor, llvm::SmallVectorImpl<uint64_t> &Scratch) {
+  if (Cursor.EnterSubBlock(CXX_METHOD_BLOCK_ID))
+    return true;
+
+  llvm::Expected<llvm::BitstreamEntry> MaybeNext = Cursor.advance();
+  if (!MaybeNext) {
+    // FIXME this drops the error on the floor.
+    consumeError(MaybeNext.takeError());
+    return false;
+  }
+  llvm::BitstreamEntry Next = MaybeNext.get();
+  while (Next.Kind != llvm::BitstreamEntry::EndBlock) {
+    if (Next.Kind == llvm::BitstreamEntry::Error)
+      return true;
+
+    if (Next.Kind == llvm::BitstreamEntry::SubBlock) {
+      // Unknown sub-block, possibly for use by a future version of the
+      // API notes format.
+      if (Cursor.SkipBlock())
+        return true;
+
+      MaybeNext = Cursor.advance();
+      if (!MaybeNext) {
+        // FIXME this drops the error on the floor.
+        consumeError(MaybeNext.takeError());
+        return false;
+      }
+      Next = MaybeNext.get();
+      continue;
+    }
+
+    Scratch.clear();
+    llvm::StringRef BlobData;
+    llvm::Expected<unsigned> MaybeKind =
+        Cursor.readRecord(Next.ID, Scratch, &BlobData);
+    if (!MaybeKind) {
+      // FIXME this drops the error on the floor.
+      consumeError(MaybeKind.takeError());
+      return false;
+    }
+    unsigned Kind = MaybeKind.get();
+    switch (Kind) {
+    case cxx_method_block::CXX_METHOD_DATA: {
+      // Already saw C++ method table.
+      if (CXXMethodTable)
+        return true;
+
+      uint32_t tableOffset;
+      cxx_method_block::CXXMethodDataLayout::readRecord(Scratch, tableOffset);
+      auto base = reinterpret_cast<const uint8_t *>(BlobData.data());
+
+      CXXMethodTable.reset(SerializedCXXMethodTable::Create(
+          base + tableOffset, base + sizeof(uint32_t), base));
+      break;
+    }
+
+    default:
+      // Unknown record, possibly for use by a future version of the
+      // module format.
+      break;
+    }
+
+    MaybeNext = Cursor.advance();
+    if (!MaybeNext) {
+      // FIXME this drops the error on the floor.
+      consumeError(MaybeNext.takeError());
+      return false;
+    }
+    Next = MaybeNext.get();
+  }
+
+  return false;
+}
+
 bool APINotesReader::Implementation::readObjCSelectorBlock(
     llvm::BitstreamCursor &Cursor, llvm::SmallVectorImpl<uint64_t> &Scratch) {
   if (Cursor.EnterSubBlock(OBJC_SELECTOR_BLOCK_ID))
@@ -1692,6 +1798,14 @@ APINotesReader::APINotesReader(llvm::MemoryBuffer *InputBuffer,
       }
       break;
 
+    case CXX_METHOD_BLOCK_ID:
+      if (!HasValidControlBlock ||
+          Implementation->readCXXMethodBlock(Cursor, Scratch)) {
+        Failed = true;
+        return;
+      }
+      break;
+
     case OBJC_SELECTOR_BLOCK_ID:
       if (!HasValidControlBlock ||
           Implementation->readObjCSelectorBlock(Cursor, Scratch)) {
@@ -1911,6 +2025,23 @@ auto APINotesReader::lookupObjCMethod(ContextID CtxID, ObjCSelectorRef Selector,
   return {Implementation->SwiftVersion, *Known};
 }
 
+auto APINotesReader::lookupCXXMethod(ContextID CtxID, llvm::StringRef Name)
+    -> VersionedInfo<CXXMethodInfo> {
+  if (!Implementation->CXXMethodTable)
+    return std::nullopt;
+
+  std::optional<IdentifierID> NameID = Implementation->getIdentifier(Name);
+  if (!NameID)
+    return std::nullopt;
+
+  auto Known = Implementation->CXXMethodTable->find(
+      SingleDeclTableKey(CtxID.Value, *NameID));
+  if (Known == Implementation->CXXMethodTable->end())
+    return std::nullopt;
+
+  return {Implementation->SwiftVersion, *Known};
+}
+
 auto APINotesReader::lookupGlobalVariable(llvm::StringRef Name,
                                           std::optional<Context> Ctx)
     -> VersionedInfo<GlobalVariableInfo> {
@@ -1965,6 +2096,24 @@ auto APINotesReader::lookupEnumConstant(llvm::StringRef Name)
   return {Implementation->SwiftVersion, *Known};
 }
 
+auto APINotesReader::lookupTagID(llvm::StringRef Name,
+                                 std::optional<Context> ParentCtx)
+    -> std::optional<ContextID> {
+  if (!Implementation->ContextIDTable)
+    return std::nullopt;
+
+  std::optional<IdentifierID> TagID = Implementation->getIdentifier(Name);
+  if (!TagID)
+    return std::nullopt;
+
+  auto KnownID = Implementation->ContextIDTable->find(
+      ContextTableKey(ParentCtx, ContextKind::Tag, *TagID));
+  if (KnownID == Implementation->ContextIDTable->end())
+    return std::nullopt;
+
+  return ContextID(*KnownID);
+}
+
 auto APINotesReader::lookupTag(llvm::StringRef Name, std::optional<Context> Ctx)
     -> VersionedInfo<TagInfo> {
   if (!Implementation->TagTable)
diff --git a/clang/lib/APINotes/APINotesWriter.cpp b/clang/lib/APINotes/APINotesWriter.cpp
index 1090d3f20df21..2a71922746ac5 100644
--- a/clang/lib/APINotes/APINotesWriter.cpp
+++ b/clang/lib/APINotes/APINotesWriter.cpp
@@ -70,6 +70,13 @@ class APINotesWriter::Implementation {
                  llvm::SmallVector<std::pair<VersionTuple, ObjCMethodInfo>, 1>>
       ObjCMethods;
 
+  /// Information about C++ methods.
+  ///
+  /// Indexed by the context ID and name ID.
+  llvm::DenseMap<SingleDeclTableKey,
+                 llvm::SmallVector<std::pair<VersionTuple, CXXMethodInfo>, 1>>
+      CXXMethods;
+
   /// Mapping from selectors to selector ID.
   llvm::DenseMap<StoredObjCSelector, SelectorID> SelectorIDs;
 
@@ -150,6 +157,7 @@ class APINotesWriter::Implementation {
   void writeContextBlock(llvm::BitstreamWriter &Stream);
   void writeObjCPropertyBlock(llvm::BitstreamWriter &Stream);
   void writeObjCMethodBlock(llvm::BitstreamWriter &Stream);
+  void writeCXXMethodBlock(llvm::BitstreamWriter &Stream);
   void writeObjCSelectorBlock(llvm::BitstreamWriter &Stream);
   void writeGlobalVariableBlock(llvm::BitstreamWriter &Stream);
   void writeGlobalFunctionBlock(llvm::BitstreamWriter &Stream);
@@ -181,6 +189,7 @@ void APINotesWriter::Implementation::writeToStream(llvm::raw_ostream &OS) {
     writeContextBlock(Stream);
     writeObjCPropertyBlock(Stream);
     writeObjCMethodBlock(Stream);
+    writeCXXMethodBlock(Stream);
     writeObjCSelectorBlock(Stream);
     writeGlobalVariableBlock(Stream);
     writeGlobalFunctionBlock(Stream);
@@ -765,6 +774,34 @@ class ObjCMethodTableInfo
     emitFunctionInfo(OS, OMI);
   }
 };
+
+/// Used to serialize the on-disk C++ method table.
+class CXXMethodTableInfo
+    : public VersionedTableInfo<CXXMethodTableInfo, SingleDeclTableKey,
+                                CXXMethodInfo> {
+public:
+  unsigned getKeyLength(key_type_ref) {
+    return sizeof(uint32_t) + sizeof(uint32_t);
+  }
+
+  void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
+    writer.write<uint32_t>(Key.parentContextID);
+    writer.write<uint32_t>(Key.nameID);
+  }
+
+  hash_value_type ComputeHash(key_type_ref key) {
+    return static_cast<size_t>(key.hashValue());
+  }
+
+  unsigned getUnversionedInfoSize(const CXXMethodInfo &OMI) {
+    return getFunctionInfoSize(OMI);
+  }
+
+  void emitUnversionedInfo(raw_ostream &OS, const CXXMethodInfo &OMI) {
+    emitFunctionInfo(OS, OMI);
+  }
+};
 } // namespace
 
 void APINotesWriter::Implementation::writeObjCMethodBlock(
@@ -794,6 +831,33 @@ void APINotesWriter::Implementation::writeObjCMethodBlock(
   }
 }
 
+void APINotesWriter::Implementation::writeCXXMethodBlock(
+    llvm::BitstreamWriter &Stream) {
+  llvm::BCBlockRAII Scope(Stream, CXX_METHOD_BLOCK_ID, 3);
+
+  if (CXXMethods.empty())
+    return;
+
+  {
+    llvm::SmallString<4096> HashTableBlob;
+    uint32_t Offset;
+    {
+      llvm::OnDiskChainedHashTableGenerator<CXXMethodTableInfo> Generator;
+      for (auto &MD : CXXMethods)
+        Generator.insert(MD.first, MD.second);
+
+      llvm::raw_svector_ostream BlobStream(HashTableBlob);
+      // Make sure that no bucket is at offset 0
+      llvm::support::endian::write<uint32_t>(BlobStream, 0,
+                                             llvm::endianness::little);
+      Offset = Generator.Emit(BlobStream);
+    }
+
+    cxx_method_block::CXXMethodDataLayout CXXMethodData(Stream);
+    CXXMethodData.emit(Scratch, Offset, HashTableBlob);
+  }
+}
+
 namespace {
 /// Used to serialize the on-disk Objective-C selector table.
 class ObjCSelectorTableInfo {
@@ -1344,6 +1408,14 @@ void APINotesWriter::addObjCMethod(ContextID CtxID, ObjCSelectorRef Selector,
   }
 }
 
+void APINotesWriter::addCXXMethod(ContextID CtxID, llvm::StringRef Name,
+                                  const CXXMethodInfo &Info,
+                                  VersionTuple SwiftVersion) {
+  IdentifierID NameID = Implementation->getIdentifier(Name);
+  SingleDeclTableKey Key(CtxID.Value, NameID);
+  Implementation->CXXMethods[Key].push_back({SwiftVersion, Info});
+}
+
 void APINotesWriter::addGlobalVariable(std::optional<Context> Ctx,
                                        llvm::StringRef Name,
                                        const GlobalVariableInfo &Info,
diff --git a/clang/lib/APINotes/APINotesYAMLCompiler.cpp b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
index 870b64e3b7a9b..060e1fdaf2fd9 100644
--- a/clang/lib/APINotes/APINotesYAMLCompiler.cpp
+++ b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
@@ -420,6 +420,7 @@ struct Tag {
   std::optional<bool> FlagEnum;
   std::optional<EnumConvenienceAliasKind> EnumConvenienceKind;
   std::optional<bool> SwiftCopyable;
+  FunctionsSeq Methods;
 };
 
 typedef std::vector<Tag> TagsSeq;
@@ -454,6 +455,7 @@ template <> struct MappingTraits<Tag> {
     IO.mapOptional("FlagEnum", T.FlagEnum);
     IO.mapOptional("EnumKind", T.EnumConvenienceKind);
     IO.mapOptional("SwiftCopyable", T.SwiftCopyable);
+    IO.mapOptional("Methods", T.Methods);
   }
 };
 } // namespace yaml
@@ -874,6 +876,96 @@ class YAMLConverter {
                          TheNamespace.Items, SwiftVersion);
   }
 
+  void convertFunction(const Function &Function, FunctionInfo &FI) {
+    convertAvailability(Function.Availability, FI, Function.Name);
+    FI.setSwiftPrivate(Function.SwiftPrivate);
+    FI.SwiftName = std::string(Function.SwiftName);
+    convertParams(Function.Params, FI);
+    convertNullability(Function.Nullability, Function.NullabilityOfRet, FI,
+                       Function.Name);
+    FI.ResultType = std::string(Function.ResultType);
+    FI.setRetainCountConvention(Function.RetainCountConvention);
+  }
+
+  void convertTagContext(std::optional<Context> ParentContext, const Tag &T,
+                         VersionTuple SwiftVersion) {
+    TagInfo TI;
+    std::optional<ContextID> ParentContextID =
+        ParentContext ? std::optional<ContextID>(ParentContext->id)
+                      : std::nullopt;
+    convertCommonType(T, TI, T.Name);
+
+    if ((T.SwiftRetainOp || T.SwiftReleaseOp) && !T.SwiftImportAs) {
+      emitError(llvm::Twine("should declare SwiftImportAs to use "
+                            "SwiftRetainOp and SwiftReleaseOp (for ") +
+                T.Name + ")");
+      return;
+    }
+    if (T.SwiftReleaseOp.has_value() != T.SwiftRetainOp.has_value()) {
+      emitError(llvm::Twine("should declare both SwiftReleaseOp and "
+                            "SwiftRetainOp (for ") +
+                T.Name + ")");
+      return;
+    }
+
+    if (T.SwiftImportAs)
+      TI.SwiftImportAs = T.SwiftImportAs;
+    if (T.SwiftRetainOp)
+      TI.SwiftRetainOp = T.SwiftRetainOp;
+    if (T.SwiftReleaseOp)
+      TI.SwiftReleaseOp = T.SwiftReleaseOp;
+
+    if (T.SwiftCopyable)
+      TI.setSwiftCopyable(T.SwiftCopyable);
+
+    if (T.EnumConvenienceKind) {
+      if (T.EnumExtensibility) {
+        emitError(
+            llvm::Twine("cannot mix EnumKind and EnumExtensibility (for ") +
+            T.Name + ")");
+        return;
+      }
+      if (T.FlagEnum) {
+        emitError(llvm::Twine("cannot mix EnumKind and FlagEnum (for ") +
+                  T.Name + ")");
+        return;
+      }
+      switch (*T.EnumConvenienceKind) {
+      case EnumConvenienceAliasKind::None:
+        TI.EnumExtensibility = EnumExtensibilityKind::None;
+        TI.setFlagEnum(false);
+        break;
+      case EnumConvenienceAliasKind::CFEnum:
+        TI.EnumExtensibility = EnumExtensibilityKind::Open;
+        TI.setFlagEnum(false);
+        break;
+      case EnumConvenienceAliasKind::CFOptions:
+        TI.EnumExtensibility = EnumExtensibilityKind::Open;
+        TI.setFlagEnum(true);
+        break;
+      case EnumConvenienceAliasKind::CFClosedEnum:
+        TI.EnumExtensibility = EnumExtensibilityKind::Closed;
+        TI.setFlagEnum(false);
+        break;
+      }
+    } else {
+      TI.EnumExtensibility = T.EnumExtensibility;
+      TI.setFlagEnum(T.FlagEnum);
+    }
+
+    Writer.addTag(ParentContext, T.Name, TI, SwiftVersion);
+
+    ContextInfo CI;
+    auto TagCtxID = Writer.addContext(ParentContextID, T.Name, ContextKind::Tag,
+                                      CI, SwiftVersion);
+
+    for (const auto &CXXMethod : T.Methods) {
+      CXXMethodInfo MI;
+      convertFunction(CXXMethod, MI);
+      Writer.addCXXMethod(TagCtxID, CXXMethod.Name, MI, SwiftVersion);
+    }
+  }
+
   void convertTopLevelItems(std::optional<Context> Ctx,
                             const TopLevelItems &TLItems,
                             VersionTuple SwiftVersion) {
@@ -950,14 +1042,7 @@ class YAMLConverter {
       }
 
       GlobalFunctionInfo GFI;
-      convertAvailability(Function.Availability, GFI, Function.Name);
-      GFI.setSwiftPrivate(Function.SwiftPrivate);
-      GFI.SwiftName = std::string(Function.SwiftName);
-      convertParams(Function.Params, GFI);
-      convertNullability(Function.Nullability, Function.NullabilityOfRet, GFI,
-                         Function.Name);
-      GFI.ResultType = std::string(Function.ResultType);
-      GFI.setRetainCountConvention(Function.RetainCountConvention);
+      convertFunction(Function, GFI);
       Writer.addGlobalFunction(Ctx, Function.Name, GFI, SwiftVersion);
     }
 
@@ -988,68 +1073,7 @@ class YAMLConverter {
         continue;
       }
 
-      TagInfo TI;
-      convertCommonType(Tag, TI, Tag.Name);
-
-      if ((Tag.SwiftRetainOp || Tag.SwiftReleaseOp) && !Tag.SwiftImportAs) {
-        emitError(llvm::Twine("should declare SwiftImportAs to use "
-                              "SwiftRetainOp and SwiftReleaseOp (for ") +
-                  Tag.Name + ")");
-        continue;
-      }
-      if (Tag.SwiftReleaseOp.has_value() != Tag.SwiftRetainOp.has_value()) {
-        emitError(llvm::Twine("should declare both SwiftReleaseOp and "
-                              "SwiftRetainOp (for ") +
-                  Tag.Name + ")");
-        continue;
-      }
-
-      if (Tag.SwiftImportAs)
-        TI.SwiftImportAs = Tag.SwiftImportAs;
-      if (Tag.SwiftRetainOp)
-        TI.SwiftRetainOp = Tag.SwiftRetainOp;
-      if (Tag.SwiftReleaseOp)
-        TI.SwiftReleaseOp = Tag.SwiftReleaseOp;
-
-      if (Tag.SwiftCopyable)
-        TI.setSwiftCopyable(Tag.SwiftCopyable);
-
-      if (Tag.EnumConvenienceKind) {
-        if (Tag.EnumExtensibility) {
-          emitError(
-              llvm::Twine("cannot mix EnumKind and EnumExtensibility (for ") +
-              Tag.Name + ")");
-          continue;
-        }
-        if (Tag.FlagEnum) {
-          emitError(llvm::Twine("cannot mix EnumKind and FlagEnum (for ") +
-                    Tag.Name + ")");
-          continue;
-        }
-        switch (*Tag.EnumConvenienceKind) {
-        case EnumConvenienceAliasKind::None:
-          TI.EnumExtensibility = EnumExtensibilityKind::None;
-          TI.setFlagEnum(false);
-          break;
-        case EnumConvenienceAliasKind::CFEnum:
-          TI.EnumExtensibility = EnumExtensibilityKind::Open;
-          TI.setFlagEnum(false);
-          break;
-        case EnumConvenienceAliasKind::CFOptions:
-          TI.EnumExtensibility = EnumExtensibilityKind::Open;
-          TI.setFlagEnum(true);
-          break;
-        case EnumConvenienceAliasKind::CFClosedEnum:
-          TI.EnumExtensibility = EnumExtensibilityKind::Closed;
-          TI.setFlagEnum(false);
-          break;
-        }
-      } else {
-        TI.EnumExtensibility = Tag.EnumExtensibility;
-        TI.setFlagEnum(Tag.FlagEnum);
-      }
-
-      Writer.addTag(Ctx, Tag.Name, TI, SwiftVersion);
+      convertTagContext(Ctx, Tag, SwiftVersion);
     }
 
     // Write all typedefs.
diff --git a/clang/lib/Sema/SemaAPINotes.cpp b/clang/lib/Sema/SemaAPINotes.cpp
index 3482f3741fce6..055e66a0c3486 100644
--- a/clang/lib/Sema/SemaAPINotes.cpp
+++ b/clang/lib/Sema/SemaAPINotes.cpp
@@ -546,6 +546,13 @@ static void ProcessAPINotes(Sema &S, FunctionOrMethod AnyFunc,
                   Metadata);
 }
 
+/// Process API notes for a C++ method.
+static void ProcessAPINotes(Sema &S, CXXMethodDecl *Method,
+                            const api_notes::CXXMethodInfo &Info,
+                            VersionedInfoMetadata Metadata) {
+  ProcessAPINotes(S, (FunctionOrMethod)Method, Info, Metadata);
+}
+
 /// Process API notes for a global function.
 static void ProcessAPINotes(Sema &S, FunctionDecl *D,
                             const api_notes::GlobalFunctionInfo &Info,
@@ -782,13 +789,9 @@ void Sema::ProcessAPINotes(Decl *D) {
   if (!D)
     return;
 
-  // Globals.
-  if (D->getDeclContext()->isFileContext() ||
-      D->getDeclContext()->isNamespace() ||
-      D->getDeclContext()->isExternCContext() ||
-      D->getDeclContext()->isExternCXXContext()) {
-    std::optional<api_notes::Context> APINotesContext;
-    if (auto NamespaceContext = dyn_cast<NamespaceDecl>(D->getDeclContext())) {
+  auto GetNamespaceContext =
+      [&](DeclContext *DC) -> std::optional<api_notes::Context> {
+    if (auto NamespaceContext = dyn_cast<NamespaceDecl>(DC)) {
       for (auto Reader :
            APINotes.findAPINotes(NamespaceContext->getLocation())) {
         // Retrieve the context ID for the parent namespace of the decl.
@@ -811,11 +814,20 @@ void Sema::ProcessAPINotes(Decl *D) {
             break;
         }
         if (NamespaceID)
-          APINotesContext = api_notes::Context(
-              *NamespaceID, api_notes::ContextKind::Namespace);
+          return api_notes::Context(*NamespaceID,
+                                    api_notes::ContextKind::Namespace);
       }
     }
+    return std::nullopt;
+  };
 
+  // Globals.
+  if (D->getDeclContext()->isFileContext() ||
+      D->getDeclContext()->isNamespace() ||
+      D->getDeclContext()->isExternCContext() ||
+      D->getDeclContext()->isExternCXXContext()) {
+    std::optional<api_notes::Context> APINotesContext =
+        GetNamespaceContext(D->getDeclContext());
     // Global variables.
     if (auto VD = dyn_cast<VarDecl>(D)) {
       for (auto Reader : APINotes.findAPINotes(D->getLocation())) {
@@ -1001,4 +1013,24 @@ void Sema::ProcessAPINotes(Decl *D) {
       return;
     }
   }
+
+  if (auto CXXRecord = dyn_cast<CXXRecordDecl>(D->getDeclContext())) {
+    auto GetRecordContext = [&](api_notes::APINotesReader *Reader)
+        -> std::optional<api_notes::ContextID> {
+      auto ParentContext = GetNamespaceContext(CXXRecord->getDeclContext());
+      if (auto Found = Reader->lookupTagID(CXXRecord->getName(), ParentContext))
+        return *Found;
+
+      return std::nullopt;
+    };
+
+    if (auto CXXMethod = dyn_cast<CXXMethodDecl>(D)) {
+      for (auto Reader : APINotes.findAPINotes(D->getLocation())) {
+        if (auto Context = GetRecordContext(Reader)) {
+          auto Info = Reader->lookupCXXMethod(*Context, CXXMethod->getName());
+          ProcessVersionedAPINotes(*this, CXXMethod, Info);
+        }
+      }
+    }
+  }
 }
diff --git a/clang/test/APINotes/Inputs/Headers/Methods.apinotes b/clang/test/APINotes/Inputs/Headers/Methods.apinotes
new file mode 100644
index 0000000000000..0fa6991a51ff4
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/Methods.apinotes
@@ -0,0 +1,8 @@
+---
+Name: Methods
+Tags:
+- Name: IntWrapper
+  Methods:
+  - Name: getIncremented
+    Availability: none
+    AvailabilityMsg: "oh no"
diff --git a/clang/test/APINotes/Inputs/Headers/Methods.h b/clang/test/APINotes/Inputs/Headers/Methods.h
new file mode 100644
index 0000000000000..f46fe31533e5d
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/Methods.h
@@ -0,0 +1,14 @@
+struct IntWrapper {
+  int value;
+
+  IntWrapper getIncremented() const { return {value + 1}; }
+};
+
+// TODO: support nested tags
+struct Outer {
+  struct Inner {
+    int value;
+
+    Inner getDecremented() const { return {value - 1}; }
+  };
+};
diff --git a/clang/test/APINotes/Inputs/Headers/Namespaces.apinotes b/clang/test/APINotes/Inputs/Headers/Namespaces.apinotes
index e9da36787b638..68073932d600e 100644
--- a/clang/test/APINotes/Inputs/Headers/Namespaces.apinotes
+++ b/clang/test/APINotes/Inputs/Headers/Namespaces.apinotes
@@ -38,6 +38,9 @@ Namespaces:
         Tags:
           - Name: char_box
             SwiftName: NestedCharBox
+            Methods:
+              - Name: methodInNestedNamespace
+                SwiftName: swiftMethodInNestedNamespace()
         Namespaces:
           - Name: Namespace1
             Tags:
diff --git a/clang/test/APINotes/Inputs/Headers/Namespaces.h b/clang/test/APINotes/Inputs/Headers/Namespaces.h
index 6a79e996be86c..e996b8ffa6b6e 100644
--- a/clang/test/APINotes/Inputs/Headers/Namespaces.h
+++ b/clang/test/APINotes/Inputs/Headers/Namespaces.h
@@ -9,6 +9,7 @@ namespace Nested1 {
 void funcInNestedNamespace(int i);
 struct char_box {
   char c;
+  void methodInNestedNamespace();
 };
 }
 
diff --git a/clang/test/APINotes/Inputs/Headers/module.modulemap b/clang/test/APINotes/Inputs/Headers/module.modulemap
index d515169184f4f..faf6042c78d57 100644
--- a/clang/test/APINotes/Inputs/Headers/module.modulemap
+++ b/clang/test/APINotes/Inputs/Headers/module.modulemap
@@ -24,6 +24,10 @@ module BrokenTypes {
   header "BrokenTypes.h"
 }
 
+module Methods {
+  header "Methods.h"
+}
+
 module ModuleWithWrongCase {
   header "ModuleWithWrongCase.h"
 }
diff --git a/clang/test/APINotes/methods.cpp b/clang/test/APINotes/methods.cpp
new file mode 100644
index 0000000000000..692f750ed66c7
--- /dev/null
+++ b/clang/test/APINotes/methods.cpp
@@ -0,0 +1,9 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Methods -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -x c++
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Methods -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter IntWrapper::getIncremented -x c++ | FileCheck --check-prefix=CHECK-METHOD %s
+
+#include "Methods.h"
+
+// CHECK-METHOD: Dumping IntWrapper::getIncremented:
+// CHECK-METHOD-NEXT: CXXMethodDecl {{.+}} getIncremented
+// CHECK-METHOD: UnavailableAttr {{.+}} <<invalid sloc>> "oh no"
diff --git a/clang/test/APINotes/namespaces.cpp b/clang/test/APINotes/namespaces.cpp
index c19eee565c2da..a6517a324b9c5 100644
--- a/clang/test/APINotes/namespaces.cpp
+++ b/clang/test/APINotes/namespaces.cpp
@@ -9,6 +9,7 @@
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::Nested2::varInNestedNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-ANOTHER-GLOBAL-IN-NESTED-NAMESPACE %s
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::Nested1::char_box -x objective-c++ | FileCheck -check-prefix=CHECK-STRUCT-IN-NESTED-NAMESPACE %s
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::Nested1::funcInNestedNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-FUNC-IN-NESTED-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::Nested1::char_box::methodInNestedNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-METHOD-IN-NESTED-NAMESPACE %s
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::Nested1::Namespace1::char_box -x objective-c++ | FileCheck -check-prefix=CHECK-STRUCT-IN-DEEP-NESTED-NAMESPACE %s
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter varInInlineNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-GLOBAL-IN-INLINE-NAMESPACE %s
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter funcInInlineNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-FUNC-IN-INLINE-NAMESPACE %s
@@ -55,6 +56,10 @@
 // CHECK-STRUCT-IN-NESTED-NAMESPACE-NEXT: CXXRecordDecl {{.+}} imported in Namespaces <undeserialized declarations> struct char_box
 // CHECK-STRUCT-IN-NESTED-NAMESPACE: SwiftNameAttr {{.+}} <<invalid sloc>> "NestedCharBox"
 
+// CHECK-METHOD-IN-NESTED-NAMESPACE: Dumping Namespace1::Nested1::char_box::methodInNestedNamespace:
+// CHECK-METHOD-IN-NESTED-NAMESPACE-NEXT: CXXMethodDecl {{.+}} imported in Namespaces methodInNestedNamespace
+// CHECK-METHOD-IN-NESTED-NAMESPACE: SwiftNameAttr {{.+}} <<invalid sloc>> "swiftMethodInNestedNamespace()"
+
 // CHECK-STRUCT-IN-DEEP-NESTED-NAMESPACE: Dumping Namespace1::Nested1::Namespace1::char_box:
 // CHECK-STRUCT-IN-DEEP-NESTED-NAMESPACE-NEXT: CXXRecordDecl {{.+}} imported in Namespaces <undeserialized declarations> struct char_box
 // CHECK-STRUCT-IN-DEEP-NESTED-NAMESPACE: SwiftNameAttr {{.+}} <<invalid sloc>> "DeepNestedCharBox"

From d31603eefc2d8becfd1f41327b6a8db3e0e91a27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Fri, 19 Jul 2024 13:20:17 +0200
Subject: [PATCH 192/486] [clang][Interp] Control InitStack activity state in
 visitInitList

This doesn't change anything about the current tests, but helps
once those tests change because of #97308
---
 clang/lib/AST/Interp/Compiler.cpp | 12 ++++++------
 clang/lib/AST/Interp/Compiler.h   | 16 ++++++++++++++++
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/clang/lib/AST/Interp/Compiler.cpp b/clang/lib/AST/Interp/Compiler.cpp
index 24140b23c1f0b..7da711ed485db 100644
--- a/clang/lib/AST/Interp/Compiler.cpp
+++ b/clang/lib/AST/Interp/Compiler.cpp
@@ -1334,6 +1334,7 @@ bool Compiler<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
 
     auto initPrimitiveField = [=](const Record::Field *FieldToInit,
                                   const Expr *Init, PrimType T) -> bool {
+      InitStackScope<Emitter> ISS(this, isa<CXXDefaultInitExpr>(Init));
       if (!this->visit(Init))
         return false;
 
@@ -1344,6 +1345,7 @@ bool Compiler<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
 
     auto initCompositeField = [=](const Record::Field *FieldToInit,
                                   const Expr *Init) -> bool {
+      InitStackScope<Emitter> ISS(this, isa<CXXDefaultInitExpr>(Init));
       InitLinkScope<Emitter> ILS(this, InitLink::Field(FieldToInit->Offset));
       // Non-primitive case. Get a pointer to the field-to-initialize
       // on the stack and recurse into visitInitializer().
@@ -4088,12 +4090,7 @@ template <class Emitter>
 bool Compiler<Emitter>::VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *E) {
   SourceLocScope<Emitter> SLS(this, E);
 
-  bool Old = InitStackActive;
-  InitStackActive =
-      !(E->getUsedContext()->getDeclKind() == Decl::CXXConstructor);
-  bool Result = this->delegate(E->getExpr());
-  InitStackActive = Old;
-  return Result;
+  return this->delegate(E->getExpr());
 }
 
 template <class Emitter>
@@ -4151,6 +4148,9 @@ bool Compiler<Emitter>::VisitCXXThisExpr(const CXXThisExpr *E) {
   // instance pointer of the current function frame, but e.g. to the declaration
   // currently being initialized. Here we emit the necessary instruction(s) for
   // this scenario.
+  if (!InitStackActive || !E->isImplicit())
+    return this->emitThis(E);
+
   if (InitStackActive && !InitStack.empty()) {
     unsigned StartIndex = 0;
     for (StartIndex = InitStack.size() - 1; StartIndex > 0; --StartIndex) {
diff --git a/clang/lib/AST/Interp/Compiler.h b/clang/lib/AST/Interp/Compiler.h
index 6df723df2b444..084f5aef25f8e 100644
--- a/clang/lib/AST/Interp/Compiler.h
+++ b/clang/lib/AST/Interp/Compiler.h
@@ -33,6 +33,7 @@ template <class Emitter> class DestructorScope;
 template <class Emitter> class VariableScope;
 template <class Emitter> class DeclScope;
 template <class Emitter> class InitLinkScope;
+template <class Emitter> class InitStackScope;
 template <class Emitter> class OptionScope;
 template <class Emitter> class ArrayIndexScope;
 template <class Emitter> class SourceLocScope;
@@ -298,6 +299,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   friend class DestructorScope<Emitter>;
   friend class DeclScope<Emitter>;
   friend class InitLinkScope<Emitter>;
+  friend class InitStackScope<Emitter>;
   friend class OptionScope<Emitter>;
   friend class ArrayIndexScope<Emitter>;
   friend class SourceLocScope<Emitter>;
@@ -612,6 +614,20 @@ template <class Emitter> class InitLinkScope final {
   Compiler<Emitter> *Ctx;
 };
 
+template <class Emitter> class InitStackScope final {
+public:
+  InitStackScope(Compiler<Emitter> *Ctx, bool Active)
+      : Ctx(Ctx), OldValue(Ctx->InitStackActive) {
+    Ctx->InitStackActive = Active;
+  }
+
+  ~InitStackScope() { this->Ctx->InitStackActive = OldValue; }
+
+private:
+  Compiler<Emitter> *Ctx;
+  bool OldValue;
+};
+
 } // namespace interp
 } // namespace clang
 

From 47c08fb8d79ec1bf85cc542be6ca2591ebf2d361 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 19 Jul 2024 14:51:27 +0100
Subject: [PATCH 193/486] [TBAA] Remove references to entry BB in check lines
 for tbaa-pointers.c

Follow-up to 123c036bd361d to remove references to entry BB name from
checks, to fix tests for builds that do not generate block names.
---
 clang/test/CodeGen/tbaa-pointers.c | 31 ++++++++++++------------------
 1 file changed, 12 insertions(+), 19 deletions(-)

diff --git a/clang/test/CodeGen/tbaa-pointers.c b/clang/test/CodeGen/tbaa-pointers.c
index f85fca5e34fc0..c036110f45c70 100644
--- a/clang/test/CodeGen/tbaa-pointers.c
+++ b/clang/test/CodeGen/tbaa-pointers.c
@@ -6,15 +6,14 @@
 void p2unsigned(unsigned **ptr) {
   // COMMON-LABEL: define void @p2unsigned(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
-  // COMMON-NEXT:  entry:
-  // COMMON-NEXT:   [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // COMMON:        [[PTR_ADDR:%.+]] = alloca ptr, align 8
   // ENABLED-NEXT:  store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0:!.+]]
   // ENABLED-NEXT:  [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
   // ENABLED-NEXT:  store ptr null, ptr [[BASE]], align 8, !tbaa [[P1INT_0:!.+]]
   // DEFAULT-NEXT:  store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR:!.+]]
   // DEFAULT-NEXT:  [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:  store ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
-  // COMMON-NEXT:  ret void
+  // COMMON-NEXT:   ret void
   //
   *ptr = 0;
 }
@@ -22,15 +21,14 @@ void p2unsigned(unsigned **ptr) {
 void p2unsigned_volatile(unsigned *volatile *ptr) {
   // COMMON-LABEL: define void @p2unsigned_volatile(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
-  // COMMON-NEXT:  entry:
-  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
   // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
   // ENABLED-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
   // ENABLED-NEXT:   store volatile ptr null, ptr [[BASE]], align 8, !tbaa [[P1INT_0]]
   // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:   store volatile ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
-  // COMMON-NEXT:   ret void
+  // COMMON-NEXT:    ret void
   //
   *ptr = 0;
 }
@@ -38,8 +36,7 @@ void p2unsigned_volatile(unsigned *volatile *ptr) {
 void p3int(int ***ptr) {
   // COMMON-LABEL: define void @p3int(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
-  // COMMON-NEXT:  entry:
-  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
   // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P3INT_0:!.+]]
   // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P3INT_0]]
   // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P2INT_0]]
@@ -48,8 +45,7 @@ void p3int(int ***ptr) {
   // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:   store ptr null, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
-
-  // COMMON-NEXT:   ret void
+  // COMMON-NEXT:    ret void
   //
   **ptr = 0;
 }
@@ -57,8 +53,7 @@ void p3int(int ***ptr) {
 void p4char(char ****ptr) {
   // COMMON-LABEL: define void @p4char(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
-  // COMMON-NEXT:  entry:
-  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
   // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0:!.+]]
   // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
   // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0:!.+]]
@@ -69,7 +64,7 @@ void p4char(char ****ptr) {
   // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
-  // COMMON-NEXT:   ret void
+  // COMMON-NEXT:    ret void
   //
   ***ptr = 0;
 }
@@ -77,8 +72,7 @@ void p4char(char ****ptr) {
 void p4char_const1(const char ****ptr) {
   // COMMON-LABEL: define void @p4char_const1(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
-  // COMMON-NEXT:  entry:
-  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
   // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
   // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
   // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0]]
@@ -109,7 +103,7 @@ void p4char_const2(const char **const **ptr) {
   // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
-  // COMMON-NEXT:   ret void
+  // COMMON-NEXT:    ret void
   //
   ***ptr = 0;
 }
@@ -122,15 +116,14 @@ struct S1 {
 void p2struct(struct S1 **ptr) {
   // COMMON-LABEL: define void @p2struct(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
-  // COMMON-NEXT:  entry:
-  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
   // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2S1_0:!.+]]
   // ENABLED-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2S1_0]]
   // ENABLED-NEXT:   store ptr null, ptr [[BASE]], align 8, !tbaa [[P1S1_:!.+]]
   // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
   // DEFAULT-NEXT:   store ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
-  // COMMON-NEXT:   ret void
+  // COMMON-NEXT:    ret void
   //
   *ptr = 0;
 }

From 9145ffa134ed57c25ec62879c1aeff50595d08be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Thu, 18 Jul 2024 17:37:47 +0200
Subject: [PATCH 194/486] [clang][Interp] Only diagnose out of bounds enum
 values in C++

---
 clang/lib/AST/Interp/Compiler.cpp | 2 +-
 clang/test/Sema/switch.c          | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/Interp/Compiler.cpp b/clang/lib/AST/Interp/Compiler.cpp
index 7da711ed485db..ef579bc5d8972 100644
--- a/clang/lib/AST/Interp/Compiler.cpp
+++ b/clang/lib/AST/Interp/Compiler.cpp
@@ -468,7 +468,7 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
 
     // Possibly diagnose casts to enum types if the target type does not
     // have a fixed size.
-    if (CE->getType()->isEnumeralType()) {
+    if (Ctx.getLangOpts().CPlusPlus && CE->getType()->isEnumeralType()) {
       if (const auto *ET = CE->getType().getCanonicalType()->getAs<EnumType>();
           ET && !ET->getDecl()->isFixed()) {
         if (!this->emitCheckEnumValue(*FromT, ET->getDecl(), CE))
diff --git a/clang/test/Sema/switch.c b/clang/test/Sema/switch.c
index 69b34f96820d3..6e912d02d6cc7 100644
--- a/clang/test/Sema/switch.c
+++ b/clang/test/Sema/switch.c
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -Wswitch-enum -Wcovered-switch-default -triple x86_64-linux-gnu %s
+// RUN: %clang_cc1 -fsyntax-only -verify -Wswitch-enum -Wcovered-switch-default -triple x86_64-linux-gnu %s -fexperimental-new-constant-interpreter
 void f (int z) { 
   while (z) { 
     default: z--;            // expected-error {{statement not in switch}}

From 38d0b2d174efe05504a18988299b4d78d37999b7 Mon Sep 17 00:00:00 2001
From: Rafael Ubal <rubal@mathworks.com>
Date: Fri, 19 Jul 2024 10:09:31 -0400
Subject: [PATCH 195/486] [mlir] New canonicalization patterns for
 shape.shape_of and tensor.reshape (#98531)

This PR includes 3 new canonicalization patterns:

- Operation `shape.shape_of`: shape of reshape

```
// Before
func.func @f(%arg0: tensor<*xf32>, %arg1: tensor<?xindex>) -> tensor<?xindex> {
  %reshape = tensor.reshape %arg0(%arg1) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
  %0 = shape.shape_of %reshape : tensor<*xf32> -> tensor<?xindex>
  return %0 : tensor<?xindex>
}

// After
func.func @f(%arg0: tensor<*xf32>, %arg1: tensor<?xindex>) -> tensor<?xindex> {
  return %arg1 : tensor<?xindex>
}
```

- Operation `tensor.reshape`: reshape of reshape

```
// Before
func.func @fold_tensor_reshape(%arg0: tensor<*xf32>, %arg1: tensor<?xindex>, %arg2: tensor<?xindex>) -> tensor<*xf32> {
  %0 = tensor.reshape %arg0(%arg1) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
  %1 = tensor.reshape %0(%arg2) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
  return %1 : tensor<*xf32>
}

// After
func.func @fold_tensor_reshape(%arg0: tensor<*xf32>, %arg1: tensor<?xindex>, %arg2: tensor<?xindex>) -> tensor<*xf32> {
  %reshape = tensor.reshape %arg0(%arg2) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
  return %reshape : tensor<*xf32>
}
```

- Operation `tensor.reshape`: reshape 1D to 1D

```
// Before
func.func @fold_reshape_1d(%input: tensor<?xf32>, %shape: tensor<1xindex>) -> tensor<?xf32> {
  %0 = tensor.reshape %input(%shape) : (tensor<?xf32>, tensor<1xindex>) -> tensor<?xf32>
  return %0 : tensor<?xf32>
}

// After
func.func @fold_reshape_1d(%arg0: tensor<?xf32>, %arg1: tensor<1xindex>) -> tensor<?xf32> {
  return %arg0 : tensor<?xf32>
}
```

These three canonicalization patterns cooperate to simplify the IR
structure emerging from the lowering of certain element-wise ops with
unranked tensor inputs. See file `unranked-tensor-lowering.mlir` in the
proposed change list for a detailed example and description.

For context, this PR is meant to enable code optimizations for the code
generated while lowering ops `quant.qcast` and `quant.dcast` with
unranked tensors, as proposed in
https://discourse.llvm.org/t/rfc-improvements-in-the-quant-dialect/79942
(implementation currently in progress).
---
 mlir/lib/Dialect/Shape/IR/Shape.cpp           | 36 ++++++--
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      | 14 ++-
 mlir/test/Dialect/Shape/canonicalize.mlir     | 39 ++++++++
 .../Shape/unranked-tensor-lowering.mlir       | 90 +++++++++++++++++++
 mlir/test/Dialect/Tensor/canonicalize.mlir    | 27 ++++++
 5 files changed, 196 insertions(+), 10 deletions(-)
 create mode 100644 mlir/test/Dialect/Shape/unranked-tensor-lowering.mlir

diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 58c3f4c334577..8eb8e579954fa 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -1702,18 +1702,36 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
   }
 };
 
-struct ShapeOfWithTensor : public OpRewritePattern<shape::ShapeOfOp> {
+// Canonicalize
+//
+// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
+// %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
+//
+// to
+//
+// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
+// %1 = %shape
+//
+struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
   using OpRewritePattern<shape::ShapeOfOp>::OpRewritePattern;
 
   LogicalResult matchAndRewrite(shape::ShapeOfOp op,
                                 PatternRewriter &rewriter) const override {
-    if (!llvm::isa<ShapedType>(op.getArg().getType()))
-      return failure();
-    if (llvm::isa<ShapedType>(op.getType()))
-      return failure();
-
-    rewriter.replaceOpWithNewOp<shape::ShapeOfOp>(op.getOperation(),
-                                                  op.getArg());
+    auto tensorReshapeOp = op.getArg().getDefiningOp<tensor::ReshapeOp>();
+    if (!tensorReshapeOp)
+      return rewriter.notifyMatchFailure(op, "producer is not tensor.reshape");
+    if (!isa<TensorType>(op.getType()))
+      return rewriter.notifyMatchFailure(op, "result is not a tensor");
+
+    // Operand 'shape' of 'tensor.reshape' may now be used as the result of
+    // 'shape.shape_of'. While its type is guaranteed to be compatible in well-
+    // formed IR, it may not be identical (dynamically vs statically shaped),
+    // in which case it needs to be cast first.
+    Value shape = tensorReshapeOp.getShape();
+    if (op.getType() != shape.getType())
+      shape = rewriter.create<tensor::CastOp>(op.getLoc(), op.getType(), shape);
+
+    rewriter.replaceOp(op, shape);
     return success();
   }
 };
@@ -1753,7 +1771,7 @@ struct ShapeOfCastExtentTensor : public OpRewritePattern<tensor::CastOp> {
 
 void ShapeOfOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
                                             MLIRContext *context) {
-  patterns.add<ShapeOfCastExtentTensor, ShapeOfWithTensor,
+  patterns.add<ShapeOfCastExtentTensor, ShapeOfFromReshape,
                ExtractFromShapeOfExtentTensor, ShapeOfOpToConstShapeOp>(
       context);
 }
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 0e840da9530ed..0751ffc419cbf 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -1585,13 +1585,25 @@ OpFoldResult ReshapeOp::fold(FoldAdaptor adaptor) {
           getResult().getType()))
     return reshapedSource;
 
+  // If the producer of operand 'source' is another 'tensor.reshape' op, use the
+  // producer's input instead as the original tensor to reshape. This could
+  // render such producer dead code.
+  if (auto reshapeOpProducer = getSource().getDefiningOp<ReshapeOp>()) {
+    getSourceMutable().assign(reshapeOpProducer.getSource());
+    return getResult();
+  }
+
   auto source = getSource();
   auto sourceTy = dyn_cast<RankedTensorType>(source.getType());
   auto resultTy = dyn_cast<RankedTensorType>(getType());
-
   if (!sourceTy || !resultTy || sourceTy != resultTy)
     return {};
 
+  // If the source and result are both 1D tensors and have the same type, the
+  // reshape has no effect, even if the tensor is dynamically shaped.
+  if (sourceTy.getRank() == 1)
+    return source;
+
   if (auto fromElements = getShape().getDefiningOp<tensor::FromElementsOp>()) {
     auto elements = fromElements.getElements();
     bool dynamicNoop =
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index 40b137f1fa36e..5b98a7790debf 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -1361,6 +1361,45 @@ func.func @broadcast_as_from_extent_tensor(%a : tensor<?xindex>) -> !shape.shape
 
 // -----
 
+// CHECK-LABEL: func @shape_of_from_reshape
+// CHECK-SAME: %[[INPUT:.*]]: tensor<*xf32>
+// CHECK-SAME: %[[SHAPE:.*]]: tensor<?xindex>
+func.func @shape_of_from_reshape(%arg0: tensor<*xf32>, %arg1: tensor<?xindex>) -> tensor<?xindex> {
+  // CHECK: return %[[SHAPE]] : tensor<?xindex>
+  %0 = tensor.reshape %arg0(%arg1) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
+  %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
+  return %1 : tensor<?xindex>
+}
+
+// -----
+
+// CHECK-LABEL: func @shape_of_from_reshape_compatible_types
+// CHECK-SAME: %[[INPUT:.*]]: tensor<*xf32>
+// CHECK-SAME: %[[SHAPE:.*]]: tensor<5xindex>
+func.func @shape_of_from_reshape_compatible_types(%arg0: tensor<*xf32>, %arg1: tensor<5xindex>) -> tensor<?xindex> {
+  // CHECK: %[[CAST_SHAPE:.*]] = tensor.cast %[[SHAPE]] : tensor<5xindex> to tensor<?xindex>
+  // CHECK: return %[[CAST_SHAPE]] : tensor<?xindex>
+  %0 = tensor.reshape %arg0(%arg1) : (tensor<*xf32>, tensor<5xindex>) -> tensor<*xf32>
+  %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
+  return %1 : tensor<?xindex>
+}
+
+// -----
+
+// CHECK-LABEL: func @shape_of_from_reshape_nofold
+// CHECK-SAME: %[[INPUT:.*]]: tensor<*xf32>
+// CHECK-SAME: %[[SHAPE:.*]]: tensor<?xindex>
+func.func @shape_of_from_reshape_nofold(%arg0: tensor<*xf32>, %arg1: tensor<?xindex>) -> !shape.shape {
+  // CHECK: %[[RESHAPED:.*]] = tensor.reshape %[[INPUT]](%[[SHAPE]]) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
+  // CHECK: %[[SHAPE_OF:.*]] = shape.shape_of %[[RESHAPED]] : tensor<*xf32> -> !shape.shape
+  // CHECK: return %[[SHAPE_OF]] : !shape.shape
+  %0 = tensor.reshape %arg0(%arg1) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
+  %1 = shape.shape_of %0 : tensor<*xf32> -> !shape.shape
+  return %1 : !shape.shape
+}
+
+// -----
+
 // CHECK-LABEL: @cast_extent_tensor
 // CHECK-SAME: (%[[ARG:.*]]: tensor<?x?x?xf32>) -> tensor<?xindex>
 func.func @cast_extent_tensor(%arg : tensor<?x?x?xf32>) -> tensor<?xindex> {
diff --git a/mlir/test/Dialect/Shape/unranked-tensor-lowering.mlir b/mlir/test/Dialect/Shape/unranked-tensor-lowering.mlir
new file mode 100644
index 0000000000000..c1fcbd1ad045f
--- /dev/null
+++ b/mlir/test/Dialect/Shape/unranked-tensor-lowering.mlir
@@ -0,0 +1,90 @@
+// RUN: mlir-opt -split-input-file -canonicalize -cse %s | FileCheck %s
+
+// This test verifies the simplification of IR patterns that emerge when
+// lowering high-level element-wise ops with unranked tensor inputs. Consider
+// the following function incrementing and doubling the value of an input
+// unranked tensor using ops in a hypothetical high-level dialect called 'hl':
+//
+//  func.func @f(%input: tensor<*xf32>) -> tensor<*xf32> {
+//    %0 = hl.inc %input : tensor<*xf32>
+//    %1 = hl.double %0 : tensor<*xf32>
+//    return %1 : tensor<*xf32>
+//  }
+//
+// A possible strategy to lower 'hl.inc' consists in reshaping its operand into
+// a 1D tensor, creating a 1D tensor splat with the same total size as the input
+// operand and with value 1.0, adding both 1D tensors using 'arith.addf', and
+// reshaping the result back into the original input shape. A similar process
+// applies for 'hl.double', except with a tensor splat with value 2.0 and an
+// 'arith.mulf' op. The body of the function in the test below contains the full
+// sequence.
+//
+// Since such lowering process would operate on individual 'hl' ops in a
+// context-oblivious manner, the emitted code produces a redundant IR pattern
+// where the result of 'arith.addf' is reshaped into an unranked tensor, just
+// for it to be immediately reshaped back into the 1D tensor consumed by
+// 'arith.mulf'. This entails the overhead of re-computing the unranked tensor
+// shape ('shape.shape_of') and size ('shape.num_elements').
+//
+// This test verifies that the consecutive application of a canonicalization and
+// a CSE pass successfully simplifies this emerging pattern, leading to a
+// version of the code in which the result of the emitted 'arith.addf' op
+// associated with 'hl.inc' is directly consumed by the 'arith.mulf' op
+// associated with 'hl.double', as observed in the FileCheck directives. The
+// main rewrite patterns at play are 'shape.shape_of' canonicalization,
+// 'tensor.reshape' canonicalization, and 'shape.num_elements' subexpression
+// elimination.
+//
+
+// CHECK-LABEL: @unranked_tensor_lowering
+// CHECK-SAME: %[[INPUT:.*]]: tensor<*xf32>
+
+// CHECK-DAG: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
+// CHECK-DAG: %[[TWO:.*]] = arith.constant 2.000000e+00 : f32
+
+// CHECK: %[[INPUT_SHAPE:.*]] = shape.shape_of %[[INPUT]] : tensor<*xf32> -> tensor<?xindex>
+// CHECK: %[[INPUT_SIZE:.*]] = shape.num_elements %[[INPUT_SHAPE]] : tensor<?xindex> -> index
+// CHECK: %[[INPUT_COLLAPSED_SHAPE:.*]] = tensor.from_elements %[[INPUT_SIZE]] : tensor<1xindex>
+// CHECK: %[[INPUT_COLLAPSED:.*]] = tensor.reshape %[[INPUT]](%[[INPUT_COLLAPSED_SHAPE]]) : (tensor<*xf32>, tensor<1xindex>) -> tensor<?xf32>
+
+// CHECK: %[[ONE_SPLAT:.*]] = tensor.splat %[[ONE]]{{\[}}%[[INPUT_SIZE]]] : tensor<?xf32>
+// CHECK: %[[SUM_COLLAPSED:.*]] = arith.addf %[[INPUT_COLLAPSED]], %[[ONE_SPLAT]] : tensor<?xf32>
+
+// CHECK: %[[TWO_SPLAT:.*]] = tensor.splat %[[TWO]]{{\[}}%[[INPUT_SIZE]]] : tensor<?xf32>
+// CHECK: %[[PRODUCT_COLLAPSED:.*]] = arith.mulf %[[SUM_COLLAPSED]], %[[TWO_SPLAT]] : tensor<?xf32>
+
+// CHECK: %[[PRODUCT:.*]] = tensor.reshape %[[PRODUCT_COLLAPSED]](%[[INPUT_SHAPE]]) : (tensor<?xf32>, tensor<?xindex>) -> tensor<*xf32>
+// CHECK: return %[[PRODUCT]] : tensor<*xf32>
+
+func.func @unranked_tensor_lowering(%input: tensor<*xf32>) -> tensor<*xf32> {
+
+  // Collapse input
+  %input_shape = shape.shape_of %input : tensor<*xf32> -> tensor<?xindex>
+  %input_size = shape.num_elements %input_shape : tensor<?xindex> -> index
+  %input_collapsed_shape = tensor.from_elements %input_size : tensor<1xindex>
+  %input_collapsed = tensor.reshape %input(%input_collapsed_shape) : (tensor<*xf32>, tensor<1xindex>) -> tensor<?xf32>
+
+  // Second operand for sum
+  %one = arith.constant 1.0 : f32
+  %one_splat = tensor.splat %one[%input_size] : tensor<?xf32>
+
+  // Compute sum and expand it
+  %sum_collapsed = arith.addf %input_collapsed, %one_splat : tensor<?xf32>
+  %sum = tensor.reshape %sum_collapsed(%input_shape) : (tensor<?xf32>, tensor<?xindex>) -> tensor<*xf32>
+
+  // Collapse sum
+  %sum_shape = shape.shape_of %sum : tensor<*xf32> -> tensor<?xindex>
+  %sum_size = shape.num_elements %sum_shape : tensor<?xindex> -> index
+  %sum_collapsed_shape = tensor.from_elements %sum_size : tensor<1xindex>
+  %sum_collapsed_0 = tensor.reshape %sum(%sum_collapsed_shape) : (tensor<*xf32>, tensor<1xindex>) -> tensor<?xf32>
+
+  // Second operand for product
+  %two = arith.constant 2.0 : f32
+  %two_splat = tensor.splat %two[%sum_size] : tensor<?xf32>
+
+  // Compute product and expand it
+  %product_collapsed = arith.mulf %sum_collapsed_0, %two_splat : tensor<?xf32>
+  %product = tensor.reshape %product_collapsed(%sum_shape) : (tensor<?xf32>, tensor<?xindex>) -> tensor<*xf32>
+
+  return %product : tensor<*xf32>
+}
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index baa205b9f42c6..4b8efde78cc23 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -847,6 +847,33 @@ func.func @fold_reshape_constant_splat(%shape : tensor<1xi32>) -> tensor<4xf32>
 
 // -----
 
+// CHECK-LABEL: func @fold_reshape_chain
+//  CHECK-SAME: %[[INPUT:[a-zA-Z0-9_]+]]: tensor<*xf32>
+//  CHECK-SAME: %[[SHAPE_0:[a-zA-Z0-9_]+]]: tensor<?xindex>
+//  CHECK-SAME: %[[SHAPE_1:[a-zA-Z0-9_]+]]: tensor<?xindex>
+//  CHECK-SAME: %[[SHAPE_2:[a-zA-Z0-9_]+]]: tensor<?xindex>
+//       CHECK: %[[RESULT:.*]] = tensor.reshape %[[INPUT]](%[[SHAPE_2]])
+//       CHECK: return %[[RESULT]]
+func.func @fold_reshape_chain(%input: tensor<*xf32>, %shape_0: tensor<?xindex>, %shape_1: tensor<?xindex>, %shape_2: tensor<?xindex>) -> tensor<*xf32> {
+  %0 = tensor.reshape %input(%shape_0) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
+  %1 = tensor.reshape %0(%shape_1) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
+  %2 = tensor.reshape %1(%shape_2) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
+  return %2 : tensor<*xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @fold_reshape_1d
+//  CHECK-SAME: %[[INPUT:[a-zA-Z0-9_]+]]: tensor<?xf32>
+//  CHECK-SAME: %[[SHAPE:[a-zA-Z0-9_]+]]: tensor<1xindex>
+//       CHECK: return %[[INPUT]]
+func.func @fold_reshape_1d(%input: tensor<?xf32>, %shape: tensor<1xindex>) -> tensor<?xf32> {
+  %0 = tensor.reshape %input(%shape) : (tensor<?xf32>, tensor<1xindex>) -> tensor<?xf32>
+  return %0 : tensor<?xf32>
+}
+
+// -----
+
 // CHECK-LABEL: func @fold_extract_constant_splat
 //   CHECK-NOT: tensor.extract_slice
 //       CHECK: arith.constant dense<42> : tensor<4x4xi32>

From 4c28494e7842b26a566f9406f71510eda4a9e576 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Fri, 19 Jul 2024 06:47:50 -0700
Subject: [PATCH 196/486] [SLP][NFC]Add a test for incorrect minbitwidth
 analysis for trunc'ed bv, NFC.

---
 .../RISCV/trunc-bv-multi-uses.ll              | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/trunc-bv-multi-uses.ll

diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-bv-multi-uses.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-bv-multi-uses.ll
new file mode 100644
index 0000000000000..53acc37155c9f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-bv-multi-uses.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S --passes=slp-vectorizer -mtriple=riscv64-unknown-linux-gnu -mattr="+v" < %s -slp-threshold=-10 | FileCheck %s
+
+define i32 @test(i64 %v1, i64 %v2) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i64 [[V1:%.*]], i64 [[V2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[V1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i64> [[TMP0]], i64 [[V2]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr <2 x i64> [[TMP3]], <i64 32, i64 32>
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc <2 x i64> [[TMP4]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[TMP6]], i32 1
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+entry:
+  %t1 = trunc i64 %v1 to i32
+  %t2 = trunc i64 %v2 to i32
+  %lshr1 = lshr i64 %v1, 32
+  %lshr2 = lshr i64 %v2, 32
+  %t3 = trunc i64 %lshr1 to i32
+  %t4 = trunc i64 %lshr2 to i32
+  %add1 = add i32 %t1, %t3
+  %add2 = add i32 %t2, %t4
+  %mul = mul i32 %add1, %add2
+  ret i32 %mul
+}

From c719d7b390481b095c7498fe75a0bcf60a24bad9 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas@arm.com>
Date: Fri, 19 Jul 2024 15:17:06 +0100
Subject: [PATCH 197/486] [FMV][AArch64] Do not optimize away runtime checks
 for implied features (#99522)

When generating the body of the ifunc resolver, clang skips runtime
checks for features that are implied from the command line. We bend this
rule for certain features (memtag, bti, dgh), but this happens quite
arbitrarily in my opinion. The reasoning is that some features are in
the HINT instruction space, meaning they operate as NOPs if the hardware
does not support them. Still the user wants to detect their presence
with runtime checks. See #90928 for details.

I think we should always perform runtime checks regardless of the
feature and then try to statically resolve calls whenever a function is
compiled with a sufficiently high set of architecture features (so
including target/target_version/target_clones attributes, and command
line options). This is what GCC does. We have an open PR in LLVM
GlobalOpt since it was suggested not to perform such codegen
optimizations in clang anyway. See #87939.
---
 clang/lib/CodeGen/CodeGenFunction.cpp         | 12 +--
 .../test/CodeGen/attr-target-clones-aarch64.c | 16 ++++
 clang/test/CodeGen/attr-target-version.c      | 74 +++++++++----------
 3 files changed, 55 insertions(+), 47 deletions(-)

diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 551db09165dbe..1e98bea8c8ce3 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -2797,16 +2797,8 @@ void CodeGenFunction::EmitKCFIOperandBundle(
 llvm::Value *CodeGenFunction::FormAArch64ResolverCondition(
     const MultiVersionResolverOption &RO) {
   llvm::SmallVector<StringRef, 8> CondFeatures;
-  for (const StringRef &Feature : RO.Conditions.Features) {
-    // Optimize the Function Multi Versioning resolver by creating conditions
-    // only for features that are not enabled in the target. The exception is
-    // for features whose extension instructions are executed as NOP on targets
-    // without extension support.
-    if (!getContext().getTargetInfo().hasFeature(Feature) || Feature == "bti" ||
-        Feature == "memtag" || Feature == "memtag2" || Feature == "memtag3" ||
-        Feature == "dgh")
-      CondFeatures.push_back(Feature);
-  }
+  for (const StringRef &Feature : RO.Conditions.Features)
+    CondFeatures.push_back(Feature);
   if (!CondFeatures.empty()) {
     return EmitAArch64CpuSupports(CondFeatures);
   }
diff --git a/clang/test/CodeGen/attr-target-clones-aarch64.c b/clang/test/CodeGen/attr-target-clones-aarch64.c
index 846b08298cc72..274e05de594b8 100644
--- a/clang/test/CodeGen/attr-target-clones-aarch64.c
+++ b/clang/test/CodeGen/attr-target-clones-aarch64.c
@@ -157,7 +157,15 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK:       resolver_return:
 // CHECK-NEXT:    ret ptr @ftc_dup2._McrcMdotprod
 // CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 256
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 256
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
 // CHECK-NEXT:    ret ptr @ftc_dup2._Mfp
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    ret ptr @ftc_dup2.default
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
@@ -571,7 +579,15 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-MTE-BTI:       resolver_return:
 // CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup2._McrcMdotprod
 // CHECK-MTE-BTI:       resolver_else:
+// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 256
+// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 256
+// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK-MTE-BTI:       resolver_return1:
 // CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup2._Mfp
+// CHECK-MTE-BTI:       resolver_else2:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup2.default
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c
index fbe34a51b40b6..b058f84f78baa 100644
--- a/clang/test/CodeGen/attr-target-version.c
+++ b/clang/test/CodeGen/attr-target-version.c
@@ -1,5 +1,5 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-attributes --check-globals --include-generated-funcs
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -v9.5a -target-feature -fp-armv8 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fmv -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-NOFMV
 
 int __attribute__((target_version("rng+flagm+fp16fml"))) fmv(void) { return 1; }
@@ -1112,42 +1112,42 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 // CHECK-NOFMV-NEXT:    ret i32 1
 //
 //.
-// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp-armv8,+fp16fml,+fullfp16,+neon,+rand,-v9.5a" }
-// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon,+sha2,-v9.5a" }
-// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+ls64,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fp16fml,+fullfp16,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR12:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
-// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm,-v9.5a" }
-// CHECK: attributes #[[ATTR18:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR19]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
-// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fp-armv8,+fullfp16,+neon,+rdm,+sme,-v9.5a" }
-// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fp-armv8,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-v9.5a" }
-// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
-// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+fp-armv8,+jsconv,+neon,-v9.5a" }
-// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
-// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-v9.5a" }
-// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-v9.5a" }
-// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-v9.5a" }
-// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+sm4,-v9.5a" }
-// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon,+rdm,-v9.5a" }
-// CHECK: attributes #[[ATTR35:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm,-v9.5a" }
+// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp-armv8,+fp16fml,+fullfp16,+neon,+rand" }
+// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64" }
+// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon,+sha2" }
+// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+ls64,+neon" }
+// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fp16fml,+fullfp16,+neon" }
+// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" }
+// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" }
+// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti" }
+// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2" }
+// CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp" }
+// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" }
+// CHECK: attributes #[[ATTR12:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" }
+// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" }
+// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" }
+// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" }
+// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm" }
+// CHECK: attributes #[[ATTR18:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon" }
+// CHECK: attributes #[[ATTR19]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon" }
+// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb" }
+// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fp-armv8,+fullfp16,+neon,+sve" }
+// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fp-armv8,+fullfp16,+neon,+rdm,+sme" }
+// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fp-armv8,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve" }
+// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fp-armv8,+fullfp16,+neon,+sve" }
+// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc" }
+// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+fp-armv8,+jsconv,+neon" }
+// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc" }
+// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+sve" }
+// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3" }
+// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm" }
+// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4" }
+// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3" }
+// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+sm4" }
+// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon,+rdm" }
+// CHECK: attributes #[[ATTR35:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm" }
 //.
 // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" }
 // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" }

From e2f463b5b64a3574e90be7375b2d2c87fa7e92c1 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Fri, 19 Jul 2024 10:18:23 -0400
Subject: [PATCH 198/486] [aarch64] Add hyperbolic and arc trig intrinsic
 lowering (#98937)

## The change(s)
- `VecFuncs.def`: define intrinsic to  sleef/armpl mapping
- `LegalizerHelper.cpp`: add missing `fewerElementsVector` handling for
the new trig intrinsics
- `AArch64ISelLowering.cpp`: Add arch64 specializations for lowering
like neon instructions
- `AArch64LegalizerInfo.cpp`: Legalize the new trig intrinsics. aarch64
has specail legalization requirments in `AArch64LegalizerInfo.cpp`. If
we redirect the clang builtin without handling this we will break the
aarch64 compiler

## History
This change is part of an implementation of
https://github.com/llvm/llvm-project/issues/87367's investigation on
supporting IEEE math operations as intrinsics.
Which was discussed in this RFC:
https://discourse.llvm.org/t/rfc-all-the-math-intrinsics/78294

This change adds wasm lowering cases for `acos`, `asin`, `atan`, `cosh`,
`sinh`, and `tanh`.

https://github.com/llvm/llvm-project/issues/70079
https://github.com/llvm/llvm-project/issues/70080
https://github.com/llvm/llvm-project/issues/70081
https://github.com/llvm/llvm-project/issues/70083
https://github.com/llvm/llvm-project/issues/70084
https://github.com/llvm/llvm-project/issues/95966

## Why is aarch64 needed
The last step is to redirect the `acos`, `asin`, `atan`, `cosh`, `sinh`,
and `tanh` to emit the intrinsic. We can't emit the intrinsic without
the intrinsics becoming legal for aarch64 in `AArch64LegalizerInfo.cpp`
---
 llvm/include/llvm/Analysis/VecFuncs.def       |  66 +++
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |   6 +
 .../Target/AArch64/AArch64ISelLowering.cpp    |  18 +
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   5 +-
 .../AArch64/GlobalISel/arm64-irtranslator.ll  |  48 ++
 .../AArch64/GlobalISel/legalize-acos.mir      | 227 +++++++++
 .../AArch64/GlobalISel/legalize-asin.mir      | 227 +++++++++
 .../AArch64/GlobalISel/legalize-atan.mir      | 227 +++++++++
 .../AArch64/GlobalISel/legalize-cosh.mir      | 227 +++++++++
 .../AArch64/GlobalISel/legalize-sinh.mir      | 227 +++++++++
 .../AArch64/GlobalISel/legalize-tanh.mir      | 227 +++++++++
 .../GlobalISel/legalizer-info-validation.mir  |  30 +-
 llvm/test/CodeGen/AArch64/f16-instructions.ll | 156 ++++++
 .../CodeGen/AArch64/fp-intrinsics-fp16.ll     |  90 ++++
 llvm/test/CodeGen/AArch64/fp-intrinsics.ll    | 186 ++++++++
 .../test/CodeGen/AArch64/illegal-float-ops.ll | 125 +++++
 .../AArch64/replace-with-veclib-armpl.ll      | 273 ++++++++++-
 .../replace-with-veclib-sleef-scalable.ll     | 110 ++++-
 .../AArch64/replace-with-veclib-sleef.ll      | 110 ++++-
 llvm/test/CodeGen/AArch64/vec-libcalls.ll     | 191 ++++++++
 .../AArch64/veclib-calls-libsystem-darwin.ll  | 289 +++++++++++
 .../AArch64/veclib-intrinsic-calls.ll         | 448 +++++++++++++++++-
 22 files changed, 3491 insertions(+), 22 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-acos.mir
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-asin.mir
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-atan.mir
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-cosh.mir
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-sinh.mir
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-tanh.mir

diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def
index ffdf8b8c3bc79..444cef613fb00 100644
--- a/llvm/include/llvm/Analysis/VecFuncs.def
+++ b/llvm/include/llvm/Analysis/VecFuncs.def
@@ -73,12 +73,18 @@ TLI_DEFINE_VECFUNC("llvm.exp.f32", "_simd_exp_f4", FIXED(4), "_ZGV_LLVM_N4v")
 
 // Trigonometric Functions
 TLI_DEFINE_VECFUNC("acos", "_simd_acos_d2", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.acos.f64", "_simd_acos_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("acosf", "_simd_acos_f4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.acos.f32", "_simd_acos_f4", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("asin", "_simd_asin_d2", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.asin.f64", "_simd_asin_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("asinf", "_simd_asin_f4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.asin.f32", "_simd_asin_f4", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("atan", "_simd_atan_d2", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.atan.f64", "_simd_atan_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("atanf", "_simd_atan_f4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.atan.f32", "_simd_atan_f4", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("atan2", "_simd_atan2_d2", FIXED(2), "_ZGV_LLVM_N2vv")
 TLI_DEFINE_VECFUNC("atan2f", "_simd_atan2_f4", FIXED(4), "_ZGV_LLVM_N4vv")
 
@@ -109,11 +115,17 @@ TLI_DEFINE_VECFUNC("llvm.pow.f32", "_simd_pow_f4", FIXED(4), "_ZGV_LLVM_N4vv")
 
 // Hyperbolic Functions
 TLI_DEFINE_VECFUNC("sinh", "_simd_sinh_d2", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.sinh.f64", "_simd_sinh_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("sinhf", "_simd_sinh_f4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.sinh.f32", "_simd_sinh_f4", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("cosh", "_simd_cosh_d2", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.cosh.f64", "_simd_cosh_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("coshf", "_simd_cosh_f4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.cosh.f32", "_simd_cosh_f4", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("tanh", "_simd_tanh_d2", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.tanh.f64", "_simd_tanh_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("tanhf", "_simd_tanh_f4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tanh.f32", "_simd_tanh_f4", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("asinh", "_simd_asinh_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("asinhf", "_simd_asinh_f4", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("acosh", "_simd_acosh_d2", FIXED(2), "_ZGV_LLVM_N2v")
@@ -500,14 +512,17 @@ TLI_DEFINE_VECFUNC("__exp2f_finite", "__svml_exp2f16", FIXED(16), "_ZGV_LLVM_N16
 #elif defined(TLI_DEFINE_SLEEFGNUABI_VF2_VECFUNCS)
 
 TLI_DEFINE_VECFUNC("acos", "_ZGVnN2v_acos", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.acos.f64", "_ZGVnN2v_acos", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("acosh", "_ZGVnN2v_acosh", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("asin", "_ZGVnN2v_asin", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.asin.f64", "_ZGVnN2v_asin", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("asinh", "_ZGVnN2v_asinh", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("atan", "_ZGVnN2v_atan", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.atan.f64", "_ZGVnN2v_atan", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("atan2", "_ZGVnN2vv_atan2", FIXED(2), "_ZGV_LLVM_N2vv")
 
@@ -521,6 +536,7 @@ TLI_DEFINE_VECFUNC("cos", "_ZGVnN2v_cos", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("llvm.cos.f64", "_ZGVnN2v_cos", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("cosh", "_ZGVnN2v_cosh", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.cosh.f64", "_ZGVnN2v_cosh", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("cospi", "_ZGVnN2v_cospi", FIXED(2), "_ZGV_LLVM_N2v")
 
@@ -583,6 +599,7 @@ TLI_DEFINE_VECFUNC("sincos", "_ZGVnN2vl8l8_sincos", FIXED(2), "_ZGV_LLVM_N2vl8l8
 TLI_DEFINE_VECFUNC("sincospi", "_ZGVnN2vl8l8_sincospi", FIXED(2), "_ZGV_LLVM_N2vl8l8")
 
 TLI_DEFINE_VECFUNC("sinh", "_ZGVnN2v_sinh", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.sinh.f64", "_ZGVnN2v_sinh", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("sinpi", "_ZGVnN2v_sinpi", FIXED(2), "_ZGV_LLVM_N2v")
 
@@ -592,20 +609,24 @@ TLI_DEFINE_VECFUNC("tan", "_ZGVnN2v_tan", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("llvm.tan.f64", "_ZGVnN2v_tan", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("tanh", "_ZGVnN2v_tanh", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.tanh.f64", "_ZGVnN2v_tanh", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("tgamma", "_ZGVnN2v_tgamma", FIXED(2), "_ZGV_LLVM_N2v")
 
 #elif defined(TLI_DEFINE_SLEEFGNUABI_VF4_VECFUNCS)
 
 TLI_DEFINE_VECFUNC("acosf", "_ZGVnN4v_acosf", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.acos.f32", "_ZGVnN4v_acosf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("acoshf", "_ZGVnN4v_acoshf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("asinf", "_ZGVnN4v_asinf", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.asin.f32", "_ZGVnN4v_asinf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("asinhf", "_ZGVnN4v_asinhf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("atanf", "_ZGVnN4v_atanf", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.atan.f32", "_ZGVnN4v_atanf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("atan2f", "_ZGVnN4vv_atan2f", FIXED(4), "_ZGV_LLVM_N4vv")
 
@@ -619,6 +640,7 @@ TLI_DEFINE_VECFUNC("cosf", "_ZGVnN4v_cosf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("llvm.cos.f32", "_ZGVnN4v_cosf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("coshf", "_ZGVnN4v_coshf", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.cosh.f32", "_ZGVnN4v_coshf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("cospif", "_ZGVnN4v_cospif", FIXED(4), "_ZGV_LLVM_N4v")
 
@@ -681,6 +703,7 @@ TLI_DEFINE_VECFUNC("sincosf", "_ZGVnN4vl4l4_sincosf", FIXED(4), "_ZGV_LLVM_N4vl4
 TLI_DEFINE_VECFUNC("sincospif", "_ZGVnN4vl4l4_sincospif", FIXED(4), "_ZGV_LLVM_N4vl4l4")
 
 TLI_DEFINE_VECFUNC("sinhf", "_ZGVnN4v_sinhf", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.sinh.f32", "_ZGVnN4v_sinhf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("sinpif", "_ZGVnN4v_sinpif", FIXED(4), "_ZGV_LLVM_N4v")
 
@@ -690,6 +713,7 @@ TLI_DEFINE_VECFUNC("tanf", "_ZGVnN4v_tanf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("llvm.tan.f32", "_ZGVnN4v_tanf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("tanhf", "_ZGVnN4v_tanhf", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tanh.f32", "_ZGVnN4v_tanhf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("tgammaf", "_ZGVnN4v_tgammaf", FIXED(4), "_ZGV_LLVM_N4v")
 
@@ -697,18 +721,24 @@ TLI_DEFINE_VECFUNC("tgammaf", "_ZGVnN4v_tgammaf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("acos", "_ZGVsMxv_acos",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("acosf", "_ZGVsMxv_acosf", SCALABLE(4), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.acos.f64", "_ZGVsMxv_acos", SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.acos.f32", "_ZGVsMxv_acosf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
 TLI_DEFINE_VECFUNC("acosh", "_ZGVsMxv_acosh",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("acoshf", "_ZGVsMxv_acoshf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
 TLI_DEFINE_VECFUNC("asin", "_ZGVsMxv_asin",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("asinf", "_ZGVsMxv_asinf", SCALABLE(4), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.asin.f64", "_ZGVsMxv_asin", SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.asin.f32", "_ZGVsMxv_asinf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
 TLI_DEFINE_VECFUNC("asinh", "_ZGVsMxv_asinh",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("asinhf", "_ZGVsMxv_asinhf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
 TLI_DEFINE_VECFUNC("atan", "_ZGVsMxv_atan",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("atanf", "_ZGVsMxv_atanf", SCALABLE(4), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.atan.f64", "_ZGVsMxv_atan", SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.atan.f32", "_ZGVsMxv_atanf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
 TLI_DEFINE_VECFUNC("atan2", "_ZGVsMxvv_atan2",  SCALABLE(2), MASKED, "_ZGVsMxvv")
 TLI_DEFINE_VECFUNC("atan2f", "_ZGVsMxvv_atan2f", SCALABLE(4), MASKED, "_ZGVsMxvv")
@@ -729,6 +759,8 @@ TLI_DEFINE_VECFUNC("llvm.cos.f32", "_ZGVsMxv_cosf", SCALABLE(4), MASKED, "_ZGVsM
 
 TLI_DEFINE_VECFUNC("cosh", "_ZGVsMxv_cosh",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("coshf", "_ZGVsMxv_coshf", SCALABLE(4), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.cosh.f64", "_ZGVsMxv_cosh", SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.cosh.f32", "_ZGVsMxv_coshf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
 TLI_DEFINE_VECFUNC("cospi", "_ZGVsMxv_cospi",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("cospif", "_ZGVsMxv_cospif", SCALABLE(4), MASKED, "_ZGVsMxv")
@@ -826,6 +858,8 @@ TLI_DEFINE_VECFUNC("sincospif", "_ZGVsNxvl4l4_sincospif", SCALABLE(4), NOMASK, "
 
 TLI_DEFINE_VECFUNC("sinh", "_ZGVsMxv_sinh",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("sinhf", "_ZGVsMxv_sinhf", SCALABLE(4), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.sinh.f64", "_ZGVsMxv_sinh", SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.sinh.f32", "_ZGVsMxv_sinhf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
 TLI_DEFINE_VECFUNC("sinpi", "_ZGVsMxv_sinpi",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("sinpif", "_ZGVsMxv_sinpif", SCALABLE(4), MASKED, "_ZGVsMxv")
@@ -840,6 +874,8 @@ TLI_DEFINE_VECFUNC("llvm.tan.f32", "_ZGVsMxv_tanf", SCALABLE(4), MASKED, "_ZGVsM
 
 TLI_DEFINE_VECFUNC("tanh", "_ZGVsMxv_tanh",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("tanhf", "_ZGVsMxv_tanhf", SCALABLE(4), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.tanh.f64", "_ZGVsMxv_tanh", SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.tanh.f32", "_ZGVsMxv_tanhf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
 TLI_DEFINE_VECFUNC("tgamma", "_ZGVsMxv_tgamma",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("tgammaf", "_ZGVsMxv_tgammaf", SCALABLE(4), MASKED, "_ZGVsMxv")
@@ -851,6 +887,11 @@ TLI_DEFINE_VECFUNC("acosf", "armpl_vacosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v
 TLI_DEFINE_VECFUNC("acos", "armpl_svacos_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("acosf", "armpl_svacos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("llvm.acos.f64", "armpl_vacosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.acos.f32", "armpl_vacosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.acos.f64", "armpl_svacos_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.acos.f32", "armpl_svacos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
+
 TLI_DEFINE_VECFUNC("acosh", "armpl_vacoshq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("acoshf", "armpl_vacoshq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("acosh", "armpl_svacosh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
@@ -861,6 +902,11 @@ TLI_DEFINE_VECFUNC("asinf", "armpl_vasinq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v
 TLI_DEFINE_VECFUNC("asin", "armpl_svasin_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("asinf", "armpl_svasin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("llvm.asin.f64", "armpl_vasinq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.asin.f32", "armpl_vasinq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.asin.f64", "armpl_svasin_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.asin.f32", "armpl_svasin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
+
 TLI_DEFINE_VECFUNC("asinh", "armpl_vasinhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("asinhf", "armpl_vasinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("asinh", "armpl_svasinh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
@@ -871,6 +917,11 @@ TLI_DEFINE_VECFUNC("atanf", "armpl_vatanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v
 TLI_DEFINE_VECFUNC("atan", "armpl_svatan_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("atanf", "armpl_svatan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("llvm.atan.f64", "armpl_vatanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.atan.f32", "armpl_vatanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.atan.f64", "armpl_svatan_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.atan.f32", "armpl_svatan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
+
 TLI_DEFINE_VECFUNC("atan2", "armpl_vatan2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
 TLI_DEFINE_VECFUNC("atan2f", "armpl_vatan2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
 TLI_DEFINE_VECFUNC("atan2", "armpl_svatan2_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
@@ -906,6 +957,11 @@ TLI_DEFINE_VECFUNC("coshf", "armpl_vcoshq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v
 TLI_DEFINE_VECFUNC("cosh", "armpl_svcosh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("coshf", "armpl_svcosh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("llvm.cosh.f64", "armpl_vcoshq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.cosh.f32", "armpl_vcoshq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.cosh.f64", "armpl_svcosh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.cosh.f32", "armpl_svcosh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
+
 TLI_DEFINE_VECFUNC("cospi", "armpl_vcospiq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("cospif", "armpl_vcospiq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("cospi", "armpl_svcospi_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
@@ -1081,6 +1137,11 @@ TLI_DEFINE_VECFUNC("sinhf", "armpl_vsinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v
 TLI_DEFINE_VECFUNC("sinh", "armpl_svsinh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("sinhf", "armpl_svsinh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("llvm.sinh.f64", "armpl_vsinhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.sinh.f32", "armpl_vsinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.sinh.f64", "armpl_svsinh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.sinh.f32", "armpl_svsinh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
+
 TLI_DEFINE_VECFUNC("sinpi", "armpl_vsinpiq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("sinpif", "armpl_vsinpiq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("sinpi", "armpl_svsinpi_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
@@ -1106,6 +1167,11 @@ TLI_DEFINE_VECFUNC("tanhf", "armpl_vtanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v
 TLI_DEFINE_VECFUNC("tanh", "armpl_svtanh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("tanhf", "armpl_svtanh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("llvm.tanh.f64", "armpl_vtanhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.tanh.f32", "armpl_vtanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tanh.f64", "armpl_svtanh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.tanh.f32", "armpl_svtanh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
+
 TLI_DEFINE_VECFUNC("tgamma", "armpl_vtgammaq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("tgammaf", "armpl_vtgammaq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("tgamma", "armpl_svtgamma_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index bcc30390bc82e..640a425ffa735 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -4747,6 +4747,12 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_FCOS:
   case G_FSIN:
   case G_FTAN:
+  case G_FACOS:
+  case G_FASIN:
+  case G_FATAN:
+  case G_FCOSH:
+  case G_FSINH:
+  case G_FTANH:
   case G_FSQRT:
   case G_BSWAP:
   case G_BITREVERSE:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 668a9acd1f3c6..84de1ee8f8923 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -732,10 +732,14 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
   for (auto Op : {ISD::FREM,         ISD::FPOW,          ISD::FPOWI,
                   ISD::FCOS,         ISD::FSIN,          ISD::FSINCOS,
+                  ISD::FACOS,        ISD::FASIN,         ISD::FATAN,
+                  ISD::FCOSH,        ISD::FSINH,         ISD::FTANH,
                   ISD::FTAN,         ISD::FEXP,          ISD::FEXP2,
                   ISD::FEXP10,       ISD::FLOG,          ISD::FLOG2,
                   ISD::FLOG10,       ISD::STRICT_FREM,   ISD::STRICT_FPOW,
                   ISD::STRICT_FPOWI, ISD::STRICT_FCOS,   ISD::STRICT_FSIN,
+                  ISD::STRICT_FACOS, ISD::STRICT_FASIN,  ISD::STRICT_FATAN,
+                  ISD::STRICT_FCOSH, ISD::STRICT_FSINH,  ISD::STRICT_FTANH,
                   ISD::STRICT_FEXP,  ISD::STRICT_FEXP2,  ISD::STRICT_FLOG,
                   ISD::STRICT_FLOG2, ISD::STRICT_FLOG10, ISD::STRICT_FTAN}) {
     setOperationAction(Op, MVT::f16, Promote);
@@ -1176,6 +1180,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
           ISD::FNEG,              ISD::FABS,           ISD::FCEIL,
           ISD::FSQRT,             ISD::FFLOOR,         ISD::FNEARBYINT,
           ISD::FSIN,              ISD::FCOS,           ISD::FTAN,
+          ISD::FASIN,             ISD::FACOS,          ISD::FATAN,
+          ISD::FSINH,             ISD::FCOSH,          ISD::FTANH,
           ISD::FPOW,              ISD::FLOG,           ISD::FLOG2,          
           ISD::FLOG10,            ISD::FEXP,           ISD::FEXP2,
           ISD::FEXP10,            ISD::FRINT,          ISD::FROUND,
@@ -1615,6 +1621,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::FSIN, VT, Expand);
       setOperationAction(ISD::FSINCOS, VT, Expand);
       setOperationAction(ISD::FTAN, VT, Expand);
+      setOperationAction(ISD::FACOS, VT, Expand);
+      setOperationAction(ISD::FASIN, VT, Expand);
+      setOperationAction(ISD::FATAN, VT, Expand);
+      setOperationAction(ISD::FCOSH, VT, Expand);
+      setOperationAction(ISD::FSINH, VT, Expand);
+      setOperationAction(ISD::FTANH, VT, Expand);
       setOperationAction(ISD::FEXP, VT, Expand);
       setOperationAction(ISD::FEXP2, VT, Expand);
       setOperationAction(ISD::FEXP10, VT, Expand);
@@ -1822,6 +1834,12 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
     setOperationAction(ISD::FSIN, VT, Expand);
     setOperationAction(ISD::FCOS, VT, Expand);
     setOperationAction(ISD::FTAN, VT, Expand);
+    setOperationAction(ISD::FASIN, VT, Expand);
+    setOperationAction(ISD::FACOS, VT, Expand);
+    setOperationAction(ISD::FATAN, VT, Expand);
+    setOperationAction(ISD::FSINH, VT, Expand);
+    setOperationAction(ISD::FCOSH, VT, Expand);
+    setOperationAction(ISD::FTANH, VT, Expand);
     setOperationAction(ISD::FPOW, VT, Expand);
     setOperationAction(ISD::FLOG, VT, Expand);
     setOperationAction(ISD::FLOG2, VT, Expand);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index a73c971020bd8..3f8641945bcd7 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -267,8 +267,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .libcallFor({{s64, s128}})
       .minScalarOrElt(1, MinFPScalar);
 
-  getActionDefinitionsBuilder({G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2,
-                               G_FLOG10, G_FTAN, G_FEXP, G_FEXP2, G_FEXP10})
+  getActionDefinitionsBuilder(
+      {G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2, G_FLOG10, G_FTAN, G_FEXP,
+       G_FEXP2, G_FEXP10, G_FACOS, G_FASIN, G_FATAN, G_FCOSH, G_FSINH, G_FTANH})
       // We need a call for these, so we always need to scalarize.
       .scalarize(0)
       // Regardless of FP16 support, widen 16-bit elements to 32-bits.
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index eb94cc5d0fb61..314c5458e3090 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -2321,6 +2321,54 @@ define float @test_tan_f32(float %x) {
   ret float %y
 }
 
+declare float @llvm.acos.f32(float)
+define float @test_acos_f32(float %x) {
+  ; CHECK-LABEL: name:            test_acos_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FACOS %{{[0-9]+}}
+  %y = call float @llvm.acos.f32(float %x)
+  ret float %y
+}
+
+declare float @llvm.asin.f32(float)
+define float @test_asin_f32(float %x) {
+  ; CHECK-LABEL: name:            test_asin_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FASIN %{{[0-9]+}}
+  %y = call float @llvm.asin.f32(float %x)
+  ret float %y
+}
+
+declare float @llvm.atan.f32(float)
+define float @test_atan_f32(float %x) {
+  ; CHECK-LABEL: name:            test_atan_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FATAN %{{[0-9]+}}
+  %y = call float @llvm.atan.f32(float %x)
+  ret float %y
+}
+
+declare float @llvm.cosh.f32(float)
+define float @test_cosh_f32(float %x) {
+  ; CHECK-LABEL: name:            test_cosh_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FCOSH %{{[0-9]+}}
+  %y = call float @llvm.cosh.f32(float %x)
+  ret float %y
+}
+
+declare float @llvm.sinh.f32(float)
+define float @test_sinh_f32(float %x) {
+  ; CHECK-LABEL: name:            test_sinh_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FSINH %{{[0-9]+}}
+  %y = call float @llvm.sinh.f32(float %x)
+  ret float %y
+}
+
+declare float @llvm.tanh.f32(float)
+define float @test_tanh_f32(float %x) {
+  ; CHECK-LABEL: name:            test_tanh_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FTANH %{{[0-9]+}}
+  %y = call float @llvm.tanh.f32(float %x)
+  ret float %y
+}
+
 declare float @llvm.sqrt.f32(float)
 define float @test_sqrt_f32(float %x) {
   ; CHECK-LABEL: name:            test_sqrt_f32
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-acos.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-acos.mir
new file mode 100644
index 0000000000000..fd33a4198da6b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-acos.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.acos
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.acos
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &acosf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &acosf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &acosf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &acosf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FACOS %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.acos
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.acos
+
+    ; This is big, so let's just check for the 8 calls to acosf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &acosf
+    ; CHECK: BL &acosf
+    ; CHECK: BL &acosf
+    ; CHECK: BL &acosf
+    ; CHECK: BL &acosf
+    ; CHECK: BL &acosf
+    ; CHECK: BL &acosf
+    ; CHECK: BL &acosf
+    ; CHECK: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FACOS %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.acos
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.acos
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &acosf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &acosf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FACOS %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.acos
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.acos
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &acosf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &acosf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &acosf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &acosf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FACOS %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.acos
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.acos
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &acos
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &acos
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FACOS %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_acos_half
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_acos_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &acosf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FACOS %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-asin.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-asin.mir
new file mode 100644
index 0000000000000..981a3ff542042
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-asin.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.asin
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.asin
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &asinf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &asinf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &asinf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &asinf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FASIN %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.asin
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.asin
+
+    ; This is big, so let's just check for the 8 calls to asinf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &asinf
+    ; CHECK: BL &asinf
+    ; CHECK: BL &asinf
+    ; CHECK: BL &asinf
+    ; CHECK: BL &asinf
+    ; CHECK: BL &asinf
+    ; CHECK: BL &asinf
+    ; CHECK: BL &asinf
+    ; CHECK: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FASIN %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.asin
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.asin
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &asinf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &asinf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FASIN %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.asin
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.asin
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &asinf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &asinf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &asinf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &asinf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FASIN %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.asin
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.asin
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &asin
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &asin
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FASIN %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_asin_half
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_asin_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &asinf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FASIN %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-atan.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-atan.mir
new file mode 100644
index 0000000000000..7e0015ff7c797
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-atan.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.atan
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.atan
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &atanf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &atanf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &atanf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &atanf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FATAN %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.atan
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.atan
+
+    ; This is big, so let's just check for the 8 calls to atanf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &atanf
+    ; CHECK: BL &atanf
+    ; CHECK: BL &atanf
+    ; CHECK: BL &atanf
+    ; CHECK: BL &atanf
+    ; CHECK: BL &atanf
+    ; CHECK: BL &atanf
+    ; CHECK: BL &atanf
+    ; CHECK: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FATAN %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.atan
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.atan
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &atanf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &atanf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FATAN %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.atan
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.atan
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &atanf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &atanf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &atanf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &atanf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FATAN %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.atan
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.atan
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &atan
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &atan
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FATAN %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_atan_half
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_atan_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &atanf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FATAN %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cosh.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cosh.mir
new file mode 100644
index 0000000000000..653b447ed24b8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cosh.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.cosh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.cosh
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &coshf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &coshf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &coshf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &coshf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FCOSH %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.cosh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.cosh
+
+    ; This is big, so let's just check for the 8 calls to coshf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &coshf
+    ; CHECK: BL &coshf
+    ; CHECK: BL &coshf
+    ; CHECK: BL &coshf
+    ; CHECK: BL &coshf
+    ; CHECK: BL &coshf
+    ; CHECK: BL &coshf
+    ; CHECK: BL &coshf
+    ; CHECK: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FCOSH %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.cosh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.cosh
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &coshf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &coshf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FCOSH %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.cosh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.cosh
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &coshf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &coshf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &coshf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &coshf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FCOSH %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.cosh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.cosh
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &cosh
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &cosh
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FCOSH %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_cosh_half
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_cosh_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &coshf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FCOSH %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sinh.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sinh.mir
new file mode 100644
index 0000000000000..8cf3409fd0d2d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sinh.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.sinh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.sinh
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &sinhf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &sinhf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &sinhf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &sinhf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FSINH %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.sinh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.sinh
+
+    ; This is big, so let's just check for the 8 calls to sinhf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &sinhf
+    ; CHECK: BL &sinhf
+    ; CHECK: BL &sinhf
+    ; CHECK: BL &sinhf
+    ; CHECK: BL &sinhf
+    ; CHECK: BL &sinhf
+    ; CHECK: BL &sinhf
+    ; CHECK: BL &sinhf
+    ; CHECK: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FSINH %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.sinh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.sinh
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &sinhf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &sinhf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FSINH %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.sinh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.sinh
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &sinhf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &sinhf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &sinhf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &sinhf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FSINH %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.sinh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.sinh
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &sinh
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &sinh
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FSINH %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_sinh_half
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_sinh_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &sinhf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FSINH %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-tanh.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-tanh.mir
new file mode 100644
index 0000000000000..2cab146996faa
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-tanh.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.tanh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.tanh
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &tanhf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &tanhf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &tanhf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &tanhf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FTANH %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.tanh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.tanh
+
+    ; This is big, so let's just check for the 8 calls to tanhf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &tanhf
+    ; CHECK: BL &tanhf
+    ; CHECK: BL &tanhf
+    ; CHECK: BL &tanhf
+    ; CHECK: BL &tanhf
+    ; CHECK: BL &tanhf
+    ; CHECK: BL &tanhf
+    ; CHECK: BL &tanhf
+    ; CHECK: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FTANH %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.tanh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.tanh
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &tanhf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &tanhf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FTANH %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.tanh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.tanh
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &tanhf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &tanhf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &tanhf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &tanhf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FTANH %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.tanh
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.tanh
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &tanh
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &tanh
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FTANH %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_tanh_half
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_tanh_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &tanhf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FTANH %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index b8da462ed78a1..0e7804e98ae6d 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -692,23 +692,29 @@
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FACOS (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FASIN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FATAN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FCOSH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FSINH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FTANH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FSQRT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/AArch64/f16-instructions.ll b/llvm/test/CodeGen/AArch64/f16-instructions.ll
index 998f7dd38c3f5..8710703ab970e 100644
--- a/llvm/test/CodeGen/AArch64/f16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/f16-instructions.ll
@@ -760,6 +760,12 @@ declare half @llvm.powi.f16.i32(half %a, i32 %b) #0
 declare half @llvm.sin.f16(half %a) #0
 declare half @llvm.cos.f16(half %a) #0
 declare half @llvm.tan.f16(half %a) #0
+declare half @llvm.asin.f16(half %a) #0
+declare half @llvm.acos.f16(half %a) #0
+declare half @llvm.atan.f16(half %a) #0
+declare half @llvm.sinh.f16(half %a) #0
+declare half @llvm.cosh.f16(half %a) #0
+declare half @llvm.tanh.f16(half %a) #0
 declare half @llvm.pow.f16(half %a, half %b) #0
 declare half @llvm.exp.f16(half %a) #0
 declare half @llvm.exp2.f16(half %a) #0
@@ -896,6 +902,156 @@ define half @test_tan(half %a) #0 {
   ret half %r
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_acos
+; FALLBACK-FP16-NOT: remark:{{.*}}test_acos
+
+; CHECK-COMMON-LABEL: test_acos:
+; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
+; CHECK-COMMON-NEXT: mov  x29, sp
+; CHECK-COMMON-NEXT: fcvt s0, h0
+; CHECK-COMMON-NEXT: bl {{_?}}acosf
+; CHECK-COMMON-NEXT: fcvt h0, s0
+; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
+; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_acos:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}acosf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
+define half @test_acos(half %a) #0 {
+  %r = call half @llvm.acos.f16(half %a)
+  ret half %r
+}
+
+; FALLBACK-NOT: remark:{{.*}}test_asin
+; FALLBACK-FP16-NOT: remark:{{.*}}test_asin
+
+; CHECK-COMMON-LABEL: test_asin:
+; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
+; CHECK-COMMON-NEXT: mov  x29, sp
+; CHECK-COMMON-NEXT: fcvt s0, h0
+; CHECK-COMMON-NEXT: bl {{_?}}asinf
+; CHECK-COMMON-NEXT: fcvt h0, s0
+; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
+; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_asin:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}asinf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
+define half @test_asin(half %a) #0 {
+  %r = call half @llvm.asin.f16(half %a)
+  ret half %r
+}
+
+; FALLBACK-NOT: remark:{{.*}}test_atan
+; FALLBACK-FP16-NOT: remark:{{.*}}test_atan
+
+; CHECK-COMMON-LABEL: test_atan:
+; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
+; CHECK-COMMON-NEXT: mov  x29, sp
+; CHECK-COMMON-NEXT: fcvt s0, h0
+; CHECK-COMMON-NEXT: bl {{_?}}atanf
+; CHECK-COMMON-NEXT: fcvt h0, s0
+; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
+; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_atan:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}atanf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
+define half @test_atan(half %a) #0 {
+  %r = call half @llvm.atan.f16(half %a)
+  ret half %r
+}
+
+; FALLBACK-NOT: remark:{{.*}}test_cosh
+; FALLBACK-FP16-NOT: remark:{{.*}}test_cosh
+
+; CHECK-COMMON-LABEL: test_cosh:
+; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
+; CHECK-COMMON-NEXT: mov  x29, sp
+; CHECK-COMMON-NEXT: fcvt s0, h0
+; CHECK-COMMON-NEXT: bl {{_?}}coshf
+; CHECK-COMMON-NEXT: fcvt h0, s0
+; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
+; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_cosh:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}coshf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
+define half @test_cosh(half %a) #0 {
+  %r = call half @llvm.cosh.f16(half %a)
+  ret half %r
+}
+
+; FALLBACK-NOT: remark:{{.*}}test_sinh
+; FALLBACK-FP16-NOT: remark:{{.*}}test_sinh
+
+; CHECK-COMMON-LABEL: test_sinh:
+; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
+; CHECK-COMMON-NEXT: mov  x29, sp
+; CHECK-COMMON-NEXT: fcvt s0, h0
+; CHECK-COMMON-NEXT: bl {{_?}}sinhf
+; CHECK-COMMON-NEXT: fcvt h0, s0
+; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
+; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_sinh:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}sinhf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
+define half @test_sinh(half %a) #0 {
+  %r = call half @llvm.sinh.f16(half %a)
+  ret half %r
+}
+
+; FALLBACK-NOT: remark:{{.*}}test_tanh
+; FALLBACK-FP16-NOT: remark:{{.*}}test_tanh
+
+; CHECK-COMMON-LABEL: test_tanh:
+; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
+; CHECK-COMMON-NEXT: mov  x29, sp
+; CHECK-COMMON-NEXT: fcvt s0, h0
+; CHECK-COMMON-NEXT: bl {{_?}}tanhf
+; CHECK-COMMON-NEXT: fcvt h0, s0
+; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
+; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_tanh:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}tanhf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
+define half @test_tanh(half %a) #0 {
+  %r = call half @llvm.tanh.f16(half %a)
+  ret half %r
+}
+
 ; CHECK-COMMON-LABEL: test_pow:
 ; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
 ; CHECK-COMMON-NEXT: mov  x29, sp
diff --git a/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll b/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll
index b09ed8d3eb764..cbdfb4c932775 100644
--- a/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll
+++ b/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll
@@ -353,6 +353,96 @@ define half @tan_f16(half %x) #0 {
   ret half %val
 }
 
+define half @asin_f16(half %x) #0 {
+; CHECK-LABEL: asin_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl asinf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %val = call half @llvm.experimental.constrained.asin.f16(half %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret half %val
+}
+
+define half @acos_f16(half %x) #0 {
+; CHECK-LABEL: acos_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl acosf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %val = call half @llvm.experimental.constrained.acos.f16(half %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret half %val
+}
+
+define half @atan_f16(half %x) #0 {
+; CHECK-LABEL: atan_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl atanf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %val = call half @llvm.experimental.constrained.atan.f16(half %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret half %val
+}
+
+define half @sinh_f16(half %x) #0 {
+; CHECK-LABEL: sinh_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl sinhf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %val = call half @llvm.experimental.constrained.sinh.f16(half %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret half %val
+}
+
+define half @cosh_f16(half %x) #0 {
+; CHECK-LABEL: cosh_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl coshf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %val = call half @llvm.experimental.constrained.cosh.f16(half %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret half %val
+}
+
+define half @tanh_f16(half %x) #0 {
+; CHECK-LABEL: tanh_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl tanhf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %val = call half @llvm.experimental.constrained.tanh.f16(half %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret half %val
+}
+
 define half @pow_f16(half %x, half %y) #0 {
 ; CHECK-LABEL: pow_f16:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
index 67d0b63f4076f..62b4a79b26d8e 100644
--- a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
@@ -153,6 +153,48 @@ define float @tan_f32(float %x) #0 {
   ret float %val
 }
 
+; CHECK-LABEL: asin_f32:
+; CHECK: bl asinf
+define float @asin_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.asin.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: acos_f32:
+; CHECK: bl acosf
+define float @acos_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.acos.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: atan_f32:
+; CHECK: bl atanf
+define float @atan_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.atan.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: sinh_f32:
+; CHECK: bl sinhf
+define float @sinh_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.sinh.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: cosh_f32:
+; CHECK: bl coshf
+define float @cosh_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.cosh.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: tanh_f32:
+; CHECK: bl tanhf
+define float @tanh_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.tanh.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
 ; CHECK-LABEL: pow_f32:
 ; CHECK: bl powf
 define float @pow_f32(float %x, float %y) #0 {
@@ -644,6 +686,48 @@ define double @tan_f64(double %x) #0 {
   ret double %val
 }
 
+; CHECK-LABEL: asin_f64:
+; CHECK: bl asin
+define double @asin_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.asin.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: acos_f64:
+; CHECK: bl acos
+define double @acos_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.acos.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: atan_f64:
+; CHECK: bl atan
+define double @atan_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.atan.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: sinh_f64:
+; CHECK: bl sinh
+define double @sinh_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.sinh.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: cosh_f64:
+; CHECK: bl cosh
+define double @cosh_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.cosh.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: tanh_f64:
+; CHECK: bl tanh
+define double @tanh_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.tanh.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
 ; CHECK-LABEL: pow_f64:
 ; CHECK: bl pow
 define double @pow_f64(double %x, double %y) #0 {
@@ -1135,6 +1219,48 @@ define fp128 @tan_f128(fp128 %x) #0 {
   ret fp128 %val
 }
 
+; CHECK-LABEL: asin_f128:
+; CHECK: bl asinl
+define fp128 @asin_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.asin.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: acos_f128:
+; CHECK: bl acosl
+define fp128 @acos_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.acos.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: atan_f128:
+; CHECK: bl atanl
+define fp128 @atan_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.atan.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: sinh_f128:
+; CHECK: bl sinhl
+define fp128 @sinh_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.sinh.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: cosh_f128:
+; CHECK: bl coshl
+define fp128 @cosh_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.cosh.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: tanh_f128:
+; CHECK: bl tanhl
+define fp128 @tanh_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.tanh.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
 ; CHECK-LABEL: pow_f128:
 ; CHECK: bl powl
 define fp128 @pow_f128(fp128 %x, fp128 %y) #0 {
@@ -1519,6 +1645,48 @@ define <1 x double> @tan_v1f64(<1 x double> %x, <1 x double> %y) #0 {
   ret <1 x double> %val
 }
 
+; CHECK-LABEL: asin_v1f64:
+; CHECK: bl asin
+define <1 x double> @asin_v1f64(<1 x double> %x, <1 x double> %y) #0 {
+  %val = call <1 x double> @llvm.experimental.constrained.asin.v1f64(<1 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <1 x double> %val
+}
+
+; CHECK-LABEL: acos_v1f64:
+; CHECK: bl acos
+define <1 x double> @acos_v1f64(<1 x double> %x, <1 x double> %y) #0 {
+  %val = call <1 x double> @llvm.experimental.constrained.acos.v1f64(<1 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <1 x double> %val
+}
+
+; CHECK-LABEL: atan_v1f64:
+; CHECK: bl atan
+define <1 x double> @atan_v1f64(<1 x double> %x, <1 x double> %y) #0 {
+  %val = call <1 x double> @llvm.experimental.constrained.atan.v1f64(<1 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <1 x double> %val
+}
+
+; CHECK-LABEL: sinh_v1f64:
+; CHECK: bl sinh
+define <1 x double> @sinh_v1f64(<1 x double> %x, <1 x double> %y) #0 {
+  %val = call <1 x double> @llvm.experimental.constrained.sinh.v1f64(<1 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <1 x double> %val
+}
+
+; CHECK-LABEL: cosh_v1f64:
+; CHECK: bl cosh
+define <1 x double> @cosh_v1f64(<1 x double> %x, <1 x double> %y) #0 {
+  %val = call <1 x double> @llvm.experimental.constrained.cosh.v1f64(<1 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <1 x double> %val
+}
+
+; CHECK-LABEL: tanh_v1f64:
+; CHECK: bl tanh
+define <1 x double> @tanh_v1f64(<1 x double> %x, <1 x double> %y) #0 {
+  %val = call <1 x double> @llvm.experimental.constrained.tanh.v1f64(<1 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <1 x double> %val
+}
+
 ; CHECK-LABEL: pow_v1f64:
 ; CHECK: bl pow
 define <1 x double> @pow_v1f64(<1 x double> %x, <1 x double> %y) #0 {
@@ -1584,6 +1752,12 @@ declare float @llvm.experimental.constrained.powi.f32(float, i32, metadata, meta
 declare float @llvm.experimental.constrained.sin.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.cos.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.tan.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.asin.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.acos.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.atan.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.sinh.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.cosh.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.tanh.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.pow.f32(float, float, metadata, metadata)
 declare float @llvm.experimental.constrained.log.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.log10.f32(float, metadata, metadata)
@@ -1629,6 +1803,12 @@ declare double @llvm.experimental.constrained.powi.f64(double, i32, metadata, me
 declare double @llvm.experimental.constrained.sin.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.cos.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.tan.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.asin.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.acos.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.atan.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.sinh.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.cosh.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.tanh.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.pow.f64(double, double, metadata, metadata)
 declare double @llvm.experimental.constrained.log.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.log10.f64(double, metadata, metadata)
@@ -1674,6 +1854,12 @@ declare fp128 @llvm.experimental.constrained.powi.f128(fp128, i32, metadata, met
 declare fp128 @llvm.experimental.constrained.sin.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.cos.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.tan.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.asin.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.acos.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.atan.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sinh.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.cosh.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.tanh.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.pow.f128(fp128, fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.log.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.log10.f128(fp128, metadata, metadata)
diff --git a/llvm/test/CodeGen/AArch64/illegal-float-ops.ll b/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
index 3281a98767795..08f6bb6f28532 100644
--- a/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
+++ b/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
@@ -178,7 +178,132 @@ define void @test_tan(float %float, double %double, fp128 %fp128) {
    store fp128 %tanfp128, ptr @varfp128
 ; CHECK: bl tanl
   ret void
+}
+
+declare float @llvm.acos.f32(float)
+declare double @llvm.acos.f64(double)
+declare fp128 @llvm.acos.f128(fp128)
+
+define void @test_acos(float %float, double %double, fp128 %fp128) {
+; CHECK-LABEL: test_acos:
+
+   %acosfloat = call float @llvm.acos.f32(float %float)
+   store float %acosfloat, ptr @varfloat
+; CHECK: bl acosf
+
+   %acosdouble = call double @llvm.acos.f64(double %double)
+   store double %acosdouble, ptr @vardouble
+; CHECK: bl acos
+
+   %acosfp128 = call fp128 @llvm.acos.f128(fp128 %fp128)
+   store fp128 %acosfp128, ptr @varfp128
+; CHECK: bl acosl
+  ret void
+}
+
+declare float @llvm.asin.f32(float)
+declare double @llvm.asin.f64(double)
+declare fp128 @llvm.asin.f128(fp128)
+
+define void @test_asin(float %float, double %double, fp128 %fp128) {
+; CHECK-LABEL: test_asin:
+
+   %asinfloat = call float @llvm.asin.f32(float %float)
+   store float %asinfloat, ptr @varfloat
+; CHECK: bl asinf
 
+   %asindouble = call double @llvm.asin.f64(double %double)
+   store double %asindouble, ptr @vardouble
+; CHECK: bl asin
+
+   %asinfp128 = call fp128 @llvm.asin.f128(fp128 %fp128)
+   store fp128 %asinfp128, ptr @varfp128
+; CHECK: bl asinl
+  ret void
+}
+
+declare float @llvm.atan.f32(float)
+declare double @llvm.atan.f64(double)
+declare fp128 @llvm.atan.f128(fp128)
+
+define void @test_atan(float %float, double %double, fp128 %fp128) {
+; CHECK-LABEL: test_atan:
+
+   %atanfloat = call float @llvm.atan.f32(float %float)
+   store float %atanfloat, ptr @varfloat
+; CHECK: bl atanf
+
+   %atandouble = call double @llvm.atan.f64(double %double)
+   store double %atandouble, ptr @vardouble
+; CHECK: bl atan
+
+   %atanfp128 = call fp128 @llvm.atan.f128(fp128 %fp128)
+   store fp128 %atanfp128, ptr @varfp128
+; CHECK: bl atanl
+  ret void
+}
+
+declare float @llvm.cosh.f32(float)
+declare double @llvm.cosh.f64(double)
+declare fp128 @llvm.cosh.f128(fp128)
+
+define void @test_cosh(float %float, double %double, fp128 %fp128) {
+; CHECK-LABEL: test_cosh:
+
+   %coshfloat = call float @llvm.cosh.f32(float %float)
+   store float %coshfloat, ptr @varfloat
+; CHECK: bl coshf
+
+   %coshdouble = call double @llvm.cosh.f64(double %double)
+   store double %coshdouble, ptr @vardouble
+; CHECK: bl cosh
+
+   %coshfp128 = call fp128 @llvm.cosh.f128(fp128 %fp128)
+   store fp128 %coshfp128, ptr @varfp128
+; CHECK: bl coshl
+  ret void
+}
+
+declare float @llvm.sinh.f32(float)
+declare double @llvm.sinh.f64(double)
+declare fp128 @llvm.sinh.f128(fp128)
+
+define void @test_sinh(float %float, double %double, fp128 %fp128) {
+; CHECK-LABEL: test_sinh:
+
+   %sinhfloat = call float @llvm.sinh.f32(float %float)
+   store float %sinhfloat, ptr @varfloat
+; CHECK: bl sinhf
+
+   %sinhdouble = call double @llvm.sinh.f64(double %double)
+   store double %sinhdouble, ptr @vardouble
+; CHECK: bl sinh
+
+   %sinhfp128 = call fp128 @llvm.sinh.f128(fp128 %fp128)
+   store fp128 %sinhfp128, ptr @varfp128
+; CHECK: bl sinhl
+  ret void
+}
+
+declare float @llvm.tanh.f32(float)
+declare double @llvm.tanh.f64(double)
+declare fp128 @llvm.tanh.f128(fp128)
+
+define void @test_tanh(float %float, double %double, fp128 %fp128) {
+; CHECK-LABEL: test_tanh:
+
+   %tanhfloat = call float @llvm.tanh.f32(float %float)
+   store float %tanhfloat, ptr @varfloat
+; CHECK: bl tanhf
+
+   %tanhdouble = call double @llvm.tanh.f64(double %double)
+   store double %tanhdouble, ptr @vardouble
+; CHECK: bl tanh
+
+   %tanhfp128 = call fp128 @llvm.tanh.f128(fp128 %fp128)
+   store fp128 %tanhfp128, ptr @varfp128
+; CHECK: bl tanhl
+  ret void
 }
 
 declare float @llvm.pow.f32(float, float)
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
index 7d23e870637a8..f7e95008b7123 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
@@ -15,7 +15,8 @@ declare <vscale x 2 x double> @llvm.cos.nxv2f64(<vscale x 2 x double>)
 declare <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float>)
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [36 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vtanq_f64, ptr @armpl_vtanq_f32, ptr @armpl_svtan_f64_x, ptr @armpl_svtan_f32_x], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [60 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vtanq_f64, ptr @armpl_vtanq_f32, ptr @armpl_svtan_f64_x, ptr @armpl_svtan_f32_x, ptr @armpl_vacosq_f64, ptr @armpl_vacosq_f32, ptr @armpl_svacos_f64_x, ptr @armpl_svacos_f32_x, ptr @armpl_vasinq_f64, ptr @armpl_vasinq_f32, ptr @armpl_svasin_f64_x, ptr @armpl_svasin_f32_x, ptr @armpl_vatanq_f64, ptr @armpl_vatanq_f32, ptr @armpl_svatan_f64_x, ptr @armpl_svatan_f32_x, ptr @armpl_vcoshq_f64, ptr @armpl_vcoshq_f32, ptr @armpl_svcosh_f64_x, ptr @armpl_svcosh_f32_x, ptr @armpl_vsinhq_f64, ptr @armpl_vsinhq_f32, ptr @armpl_svsinh_f64_x, ptr @armpl_svsinh_f32_x, ptr @armpl_vtanhq_f64, ptr @armpl_vtanhq_f32, ptr @armpl_svtanh_f64_x, ptr @armpl_svtanh_f32_x], section "llvm.metadata"
+
 ;.
 define <2 x double> @llvm_cos_f64(<2 x double> %in) {
 ; CHECK-LABEL: define <2 x double> @llvm_cos_f64
@@ -469,6 +470,276 @@ define <vscale x 4 x float> @llvm_tan_vscale_f32(<vscale x 4 x float> %in) #0 {
   ret <vscale x 4 x float> %1
 }
 
+declare <2 x double> @llvm.acos.v2f64(<2 x double>)
+declare <4 x float> @llvm.acos.v4f32(<4 x float>)
+declare <vscale x 2 x double> @llvm.acos.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.acos.nxv4f32(<vscale x 4 x float>)
+
+define <2 x double> @llvm_acos_f64(<2 x double> %in) {
+; CHECK-LABEL: define <2 x double> @llvm_acos_f64
+; CHECK-SAME: (<2 x double> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @armpl_vacosq_f64(<2 x double> [[IN]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.acos.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_acos_f32(<4 x float> %in) {
+; CHECK-LABEL: define <4 x float> @llvm_acos_f32
+; CHECK-SAME: (<4 x float> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @armpl_vacosq_f32(<4 x float> [[IN]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.acos.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_acos_vscale_f64(<vscale x 2 x double> %in) #0 {
+; CHECK-LABEL: define <vscale x 2 x double> @llvm_acos_vscale_f64
+; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.acos.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_acos_vscale_f32(<vscale x 4 x float> %in) #0 {
+; CHECK-LABEL: define <vscale x 4 x float> @llvm_acos_vscale_f32
+; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.acos.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
+declare <2 x double> @llvm.asin.v2f64(<2 x double>)
+declare <4 x float> @llvm.asin.v4f32(<4 x float>)
+declare <vscale x 2 x double> @llvm.asin.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.asin.nxv4f32(<vscale x 4 x float>)
+
+define <2 x double> @llvm_asin_f64(<2 x double> %in) {
+; CHECK-LABEL: define <2 x double> @llvm_asin_f64
+; CHECK-SAME: (<2 x double> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @armpl_vasinq_f64(<2 x double> [[IN]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.asin.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_asin_f32(<4 x float> %in) {
+; CHECK-LABEL: define <4 x float> @llvm_asin_f32
+; CHECK-SAME: (<4 x float> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @armpl_vasinq_f32(<4 x float> [[IN]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.asin.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_asin_vscale_f64(<vscale x 2 x double> %in) #0 {
+; CHECK-LABEL: define <vscale x 2 x double> @llvm_asin_vscale_f64
+; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.asin.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_asin_vscale_f32(<vscale x 4 x float> %in) #0 {
+; CHECK-LABEL: define <vscale x 4 x float> @llvm_asin_vscale_f32
+; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.asin.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
+declare <2 x double> @llvm.atan.v2f64(<2 x double>)
+declare <4 x float> @llvm.atan.v4f32(<4 x float>)
+declare <vscale x 2 x double> @llvm.atan.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.atan.nxv4f32(<vscale x 4 x float>)
+
+define <2 x double> @llvm_atan_f64(<2 x double> %in) {
+; CHECK-LABEL: define <2 x double> @llvm_atan_f64
+; CHECK-SAME: (<2 x double> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @armpl_vatanq_f64(<2 x double> [[IN]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.atan.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_atan_f32(<4 x float> %in) {
+; CHECK-LABEL: define <4 x float> @llvm_atan_f32
+; CHECK-SAME: (<4 x float> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @armpl_vatanq_f32(<4 x float> [[IN]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.atan.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_atan_vscale_f64(<vscale x 2 x double> %in) #0 {
+; CHECK-LABEL: define <vscale x 2 x double> @llvm_atan_vscale_f64
+; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.atan.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_atan_vscale_f32(<vscale x 4 x float> %in) #0 {
+; CHECK-LABEL: define <vscale x 4 x float> @llvm_atan_vscale_f32
+; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.atan.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
+declare <2 x double> @llvm.cosh.v2f64(<2 x double>)
+declare <4 x float> @llvm.cosh.v4f32(<4 x float>)
+declare <vscale x 2 x double> @llvm.cosh.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.cosh.nxv4f32(<vscale x 4 x float>)
+
+define <2 x double> @llvm_cosh_f64(<2 x double> %in) {
+; CHECK-LABEL: define <2 x double> @llvm_cosh_f64
+; CHECK-SAME: (<2 x double> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @armpl_vcoshq_f64(<2 x double> [[IN]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.cosh.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_cosh_f32(<4 x float> %in) {
+; CHECK-LABEL: define <4 x float> @llvm_cosh_f32
+; CHECK-SAME: (<4 x float> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @armpl_vcoshq_f32(<4 x float> [[IN]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.cosh.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_cosh_vscale_f64(<vscale x 2 x double> %in) #0 {
+; CHECK-LABEL: define <vscale x 2 x double> @llvm_cosh_vscale_f64
+; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.cosh.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_cosh_vscale_f32(<vscale x 4 x float> %in) #0 {
+; CHECK-LABEL: define <vscale x 4 x float> @llvm_cosh_vscale_f32
+; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.cosh.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
+declare <2 x double> @llvm.sinh.v2f64(<2 x double>)
+declare <4 x float> @llvm.sinh.v4f32(<4 x float>)
+declare <vscale x 2 x double> @llvm.sinh.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.sinh.nxv4f32(<vscale x 4 x float>)
+
+define <2 x double> @llvm_sinh_f64(<2 x double> %in) {
+; CHECK-LABEL: define <2 x double> @llvm_sinh_f64
+; CHECK-SAME: (<2 x double> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @armpl_vsinhq_f64(<2 x double> [[IN]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.sinh.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_sinh_f32(<4 x float> %in) {
+; CHECK-LABEL: define <4 x float> @llvm_sinh_f32
+; CHECK-SAME: (<4 x float> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @armpl_vsinhq_f32(<4 x float> [[IN]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.sinh.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_sinh_vscale_f64(<vscale x 2 x double> %in) #0 {
+; CHECK-LABEL: define <vscale x 2 x double> @llvm_sinh_vscale_f64
+; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.sinh.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_sinh_vscale_f32(<vscale x 4 x float> %in) #0 {
+; CHECK-LABEL: define <vscale x 4 x float> @llvm_sinh_vscale_f32
+; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.sinh.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
+declare <2 x double> @llvm.tanh.v2f64(<2 x double>)
+declare <4 x float> @llvm.tanh.v4f32(<4 x float>)
+declare <vscale x 2 x double> @llvm.tanh.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.tanh.nxv4f32(<vscale x 4 x float>)
+
+define <2 x double> @llvm_tanh_f64(<2 x double> %in) {
+; CHECK-LABEL: define <2 x double> @llvm_tanh_f64
+; CHECK-SAME: (<2 x double> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @armpl_vtanhq_f64(<2 x double> [[IN]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.tanh.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_tanh_f32(<4 x float> %in) {
+; CHECK-LABEL: define <4 x float> @llvm_tanh_f32
+; CHECK-SAME: (<4 x float> [[IN:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @armpl_vtanhq_f32(<4 x float> [[IN]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.tanh.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_tanh_vscale_f64(<vscale x 2 x double> %in) #0 {
+; CHECK-LABEL: define <vscale x 2 x double> @llvm_tanh_vscale_f64
+; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.tanh.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_tanh_vscale_f32(<vscale x 4 x float> %in) #0 {
+; CHECK-LABEL: define <vscale x 4 x float> @llvm_tanh_vscale_f32
+; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.tanh.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
 attributes #0 = { "target-features"="+sve" }
 ;.
 ; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
index 15d100a518c15..f3f27344ad80e 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
@@ -4,7 +4,7 @@
 target triple = "aarch64-unknown-linux-gnu"
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [18 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxv_tan, ptr @_ZGVsMxv_tanf], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [30 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxv_tan, ptr @_ZGVsMxv_tanf, ptr @_ZGVsMxv_acos, ptr @_ZGVsMxv_acosf, ptr @_ZGVsMxv_asin, ptr @_ZGVsMxv_asinf, ptr @_ZGVsMxv_atan, ptr @_ZGVsMxv_atanf, ptr @_ZGVsMxv_cosh, ptr @_ZGVsMxv_coshf, ptr @_ZGVsMxv_sinh, ptr @_ZGVsMxv_sinhf, ptr @_ZGVsMxv_tanh, ptr @_ZGVsMxv_tanhf], section "llvm.metadata"
 ;.
 define <vscale x 2 x double> @llvm_ceil_vscale_f64(<vscale x 2 x double> %in) {
 ; CHECK-LABEL: @llvm_ceil_vscale_f64(
@@ -384,6 +384,114 @@ define <vscale x 4 x float> @llvm_tan_vscale_f32(<vscale x 4 x float> %in) {
   ret <vscale x 4 x float> %1
 }
 
+define <vscale x 2 x double> @llvm_acos_vscale_f64(<vscale x 2 x double> %in) {
+; CHECK-LABEL: @llvm_acos_vscale_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.acos.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_acos_vscale_f32(<vscale x 4 x float> %in) {
+; CHECK-LABEL: @llvm_acos_vscale_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.acos.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_asin_vscale_f64(<vscale x 2 x double> %in) {
+; CHECK-LABEL: @llvm_asin_vscale_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.asin.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_asin_vscale_f32(<vscale x 4 x float> %in) {
+; CHECK-LABEL: @llvm_asin_vscale_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.asin.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_atan_vscale_f64(<vscale x 2 x double> %in) {
+; CHECK-LABEL: @llvm_atan_vscale_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.atan.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_atan_vscale_f32(<vscale x 4 x float> %in) {
+; CHECK-LABEL: @llvm_atan_vscale_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.atan.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_cosh_vscale_f64(<vscale x 2 x double> %in) {
+; CHECK-LABEL: @llvm_cosh_vscale_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.cosh.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_cosh_vscale_f32(<vscale x 4 x float> %in) {
+; CHECK-LABEL: @llvm_cosh_vscale_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.cosh.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_sinh_vscale_f64(<vscale x 2 x double> %in) {
+; CHECK-LABEL: @llvm_sinh_vscale_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.sinh.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_sinh_vscale_f32(<vscale x 4 x float> %in) {
+; CHECK-LABEL: @llvm_sinh_vscale_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.sinh.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_tanh_vscale_f64(<vscale x 2 x double> %in) {
+; CHECK-LABEL: @llvm_tanh_vscale_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.tanh.nxv2f64(<vscale x 2 x double> %in)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_tanh_vscale_f32(<vscale x 4 x float> %in) {
+; CHECK-LABEL: @llvm_tanh_vscale_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.tanh.nxv4f32(<vscale x 4 x float> %in)
+  ret <vscale x 4 x float> %1
+}
+
 
 define <vscale x 2 x double> @llvm_trunc_vscale_f64(<vscale x 2 x double> %in) {
 ; CHECK-LABEL: @llvm_trunc_vscale_f64(
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
index a3da3b8120218..59c2f94e16763 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
@@ -4,7 +4,7 @@
 target triple = "aarch64-unknown-linux-gnu"
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [18 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2v_tan, ptr @_ZGVnN4v_tanf], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [30 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2v_tan, ptr @_ZGVnN4v_tanf, ptr @_ZGVnN2v_acos, ptr @_ZGVnN4v_acosf, ptr @_ZGVnN2v_asin, ptr @_ZGVnN4v_asinf, ptr @_ZGVnN2v_atan, ptr @_ZGVnN4v_atanf, ptr @_ZGVnN2v_cosh, ptr @_ZGVnN4v_coshf, ptr @_ZGVnN2v_sinh, ptr @_ZGVnN4v_sinhf, ptr @_ZGVnN2v_tanh, ptr @_ZGVnN4v_tanhf], section "llvm.metadata"
 ;.
 define <2 x double> @llvm_ceil_f64(<2 x double> %in) {
 ; CHECK-LABEL: @llvm_ceil_f64(
@@ -384,6 +384,114 @@ define <4 x float> @llvm_tan_f32(<4 x float> %in) {
   ret <4 x float> %1
 }
 
+define <2 x double> @llvm_acos_f64(<2 x double> %in) {
+; CHECK-LABEL: @llvm_acos_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2v_acos(<2 x double> [[IN:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.acos.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_acos_f32(<4 x float> %in) {
+; CHECK-LABEL: @llvm_acos_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4v_acosf(<4 x float> [[IN:%.*]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.acos.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <2 x double> @llvm_asin_f64(<2 x double> %in) {
+; CHECK-LABEL: @llvm_asin_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2v_asin(<2 x double> [[IN:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.asin.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_asin_f32(<4 x float> %in) {
+; CHECK-LABEL: @llvm_asin_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4v_asinf(<4 x float> [[IN:%.*]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.asin.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <2 x double> @llvm_atan_f64(<2 x double> %in) {
+; CHECK-LABEL: @llvm_atan_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2v_atan(<2 x double> [[IN:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.atan.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_atan_f32(<4 x float> %in) {
+; CHECK-LABEL: @llvm_atan_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4v_atanf(<4 x float> [[IN:%.*]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.atan.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <2 x double> @llvm_cosh_f64(<2 x double> %in) {
+; CHECK-LABEL: @llvm_cosh_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2v_cosh(<2 x double> [[IN:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.cosh.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_cosh_f32(<4 x float> %in) {
+; CHECK-LABEL: @llvm_cosh_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4v_coshf(<4 x float> [[IN:%.*]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.cosh.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <2 x double> @llvm_sinh_f64(<2 x double> %in) {
+; CHECK-LABEL: @llvm_sinh_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2v_sinh(<2 x double> [[IN:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.sinh.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_sinh_f32(<4 x float> %in) {
+; CHECK-LABEL: @llvm_sinh_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4v_sinhf(<4 x float> [[IN:%.*]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.sinh.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
+define <2 x double> @llvm_tanh_f64(<2 x double> %in) {
+; CHECK-LABEL: @llvm_tanh_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2v_tanh(<2 x double> [[IN:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.tanh.v2f64(<2 x double> %in)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_tanh_f32(<4 x float> %in) {
+; CHECK-LABEL: @llvm_tanh_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4v_tanhf(<4 x float> [[IN:%.*]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.tanh.v4f32(<4 x float> %in)
+  ret <4 x float> %1
+}
+
 define <2 x double> @llvm_trunc_f64(<2 x double> %in) {
 ; CHECK-LABEL: @llvm_trunc_f64(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @llvm.trunc.v2f64(<2 x double> [[IN:%.*]])
diff --git a/llvm/test/CodeGen/AArch64/vec-libcalls.ll b/llvm/test/CodeGen/AArch64/vec-libcalls.ll
index 9bbac5d69c28c..43c1839818173 100644
--- a/llvm/test/CodeGen/AArch64/vec-libcalls.ll
+++ b/llvm/test/CodeGen/AArch64/vec-libcalls.ll
@@ -21,6 +21,12 @@ declare <3 x float> @llvm.fabs.v3f32(<3 x float>)
 declare <3 x float> @llvm.ceil.v3f32(<3 x float>)
 declare <3 x float> @llvm.cos.v3f32(<3 x float>)
 declare <3 x float> @llvm.tan.v3f32(<3 x float>)
+declare <3 x float> @llvm.asin.v3f32(<3 x float>)
+declare <3 x float> @llvm.acos.v3f32(<3 x float>)
+declare <3 x float> @llvm.atan.v3f32(<3 x float>)
+declare <3 x float> @llvm.sinh.v3f32(<3 x float>)
+declare <3 x float> @llvm.cosh.v3f32(<3 x float>)
+declare <3 x float> @llvm.tanh.v3f32(<3 x float>)
 declare <3 x float> @llvm.exp.v3f32(<3 x float>)
 declare <3 x float> @llvm.exp2.v3f32(<3 x float>)
 declare <3 x float> @llvm.floor.v3f32(<3 x float>)
@@ -329,6 +335,191 @@ define <3 x float> @tan_v3f32(<3 x float> %x) nounwind {
   ret <3 x float> %r
 }
 
+define <3 x float> @asin_v3f32(<3 x float> %x) nounwind {
+; CHECK-LABEL: asin_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    bl asinf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl asinf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl asinf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+  %r = call <3 x float> @llvm.asin.v3f32(<3 x float> %x)
+  ret <3 x float> %r
+}
+
+define <3 x float> @acos_v3f32(<3 x float> %x) nounwind {
+; CHECK-LABEL: acos_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    bl acosf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl acosf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl acosf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+  %r = call <3 x float> @llvm.acos.v3f32(<3 x float> %x)
+  ret <3 x float> %r
+}
+
+define <3 x float> @atan_v3f32(<3 x float> %x) nounwind {
+; CHECK-LABEL: atan_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    bl atanf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl atanf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl atanf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+  %r = call <3 x float> @llvm.atan.v3f32(<3 x float> %x)
+  ret <3 x float> %r
+}
+
+define <3 x float> @sinh_v3f32(<3 x float> %x) nounwind {
+; CHECK-LABEL: sinh_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    bl sinhf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl sinhf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl sinhf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+  %r = call <3 x float> @llvm.sinh.v3f32(<3 x float> %x)
+  ret <3 x float> %r
+}
+define <3 x float> @cosh_v3f32(<3 x float> %x) nounwind {
+; CHECK-LABEL: cosh_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    bl coshf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl coshf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl coshf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+  %r = call <3 x float> @llvm.cosh.v3f32(<3 x float> %x)
+  ret <3 x float> %r
+}
+
+define <3 x float> @tanh_v3f32(<3 x float> %x) nounwind {
+; CHECK-LABEL: tanh_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    bl tanhf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl tanhf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl tanhf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+  %r = call <3 x float> @llvm.tanh.v3f32(<3 x float> %x)
+  ret <3 x float> %r
+}
+
 define <3 x float> @exp_v3f32(<3 x float> %x) nounwind {
 ; CHECK-LABEL: exp_v3f32:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll
index 89a513515b205..ac6907609f5eb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll
@@ -435,6 +435,295 @@ for.end:
   ret void
 }
 
+declare float @llvm.acos.f32(float) nounwind readnone
+define void @acos_v4f32_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @acos_v4f32_intrinsic(
+; CHECK: call <4 x float> @_simd_acos_f4(<4 x float>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds float, ptr %y, i64 %iv
+  %lv = load float, ptr %gep.y, align 4
+  %call = tail call float @llvm.acos.f32(float %lv)
+  %gep.x = getelementptr inbounds float, ptr %x, i64 %iv
+  store float %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare double @llvm.acos.f64(double) nounwind readnone
+define void @acos_v2f64_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @acos_v2f64_intrinsic(
+; CHECK: call <2 x double> @_simd_acos_d2(<2 x double>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds double, ptr %y, i64 %iv
+  %lv = load double, ptr %gep.y, align 4
+  %call = tail call double @llvm.acos.f64(double %lv)
+  %gep.x = getelementptr inbounds double, ptr %x, i64 %iv
+  store double %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare float @llvm.asin.f32(float) nounwind readnone
+define void @asin_v4f32_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @asin_v4f32_intrinsic(
+; CHECK: call <4 x float> @_simd_asin_f4(<4 x float>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds float, ptr %y, i64 %iv
+  %lv = load float, ptr %gep.y, align 4
+  %call = tail call float @llvm.asin.f32(float %lv)
+  %gep.x = getelementptr inbounds float, ptr %x, i64 %iv
+  store float %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare double @llvm.asin.f64(double) nounwind readnone
+define void @asin_v2f64_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @asin_v2f64_intrinsic(
+; CHECK: call <2 x double> @_simd_asin_d2(<2 x double>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds double, ptr %y, i64 %iv
+  %lv = load double, ptr %gep.y, align 4
+  %call = tail call double @llvm.asin.f64(double %lv)
+  %gep.x = getelementptr inbounds double, ptr %x, i64 %iv
+  store double %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare float @llvm.atan.f32(float) nounwind readnone
+define void @atan_v4f32_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @atan_v4f32_intrinsic(
+; CHECK: call <4 x float> @_simd_atan_f4(<4 x float>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds float, ptr %y, i64 %iv
+  %lv = load float, ptr %gep.y, align 4
+  %call = tail call float @llvm.atan.f32(float %lv)
+  %gep.x = getelementptr inbounds float, ptr %x, i64 %iv
+  store float %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare double @llvm.atan.f64(double) nounwind readnone
+define void @atan_v2f64_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @atan_v2f64_intrinsic(
+; CHECK: call <2 x double> @_simd_atan_d2(<2 x double>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds double, ptr %y, i64 %iv
+  %lv = load double, ptr %gep.y, align 4
+  %call = tail call double @llvm.atan.f64(double %lv)
+  %gep.x = getelementptr inbounds double, ptr %x, i64 %iv
+  store double %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare float @llvm.cosh.f32(float) nounwind readnone
+define void @cosh_v4f32_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @cosh_v4f32_intrinsic(
+; CHECK: call <4 x float> @_simd_cosh_f4(<4 x float>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds float, ptr %y, i64 %iv
+  %lv = load float, ptr %gep.y, align 4
+  %call = tail call float @llvm.cosh.f32(float %lv)
+  %gep.x = getelementptr inbounds float, ptr %x, i64 %iv
+  store float %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare double @llvm.cosh.f64(double) nounwind readnone
+define void @cosh_v2f64_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @cosh_v2f64_intrinsic(
+; CHECK: call <2 x double> @_simd_cosh_d2(<2 x double>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds double, ptr %y, i64 %iv
+  %lv = load double, ptr %gep.y, align 4
+  %call = tail call double @llvm.cosh.f64(double %lv)
+  %gep.x = getelementptr inbounds double, ptr %x, i64 %iv
+  store double %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare float @llvm.sinh.f32(float) nounwind readnone
+define void @sinh_v4f32_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @sinh_v4f32_intrinsic(
+; CHECK: call <4 x float> @_simd_sinh_f4(<4 x float>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds float, ptr %y, i64 %iv
+  %lv = load float, ptr %gep.y, align 4
+  %call = tail call float @llvm.sinh.f32(float %lv)
+  %gep.x = getelementptr inbounds float, ptr %x, i64 %iv
+  store float %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare double @llvm.sinh.f64(double) nounwind readnone
+define void @sinh_v2f64_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @sinh_v2f64_intrinsic(
+; CHECK: call <2 x double> @_simd_sinh_d2(<2 x double>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds double, ptr %y, i64 %iv
+  %lv = load double, ptr %gep.y, align 4
+  %call = tail call double @llvm.sinh.f64(double %lv)
+  %gep.x = getelementptr inbounds double, ptr %x, i64 %iv
+  store double %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare float @llvm.tanh.f32(float) nounwind readnone
+define void @tanh_v4f32_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @tanh_v4f32_intrinsic(
+; CHECK: call <4 x float> @_simd_tanh_f4(<4 x float>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds float, ptr %y, i64 %iv
+  %lv = load float, ptr %gep.y, align 4
+  %call = tail call float @llvm.tanh.f32(float %lv)
+  %gep.x = getelementptr inbounds float, ptr %x, i64 %iv
+  store float %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare double @llvm.tanh.f64(double) nounwind readnone
+define void @tanh_v2f64_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @tanh_v2f64_intrinsic(
+; CHECK: call <2 x double> @_simd_tanh_d2(<2 x double>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds double, ptr %y, i64 %iv
+  %lv = load double, ptr %gep.y, align 4
+  %call = tail call double @llvm.tanh.f64(double %lv)
+  %gep.x = getelementptr inbounds double, ptr %x, i64 %iv
+  store double %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+
 declare float @cbrtf(float) nounwind readnone
 define void @cbrtf_v4f32(i64 %n, ptr noalias %y, ptr noalias %x) {
 ; CHECK-LABEL: @cbrtf_v4f32(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
index f0ae0093b2641..a107013124a71 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(cos|exp|log|sin|pow|ceil|copysign|fabs|floor|fma|m..num|nearbyint|rint|round|sqrt|tan|trunc)" --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(acos|asin|atan|cos|cosh|exp|log|sin|sinh|pow|ceil|copysign|fabs|floor|fma|m..num|nearbyint|rint|round|sqrt|tan|tanh|trunc)" --version 2
 
 ; RUN: opt -mattr=+neon -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=SLEEF-NEON
 ; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s --check-prefix=SLEEF-SVE
@@ -12,24 +12,243 @@ target triple = "aarch64-unknown-linux-gnu"
 ; are checking fixed width vectorization with NEON and scalable vectorization
 ; with SVE.
 
+declare double @llvm.acos.f64(double)
+declare float @llvm.acos.f32(float)
+
+define void @acos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @acos_f64
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_acos(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @acos_f64
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @acos_f64
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vacosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @acos_f64
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds double, ptr %in.ptr, i64 %iv
+  %in = load double, ptr %in.gep, align 8
+  %call = tail call double @llvm.acos.f64(double %in)
+  %out.gep = getelementptr inbounds double, ptr %out.ptr, i64 %iv
+  store double %call, ptr %out.gep, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @acos_f32(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @acos_f32
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_acosf(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @acos_f32
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @acos_f32
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vacosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @acos_f32
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds float, ptr %in.ptr, i64 %iv
+  %in = load float, ptr %in.gep, align 8
+  %call = tail call float @llvm.acos.f32(float %in)
+  %out.gep = getelementptr inbounds float, ptr %out.ptr, i64 %iv
+  store float %call, ptr %out.gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.asin.f64(double)
+declare float @llvm.asin.f32(float)
+
+define void @asin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @asin_f64
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_asin(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @asin_f64
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @asin_f64
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vasinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @asin_f64
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds double, ptr %in.ptr, i64 %iv
+  %in = load double, ptr %in.gep, align 8
+  %call = tail call double @llvm.asin.f64(double %in)
+  %out.gep = getelementptr inbounds double, ptr %out.ptr, i64 %iv
+  store double %call, ptr %out.gep, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @asin_f32(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @asin_f32
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_asinf(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @asin_f32
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @asin_f32
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vasinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @asin_f32
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds float, ptr %in.ptr, i64 %iv
+  %in = load float, ptr %in.gep, align 8
+  %call = tail call float @llvm.asin.f32(float %in)
+  %out.gep = getelementptr inbounds float, ptr %out.ptr, i64 %iv
+  store float %call, ptr %out.gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.atan.f64(double)
+declare float @llvm.atan.f32(float)
+
+define void @atan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @atan_f64
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_atan(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @atan_f64
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @atan_f64
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vatanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @atan_f64
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds double, ptr %in.ptr, i64 %iv
+  %in = load double, ptr %in.gep, align 8
+  %call = tail call double @llvm.atan.f64(double %in)
+  %out.gep = getelementptr inbounds double, ptr %out.ptr, i64 %iv
+  store double %call, ptr %out.gep, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @atan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @atan_f32
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_atanf(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @atan_f32
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @atan_f32
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vatanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @atan_f32
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds float, ptr %in.ptr, i64 %iv
+  %in = load float, ptr %in.gep, align 8
+  %call = tail call float @llvm.atan.f32(float %in)
+  %out.gep = getelementptr inbounds float, ptr %out.ptr, i64 %iv
+  store float %call, ptr %out.gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
 declare double @llvm.ceil.f64(double)
 declare float @llvm.ceil.f32(float)
 
 define void @ceil_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @ceil_f64
-; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
 ; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @ceil_f64
-; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
 ; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ceil_f64
-; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
 ; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ceil_f64
-; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
 ; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
@@ -231,6 +450,79 @@ define void @cos_f32(ptr noalias %in.ptr, ptr %out.ptr) {
   ret void
 }
 
+declare double @llvm.cosh.f64(double)
+declare float @llvm.cosh.f32(float)
+
+define void @cosh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @cosh_f64
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_cosh(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @cosh_f64
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @cosh_f64
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vcoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @cosh_f64
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds double, ptr %in.ptr, i64 %iv
+  %in = load double, ptr %in.gep, align 8
+  %call = tail call double @llvm.cosh.f64(double %in)
+  %out.gep = getelementptr inbounds double, ptr %out.ptr, i64 %iv
+  store double %call, ptr %out.gep, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @cosh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @cosh_f32
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_coshf(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @cosh_f32
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @cosh_f32
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vcoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @cosh_f32
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds float, ptr %in.ptr, i64 %iv
+  %in = load float, ptr %in.gep, align 8
+  %call = tail call float @llvm.cosh.f32(float %in)
+  %out.gep = getelementptr inbounds float, ptr %out.ptr, i64 %iv
+  store float %call, ptr %out.gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
 declare double @llvm.exp.f64(double)
 declare float @llvm.exp.f32(float)
 
@@ -1399,6 +1691,79 @@ define void @sin_f32(ptr noalias %in.ptr, ptr %out.ptr) {
   ret void
 }
 
+declare double @llvm.sinh.f64(double)
+declare float @llvm.sinh.f32(float)
+
+define void @sinh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @sinh_f64
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_sinh(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @sinh_f64
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @sinh_f64
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vsinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @sinh_f64
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds double, ptr %in.ptr, i64 %iv
+  %in = load double, ptr %in.gep, align 8
+  %call = tail call double @llvm.sinh.f64(double %in)
+  %out.gep = getelementptr inbounds double, ptr %out.ptr, i64 %iv
+  store double %call, ptr %out.gep, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @sinh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @sinh_f32
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_sinhf(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @sinh_f32
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @sinh_f32
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vsinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @sinh_f32
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds float, ptr %in.ptr, i64 %iv
+  %in = load float, ptr %in.gep, align 8
+  %call = tail call float @llvm.sinh.f32(float %in)
+  %out.gep = getelementptr inbounds float, ptr %out.ptr, i64 %iv
+  store float %call, ptr %out.gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
 declare double @llvm.sqrt.f64(double)
 declare float @llvm.sqrt.f32(float)
 
@@ -1545,6 +1910,79 @@ define void @tan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
   ret void
 }
 
+declare double @llvm.tanh.f64(double)
+declare float @llvm.tanh.f32(float)
+
+define void @tanh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @tanh_f64
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_tanh(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @tanh_f64
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @tanh_f64
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vtanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @tanh_f64
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds double, ptr %in.ptr, i64 %iv
+  %in = load double, ptr %in.gep, align 8
+  %call = tail call double @llvm.tanh.f64(double %in)
+  %out.gep = getelementptr inbounds double, ptr %out.ptr, i64 %iv
+  store double %call, ptr %out.gep, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @tanh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @tanh_f32
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_tanhf(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @tanh_f32
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @tanh_f32
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vtanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @tanh_f32
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds float, ptr %in.ptr, i64 %iv
+  %in = load float, ptr %in.gep, align 8
+  %call = tail call float @llvm.tanh.f32(float %in)
+  %out.gep = getelementptr inbounds float, ptr %out.ptr, i64 %iv
+  store float %call, ptr %out.gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
 declare double @llvm.trunc.f64(double)
 declare float @llvm.trunc.f32(float)
 

From def3944df822687fa04626a6750ccdea156b3870 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Fri, 19 Jul 2024 10:18:58 -0400
Subject: [PATCH 199/486] [WebAssembly] Add Support for Arc and Hyperbolic trig
 llvm intrinsics (#98755)

## Change:
- WebAssemblyRuntimeLibcallSignatures.cpp: Expose the RTLIB's for use by
WASM
-  Add trig specific test cases

## History
This change is part of an implementation of
https://github.com/llvm/llvm-project/issues/87367's investigation on
supporting IEEE math operations as intrinsics.
Which was discussed in this RFC:
https://discourse.llvm.org/t/rfc-all-the-math-intrinsics/78294

This change adds wasm lowering cases for `acos`, `asin`, `atan`, `cosh`,
`sinh`, and `tanh`.

https://github.com/llvm/llvm-project/issues/70079
https://github.com/llvm/llvm-project/issues/70080
https://github.com/llvm/llvm-project/issues/70081
https://github.com/llvm/llvm-project/issues/70083
https://github.com/llvm/llvm-project/issues/70084
https://github.com/llvm/llvm-project/issues/95966

## Why Web Assembly?
From past changes to try and support constraint intrinsics the changes
to the trig builtins to emit intrinsics\constraint intrinsics broke the
WASM build. This is an attempt to preempt any such build break.

- https://github.com/llvm/llvm-project/pull/95082
-
https://github.com/llvm/llvm-project/pull/94559#issuecomment-2159923215
---
 .../WebAssemblyRuntimeLibcallSignatures.cpp   |  18 ++
 .../test/CodeGen/WebAssembly/libcalls-trig.ll | 234 ++++++++++++++++++
 2 files changed, 252 insertions(+)
 create mode 100644 llvm/test/CodeGen/WebAssembly/libcalls-trig.ll

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
index e7810e18d44d4..ba3ab5164af26 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
@@ -204,6 +204,24 @@ struct RuntimeLibcallSignatureTable {
     Table[RTLIB::TAN_F32] = f32_func_f32;
     Table[RTLIB::TAN_F64] = f64_func_f64;
     Table[RTLIB::TAN_F128] = i64_i64_func_i64_i64;
+    Table[RTLIB::ASIN_F32] = f32_func_f32;
+    Table[RTLIB::ASIN_F64] = f64_func_f64;
+    Table[RTLIB::ASIN_F128] = i64_i64_func_i64_i64;
+    Table[RTLIB::ACOS_F32] = f32_func_f32;
+    Table[RTLIB::ACOS_F64] = f64_func_f64;
+    Table[RTLIB::ACOS_F128] = i64_i64_func_i64_i64;
+    Table[RTLIB::ATAN_F32] = f32_func_f32;
+    Table[RTLIB::ATAN_F64] = f64_func_f64;
+    Table[RTLIB::ATAN_F128] = i64_i64_func_i64_i64;
+    Table[RTLIB::SINH_F32] = f32_func_f32;
+    Table[RTLIB::SINH_F64] = f64_func_f64;
+    Table[RTLIB::SINH_F128] = i64_i64_func_i64_i64;
+    Table[RTLIB::COSH_F32] = f32_func_f32;
+    Table[RTLIB::COSH_F64] = f64_func_f64;
+    Table[RTLIB::COSH_F128] = i64_i64_func_i64_i64;
+    Table[RTLIB::TANH_F32] = f32_func_f32;
+    Table[RTLIB::TANH_F64] = f64_func_f64;
+    Table[RTLIB::TANH_F128] = i64_i64_func_i64_i64;
     Table[RTLIB::SINCOS_F32] = func_f32_iPTR_iPTR;
     Table[RTLIB::SINCOS_F64] = func_f64_iPTR_iPTR;
     Table[RTLIB::SINCOS_F128] = func_i64_i64_iPTR_iPTR;
diff --git a/llvm/test/CodeGen/WebAssembly/libcalls-trig.ll b/llvm/test/CodeGen/WebAssembly/libcalls-trig.ll
new file mode 100644
index 0000000000000..8bc9c043fcf8a
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/libcalls-trig.ll
@@ -0,0 +1,234 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck %s
+
+; Test a subset of compiler-rt/libm libcalls expected to be emitted by the wasm backend
+
+target triple = "wasm32-unknown-unknown"
+
+declare fp128 @llvm.sin.f128(fp128)
+declare fp128 @llvm.cos.f128(fp128)
+declare fp128 @llvm.tan.f128(fp128)
+declare fp128 @llvm.asin.f128(fp128)
+declare fp128 @llvm.acos.f128(fp128)
+declare fp128 @llvm.atan.f128.i32(fp128)
+declare fp128 @llvm.sinh.f128(fp128)
+declare fp128 @llvm.cosh.f128(fp128)
+declare fp128 @llvm.tanh.f128(fp128)
+
+declare double @llvm.sin.f64(double)
+declare double @llvm.cos.f64(double)
+declare double @llvm.tan.f64(double)
+declare double @llvm.asin.f64(double)
+declare double @llvm.acos.f64(double)
+declare double @llvm.atan.f64(double)
+declare double @llvm.sinh.f64(double)
+declare double @llvm.cosh.f64(double)
+declare double @llvm.tanh.f64(double)
+
+declare float @llvm.sin.f32(float)
+declare float @llvm.cos.f32(float)
+declare float @llvm.tan.f32(float)
+declare float @llvm.asin.f32(float)
+declare float @llvm.acos.f32(float)
+declare float @llvm.atan.f32(float)
+declare float @llvm.sinh.f32(float)
+declare float @llvm.cosh.f32(float)
+declare float @llvm.tanh.f32(float)
+
+
+define fp128 @fp128libcalls(fp128 %x) {
+  ; compiler-rt call
+; CHECK-LABEL: fp128libcalls:
+; CHECK:         .functype fp128libcalls (i32, i64, i64) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get      $push28=, __stack_pointer
+; CHECK-NEXT:    i32.const       $push29=, 144
+; CHECK-NEXT:    i32.sub         $push73=, $pop28, $pop29
+; CHECK-NEXT:    local.tee       $push72=, 3, $pop73
+; CHECK-NEXT:    global.set      __stack_pointer, $pop72
+; CHECK-NEXT:    local.get       $push74=, 3
+; CHECK-NEXT:    i32.const       $push62=, 128
+; CHECK-NEXT:    i32.add         $push63=, $pop74, $pop62
+; CHECK-NEXT:    local.get       $push76=, 1
+; CHECK-NEXT:    local.get       $push75=, 2
+; CHECK-NEXT:    call    sinl, $pop63, $pop76, $pop75
+; CHECK-NEXT:    local.get       $push77=, 3
+; CHECK-NEXT:    i32.const       $push58=, 112
+; CHECK-NEXT:    i32.add         $push59=, $pop77, $pop58
+; CHECK-NEXT:    local.get       $push78=, 3
+; CHECK-NEXT:    i64.load        $push3=, 128($pop78)
+; CHECK-NEXT:    local.get       $push79=, 3
+; CHECK-NEXT:    i32.const       $push60=, 128
+; CHECK-NEXT:    i32.add         $push61=, $pop79, $pop60
+; CHECK-NEXT:    i32.const       $push0=, 8
+; CHECK-NEXT:    i32.add         $push1=, $pop61, $pop0
+; CHECK-NEXT:    i64.load        $push2=, 0($pop1)
+; CHECK-NEXT:    call    cosl, $pop59, $pop3, $pop2
+; CHECK-NEXT:    local.get       $push80=, 3
+; CHECK-NEXT:    i32.const       $push54=, 96
+; CHECK-NEXT:    i32.add         $push55=, $pop80, $pop54
+; CHECK-NEXT:    local.get       $push81=, 3
+; CHECK-NEXT:    i64.load        $push6=, 112($pop81)
+; CHECK-NEXT:    local.get       $push82=, 3
+; CHECK-NEXT:    i32.const       $push56=, 112
+; CHECK-NEXT:    i32.add         $push57=, $pop82, $pop56
+; CHECK-NEXT:    i32.const       $push71=, 8
+; CHECK-NEXT:    i32.add         $push4=, $pop57, $pop71
+; CHECK-NEXT:    i64.load        $push5=, 0($pop4)
+; CHECK-NEXT:    call    tanl, $pop55, $pop6, $pop5
+; CHECK-NEXT:    local.get       $push83=, 3
+; CHECK-NEXT:    i32.const       $push50=, 80
+; CHECK-NEXT:    i32.add         $push51=, $pop83, $pop50
+; CHECK-NEXT:    local.get       $push84=, 3
+; CHECK-NEXT:    i64.load        $push9=, 96($pop84)
+; CHECK-NEXT:    local.get       $push85=, 3
+; CHECK-NEXT:    i32.const       $push52=, 96
+; CHECK-NEXT:    i32.add         $push53=, $pop85, $pop52
+; CHECK-NEXT:    i32.const       $push70=, 8
+; CHECK-NEXT:    i32.add         $push7=, $pop53, $pop70
+; CHECK-NEXT:    i64.load        $push8=, 0($pop7)
+; CHECK-NEXT:    call    asinl, $pop51, $pop9, $pop8
+; CHECK-NEXT:    local.get       $push86=, 3
+; CHECK-NEXT:    i32.const       $push46=, 64
+; CHECK-NEXT:    i32.add         $push47=, $pop86, $pop46
+; CHECK-NEXT:    local.get       $push87=, 3
+; CHECK-NEXT:    i64.load        $push12=, 80($pop87)
+; CHECK-NEXT:    local.get       $push88=, 3
+; CHECK-NEXT:    i32.const       $push48=, 80
+; CHECK-NEXT:    i32.add         $push49=, $pop88, $pop48
+; CHECK-NEXT:    i32.const       $push69=, 8
+; CHECK-NEXT:    i32.add         $push10=, $pop49, $pop69
+; CHECK-NEXT:    i64.load        $push11=, 0($pop10)
+; CHECK-NEXT:    call    acosl, $pop47, $pop12, $pop11
+; CHECK-NEXT:    local.get       $push89=, 3
+; CHECK-NEXT:    i32.const       $push42=, 48
+; CHECK-NEXT:    i32.add         $push43=, $pop89, $pop42
+; CHECK-NEXT:    local.get       $push90=, 3
+; CHECK-NEXT:    i64.load        $push15=, 64($pop90)
+; CHECK-NEXT:    local.get       $push91=, 3
+; CHECK-NEXT:    i32.const       $push44=, 64
+; CHECK-NEXT:    i32.add         $push45=, $pop91, $pop44
+; CHECK-NEXT:    i32.const       $push68=, 8
+; CHECK-NEXT:    i32.add         $push13=, $pop45, $pop68
+; CHECK-NEXT:    i64.load        $push14=, 0($pop13)
+; CHECK-NEXT:    call    atanl, $pop43, $pop15, $pop14
+; CHECK-NEXT:    local.get       $push92=, 3
+; CHECK-NEXT:    i32.const       $push38=, 32
+; CHECK-NEXT:    i32.add         $push39=, $pop92, $pop38
+; CHECK-NEXT:    local.get       $push93=, 3
+; CHECK-NEXT:    i64.load        $push18=, 48($pop93)
+; CHECK-NEXT:    local.get       $push94=, 3
+; CHECK-NEXT:    i32.const       $push40=, 48
+; CHECK-NEXT:    i32.add         $push41=, $pop94, $pop40
+; CHECK-NEXT:    i32.const       $push67=, 8
+; CHECK-NEXT:    i32.add         $push16=, $pop41, $pop67
+; CHECK-NEXT:    i64.load        $push17=, 0($pop16)
+; CHECK-NEXT:    call    sinhl, $pop39, $pop18, $pop17
+; CHECK-NEXT:    local.get       $push95=, 3
+; CHECK-NEXT:    i32.const       $push34=, 16
+; CHECK-NEXT:    i32.add         $push35=, $pop95, $pop34
+; CHECK-NEXT:    local.get       $push96=, 3
+; CHECK-NEXT:    i64.load        $push21=, 32($pop96)
+; CHECK-NEXT:    local.get       $push97=, 3
+; CHECK-NEXT:    i32.const       $push36=, 32
+; CHECK-NEXT:    i32.add         $push37=, $pop97, $pop36
+; CHECK-NEXT:    i32.const       $push66=, 8
+; CHECK-NEXT:    i32.add         $push19=, $pop37, $pop66
+; CHECK-NEXT:    i64.load        $push20=, 0($pop19)
+; CHECK-NEXT:    call    coshl, $pop35, $pop21, $pop20
+; CHECK-NEXT:    local.get       $push100=, 3
+; CHECK-NEXT:    local.get       $push98=, 3
+; CHECK-NEXT:    i64.load        $push24=, 16($pop98)
+; CHECK-NEXT:    local.get       $push99=, 3
+; CHECK-NEXT:    i32.const       $push32=, 16
+; CHECK-NEXT:    i32.add         $push33=, $pop99, $pop32
+; CHECK-NEXT:    i32.const       $push65=, 8
+; CHECK-NEXT:    i32.add         $push22=, $pop33, $pop65
+; CHECK-NEXT:    i64.load        $push23=, 0($pop22)
+; CHECK-NEXT:    call    tanhl, $pop100, $pop24, $pop23
+; CHECK-NEXT:    local.get       $push102=, 0
+; CHECK-NEXT:    local.get       $push101=, 3
+; CHECK-NEXT:    i32.const       $push64=, 8
+; CHECK-NEXT:    i32.add         $push25=, $pop101, $pop64
+; CHECK-NEXT:    i64.load        $push26=, 0($pop25)
+; CHECK-NEXT:    i64.store       8($pop102), $pop26
+; CHECK-NEXT:    local.get       $push104=, 0
+; CHECK-NEXT:    local.get       $push103=, 3
+; CHECK-NEXT:    i64.load        $push27=, 0($pop103)
+; CHECK-NEXT:    i64.store       0($pop104), $pop27
+; CHECK-NEXT:    local.get       $push105=, 3
+; CHECK-NEXT:    i32.const       $push30=, 144
+; CHECK-NEXT:    i32.add         $push31=, $pop105, $pop30
+; CHECK-NEXT:    global.set      __stack_pointer, $pop31
+; CHECK-NEXT:    return
+  ; libm calls
+  %d = call fp128 @llvm.sin.f128(fp128 %x)
+  %e = call fp128 @llvm.cos.f128(fp128 %d)
+  %f = call fp128 @llvm.tan.f128(fp128 %e)
+  %g = call fp128 @llvm.asin.f128.i32(fp128 %f)
+  %h = call fp128 @llvm.acos.f128(fp128 %g)
+  %i = call fp128 @llvm.atan.f128(fp128 %h)
+  %a = call fp128 @llvm.sinh.f128(fp128 %i)
+  %b = call fp128 @llvm.cosh.f128(fp128 %a)
+  %c = call fp128 @llvm.tanh.f128(fp128 %b)
+  ret fp128 %c
+}
+
+define double @f64libcalls(double %x) {
+; CHECK-LABEL: f64libcalls:
+; CHECK:         .functype f64libcalls (f64) -> (f64)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get $push9=, 0
+; CHECK-NEXT:    call    $push0=, sin, $pop9
+; CHECK-NEXT:    call    $push1=, cos, $pop0
+; CHECK-NEXT:    call    $push2=, tan, $pop1
+; CHECK-NEXT:    call    $push3=, asin, $pop2
+; CHECK-NEXT:    call    $push4=, acos, $pop3
+; CHECK-NEXT:    call    $push5=, atan, $pop4
+; CHECK-NEXT:    call    $push6=, sinh, $pop5
+; CHECK-NEXT:    call    $push7=, cosh, $pop6
+; CHECK-NEXT:    call    $push8=, tanh, $pop7  
+; CHECK-NEXT:    return $pop8
+
+
+ %k = call double @llvm.sin.f64(double %x)
+ %a = call double @llvm.cos.f64(double %k)
+ %b = call double @llvm.tan.f64(double %a)
+ %c = call double @llvm.asin.f64(double %b)
+ %d = call double @llvm.acos.f64(double %c)
+ %e = call double @llvm.atan.f64(double %d)
+ %f = call double @llvm.sinh.f64(double %e)
+ %g = call double @llvm.cosh.f64(double %f)
+ %h = call double @llvm.tanh.f64(double %g)
+ ret double %h
+}
+
+define float @f32libcalls(float %x) {
+; CHECK-LABEL: f32libcalls:
+; CHECK:         .functype f32libcalls (f32) -> (f32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get $push9=, 0
+; CHECK-NEXT:    call    $push0=, sinf, $pop9
+; CHECK-NEXT:    call    $push1=, cosf, $pop0
+; CHECK-NEXT:    call    $push2=, tanf, $pop1
+; CHECK-NEXT:    call    $push3=, asinf, $pop2
+; CHECK-NEXT:    call    $push4=, acosf, $pop3
+; CHECK-NEXT:    call    $push5=, atanf, $pop4
+; CHECK-NEXT:    call    $push6=, sinhf, $pop5
+; CHECK-NEXT:    call    $push7=, coshf, $pop6
+; CHECK-NEXT:    call    $push8=, tanhf, $pop7  
+; CHECK-NEXT:    return $pop8
+
+
+ %k = call float @llvm.sin.f32(float %x)
+ %a = call float @llvm.cos.f32(float %k)
+ %b = call float @llvm.tan.f32(float %a)
+ %c = call float @llvm.asin.f32(float %b)
+ %d = call float @llvm.acos.f32(float %c)
+ %e = call float @llvm.atan.f32(float %d)
+ %f = call float @llvm.sinh.f32(float %e)
+ %g = call float @llvm.cosh.f32(float %f)
+ %h = call float @llvm.tanh.f32(float %g)
+ ret float %h
+}

From a14baec0f33ec495e7d96e2ef99ed238f0d2069e Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Fri, 19 Jul 2024 10:19:41 -0400
Subject: [PATCH 200/486] [clang] Emit constraint intrinsics for arc and
 hyperbolic trig clang builtins (#98949)

## Change(s)
- `Builtins.td` - Add f16 support for libm arc and hyperbolic trig
functions
- `CGBuiltin.cpp` - Emit constraint intrinsics for trig clang builtins

## History
This change is part of an implementation of
https://github.com/llvm/llvm-project/issues/87367's investigation on
supporting IEEE math operations as intrinsics. Which was discussed in
this RFC:
https://discourse.llvm.org/t/rfc-all-the-math-intrinsics/78294

This change adds wasm lowering cases for `acos`, `asin`, `atan`, `cosh`,
`sinh`, and `tanh`.

https://github.com/llvm/llvm-project/issues/70079
https://github.com/llvm/llvm-project/issues/70080
https://github.com/llvm/llvm-project/issues/70081
https://github.com/llvm/llvm-project/issues/70083
https://github.com/llvm/llvm-project/issues/70084
https://github.com/llvm/llvm-project/issues/95966

## Precursor PR(s)

Note this PR needs Merge after:
- #98937
- #98755
---
 clang/include/clang/Basic/Builtins.td         | 36 ++++-----
 clang/lib/CodeGen/CGBuiltin.cpp               | 66 +++++++++++++++++
 clang/test/CodeGen/X86/math-builtins.c        | 48 ++++++------
 .../test/CodeGen/constrained-math-builtins.c  | 41 +++++++++++
 clang/test/CodeGen/libcalls.c                 |  8 +-
 clang/test/CodeGen/math-libcalls.c            | 73 ++++++++++---------
 clang/test/CodeGenOpenCL/builtins-f16.cl      | 18 +++++
 7 files changed, 208 insertions(+), 82 deletions(-)

diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index f5b15cf90d1f8..4133f6ff40cf3 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -83,11 +83,11 @@ class BitInt_Long_LongLongTemplate :
 // - _Constant: Argument has to constant-fold to an integer constant expression
 
 // __fp16 and __float128 builtin variants of libc/libm functions.
-def AcosF128 : Builtin {
-  let Spellings = ["__builtin_acosf128"];
+def AcosF16F128 : Builtin, F16F128MathTemplate {
+  let Spellings = ["__builtin_acos"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow,
                     ConstIgnoringErrnoAndExceptions];
-  let Prototype = "__float128(__float128)";
+  let Prototype = "T(T)";
 }
 
 def AcoshF128 : Builtin {
@@ -97,11 +97,11 @@ def AcoshF128 : Builtin {
   let Prototype = "__float128(__float128)";
 }
 
-def AsinF128 : Builtin {
-  let Spellings = ["__builtin_asinf128"];
+def AsinF16F128 : Builtin, F16F128MathTemplate {
+  let Spellings = ["__builtin_asin"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow,
                     ConstIgnoringErrnoAndExceptions];
-  let Prototype = "__float128(__float128)";
+  let Prototype = "T(T)";
 }
 
 def AsinhF128 : Builtin {
@@ -111,11 +111,11 @@ def AsinhF128 : Builtin {
   let Prototype = "__float128(__float128)";
 }
 
-def AtanF128 : Builtin {
-  let Spellings = ["__builtin_atanf128"];
+def AtanF16F128 : Builtin, F16F128MathTemplate {
+  let Spellings = ["__builtin_atan"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow,
                     ConstIgnoringErrnoAndExceptions];
-  let Prototype = "__float128(__float128)";
+  let Prototype = "T(T)";
 }
 
 def AtanhF128 : Builtin {
@@ -143,10 +143,10 @@ def CosF16F128 : Builtin, F16F128MathTemplate {
   let Prototype = "T(T)";
 }
 
-def CoshF128 : Builtin {
-  let Spellings = ["__builtin_coshf128"];
+def CoshF16F128 : Builtin, F16F128MathTemplate {
+  let Spellings = ["__builtin_cosh"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, ConstIgnoringErrnoAndExceptions];
-  let Prototype = "__float128(__float128)";
+  let Prototype = "T(T)";
 }
 
 def ErfF128 : Builtin {
@@ -468,11 +468,11 @@ def SinF16F128 : Builtin, F16F128MathTemplate {
   let Prototype = "T(T)";
 }
 
-def SinhF128 : Builtin {
-  let Spellings = ["__builtin_sinhf128"];
+def SinhF16F128 : Builtin, F16F128MathTemplate {
+  let Spellings = ["__builtin_sinh"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow,
                     ConstIgnoringErrnoAndExceptions];
-  let Prototype = "__float128(__float128)";
+  let Prototype = "T(T)";
 }
 
 def SqrtF16F128 : Builtin, F16F128MathTemplate {
@@ -489,11 +489,11 @@ def TanF16F128 : Builtin, F16F128MathTemplate {
   let Prototype = "T(T)";
 }
 
-def TanhF128 : Builtin {
-  let Spellings = ["__builtin_tanhf128"];
+def TanhF16F128 : Builtin, F16F128MathTemplate {
+  let Spellings = ["__builtin_tanh"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow,
                     ConstIgnoringErrnoAndExceptions];
-  let Prototype = "__float128(__float128)";
+  let Prototype = "T(T)";
 }
 
 def TgammaF128 : Builtin {
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 2d32a8582cfdf..5639239359ab8 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2667,6 +2667,39 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   }
   if (GenerateIntrinsics) {
     switch (BuiltinIDIfNoAsmLabel) {
+    case Builtin::BIacos:
+    case Builtin::BIacosf:
+    case Builtin::BIacosl:
+    case Builtin::BI__builtin_acos:
+    case Builtin::BI__builtin_acosf:
+    case Builtin::BI__builtin_acosf16:
+    case Builtin::BI__builtin_acosl:
+    case Builtin::BI__builtin_acosf128:
+      return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
+          *this, E, Intrinsic::acos, Intrinsic::experimental_constrained_acos));
+
+    case Builtin::BIasin:
+    case Builtin::BIasinf:
+    case Builtin::BIasinl:
+    case Builtin::BI__builtin_asin:
+    case Builtin::BI__builtin_asinf:
+    case Builtin::BI__builtin_asinf16:
+    case Builtin::BI__builtin_asinl:
+    case Builtin::BI__builtin_asinf128:
+      return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
+          *this, E, Intrinsic::asin, Intrinsic::experimental_constrained_asin));
+
+    case Builtin::BIatan:
+    case Builtin::BIatanf:
+    case Builtin::BIatanl:
+    case Builtin::BI__builtin_atan:
+    case Builtin::BI__builtin_atanf:
+    case Builtin::BI__builtin_atanf16:
+    case Builtin::BI__builtin_atanl:
+    case Builtin::BI__builtin_atanf128:
+      return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
+          *this, E, Intrinsic::atan, Intrinsic::experimental_constrained_atan));
+
     case Builtin::BIceil:
     case Builtin::BIceilf:
     case Builtin::BIceill:
@@ -2702,6 +2735,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                                    Intrinsic::cos,
                                    Intrinsic::experimental_constrained_cos));
 
+    case Builtin::BIcosh:
+    case Builtin::BIcoshf:
+    case Builtin::BIcoshl:
+    case Builtin::BI__builtin_cosh:
+    case Builtin::BI__builtin_coshf:
+    case Builtin::BI__builtin_coshf16:
+    case Builtin::BI__builtin_coshl:
+    case Builtin::BI__builtin_coshf128:
+      return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
+          *this, E, Intrinsic::cosh, Intrinsic::experimental_constrained_cosh));
+
     case Builtin::BIexp:
     case Builtin::BIexpf:
     case Builtin::BIexpl:
@@ -2918,6 +2962,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                                    Intrinsic::sin,
                                    Intrinsic::experimental_constrained_sin));
 
+    case Builtin::BIsinh:
+    case Builtin::BIsinhf:
+    case Builtin::BIsinhl:
+    case Builtin::BI__builtin_sinh:
+    case Builtin::BI__builtin_sinhf:
+    case Builtin::BI__builtin_sinhf16:
+    case Builtin::BI__builtin_sinhl:
+    case Builtin::BI__builtin_sinhf128:
+      return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
+          *this, E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh));
+
     case Builtin::BIsqrt:
     case Builtin::BIsqrtf:
     case Builtin::BIsqrtl:
@@ -2944,6 +2999,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
           *this, E, Intrinsic::tan, Intrinsic::experimental_constrained_tan));
 
+    case Builtin::BItanh:
+    case Builtin::BItanhf:
+    case Builtin::BItanhl:
+    case Builtin::BI__builtin_tanh:
+    case Builtin::BI__builtin_tanhf:
+    case Builtin::BI__builtin_tanhf16:
+    case Builtin::BI__builtin_tanhl:
+    case Builtin::BI__builtin_tanhf128:
+      return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
+          *this, E, Intrinsic::tanh, Intrinsic::experimental_constrained_tanh));
+
     case Builtin::BItrunc:
     case Builtin::BItruncf:
     case Builtin::BItruncl:
diff --git a/clang/test/CodeGen/X86/math-builtins.c b/clang/test/CodeGen/X86/math-builtins.c
index d26db19574051..48465df21cca1 100644
--- a/clang/test/CodeGen/X86/math-builtins.c
+++ b/clang/test/CodeGen/X86/math-builtins.c
@@ -168,10 +168,10 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
   /* math */
   __builtin_acos(f);       __builtin_acosf(f);      __builtin_acosl(f); __builtin_acosf128(f);
 
-// NO__ERRNO: declare double @acos(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @acosf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @acosl(x86_fp80 noundef) [[READNONE]]
-// NO__ERRNO: declare fp128 @acosf128(fp128 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.acos.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.acos.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.acos.f80(x86_fp80) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare fp128 @llvm.acos.f128(fp128) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @acos(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @acosf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @acosl(x86_fp80 noundef) [[NOT_READNONE]]
@@ -190,10 +190,10 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
 
   __builtin_asin(f);       __builtin_asinf(f);      __builtin_asinl(f); __builtin_asinf128(f);
 
-// NO__ERRNO: declare double @asin(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @asinf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @asinl(x86_fp80 noundef) [[READNONE]]
-// NO__ERRNO: declare fp128 @asinf128(fp128 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.asin.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.asin.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.asin.f80(x86_fp80) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare fp128 @llvm.asin.f128(fp128) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @asin(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @asinf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @asinl(x86_fp80 noundef) [[NOT_READNONE]]
@@ -212,10 +212,10 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
 
   __builtin_atan(f);       __builtin_atanf(f);      __builtin_atanl(f); __builtin_atanf128(f);
 
-// NO__ERRNO: declare double @atan(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @atanf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @atanl(x86_fp80 noundef) [[READNONE]]
-// NO__ERRNO: declare fp128 @atanf128(fp128 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.atan.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.atan.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.atan.f80(x86_fp80) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare fp128 @llvm.atan.f128(fp128) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @atan(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @atanf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @atanl(x86_fp80 noundef) [[NOT_READNONE]]
@@ -267,10 +267,10 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
 
   __builtin_cosh(f);       __builtin_coshf(f);      __builtin_coshl(f); __builtin_coshf128(f);
 
-// NO__ERRNO: declare double @cosh(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @coshf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @coshl(x86_fp80 noundef) [[READNONE]]
-// NO__ERRNO: declare fp128 @coshf128(fp128 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.cosh.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.cosh.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.cosh.f80(x86_fp80) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare fp128 @llvm.cosh.f128(fp128) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @cosh(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @coshf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @coshl(x86_fp80 noundef) [[NOT_READNONE]]
@@ -656,10 +656,10 @@ __builtin_sin(f);        __builtin_sinf(f);       __builtin_sinl(f); __builtin_s
 
 __builtin_sinh(f);       __builtin_sinhf(f);      __builtin_sinhl(f); __builtin_sinhf128(f);
 
-// NO__ERRNO: declare double @sinh(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @sinhf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @sinhl(x86_fp80 noundef) [[READNONE]]
-// NO__ERRNO: declare fp128 @sinhf128(fp128 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.sinh.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.sinh.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.sinh.f80(x86_fp80) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare fp128 @llvm.sinh.f128(fp128) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @sinh(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @sinhf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @sinhl(x86_fp80 noundef) [[NOT_READNONE]]
@@ -689,10 +689,10 @@ __builtin_tan(f);        __builtin_tanf(f);       __builtin_tanl(f); __builtin_t
 
 __builtin_tanh(f);       __builtin_tanhf(f);      __builtin_tanhl(f); __builtin_tanhf128(f);
 
-// NO__ERRNO: declare double @tanh(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @tanhf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @tanhl(x86_fp80 noundef) [[READNONE]]
-// NO__ERRNO: declare fp128 @tanhf128(fp128 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.tanh.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.tanh.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.tanh.f80(x86_fp80) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare fp128 @llvm.tanh.f128(fp128) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @tanh(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @tanhf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @tanhl(x86_fp80 noundef) [[NOT_READNONE]]
diff --git a/clang/test/CodeGen/constrained-math-builtins.c b/clang/test/CodeGen/constrained-math-builtins.c
index 42c9e3c5008a3..aa77620b44535 100644
--- a/clang/test/CodeGen/constrained-math-builtins.c
+++ b/clang/test/CodeGen/constrained-math-builtins.c
@@ -36,6 +36,27 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c, _
 // CHECK: call float @llvm.experimental.constrained.ldexp.f32.i32(float %{{.*}}, i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
 // CHECK: call x86_fp80 @llvm.experimental.constrained.ldexp.f80.i32(x86_fp80 %{{.*}}, i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
 
+  __builtin_acos(f);        __builtin_acosf(f);       __builtin_acosl(f); __builtin_acosf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.acos.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.acos.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.acos.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.acos.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+__builtin_asin(f);        __builtin_asinf(f);       __builtin_asinl(f); __builtin_asinf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.asin.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.asin.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.asin.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.asin.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+__builtin_atan(f);        __builtin_atanf(f);       __builtin_atanl(f); __builtin_atanf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.atan.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.atan.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.atan.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.atan.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
   __builtin_ceil(f);       __builtin_ceilf(f);      __builtin_ceill(f); __builtin_ceilf128(f);
 
 // CHECK: call double @llvm.experimental.constrained.ceil.f64(double %{{.*}}, metadata !"fpexcept.strict")
@@ -50,6 +71,13 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c, _
 // CHECK: call x86_fp80 @llvm.experimental.constrained.cos.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
 // CHECK: call fp128 @llvm.experimental.constrained.cos.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
 
+  __builtin_cosh(f);        __builtin_coshf(f);       __builtin_coshl(f); __builtin_coshf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.cosh.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.cosh.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.cosh.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.cosh.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
   __builtin_exp(f);        __builtin_expf(f);       __builtin_expl(f); __builtin_expf128(f);
 
 // CHECK: call double @llvm.experimental.constrained.exp.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -177,6 +205,13 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c, _
 // CHECK: call x86_fp80 @llvm.experimental.constrained.sin.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
 // CHECK: call fp128 @llvm.experimental.constrained.sin.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
 
+  __builtin_sinh(f);        __builtin_sinhf(f);       __builtin_sinhl(f); __builtin_sinhf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.sinh.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.sinh.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.sinh.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.sinh.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
   __builtin_sqrt(f);       __builtin_sqrtf(f);      __builtin_sqrtl(f); __builtin_sqrtf128(f);
 
 // CHECK: call double @llvm.experimental.constrained.sqrt.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -191,6 +226,12 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c, _
 // CHECK: call x86_fp80 @llvm.experimental.constrained.tan.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
 // CHECK: call fp128 @llvm.experimental.constrained.tan.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
 
+  __builtin_tanh(f);        __builtin_tanhf(f);       __builtin_tanhl(f); __builtin_tanhf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.tanh.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.tanh.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.tanh.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.tanh.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
 
   __builtin_trunc(f);      __builtin_truncf(f);     __builtin_truncl(f); __builtin_truncf128(f);
 
diff --git a/clang/test/CodeGen/libcalls.c b/clang/test/CodeGen/libcalls.c
index 645fa016f3f77..b1637121127c5 100644
--- a/clang/test/CodeGen/libcalls.c
+++ b/clang/test/CodeGen/libcalls.c
@@ -85,9 +85,9 @@ void test_builtins(double d, float f, long double ld) {
   double atan_ = atan(d);
   long double atanl_ = atanl(ld);
   float atanf_ = atanf(f);
-// CHECK-NO: declare double @atan(double noundef) [[NUW_RN:#[0-9]+]]
-// CHECK-NO: declare x86_fp80 @atanl(x86_fp80 noundef) [[NUW_RN]]
-// CHECK-NO: declare float @atanf(float noundef) [[NUW_RN]]
+// CHECK-NO: declare double @llvm.atan.f64(double) [[NUW_RNI:#[0-9]+]]
+// CHECK-NO: declare x86_fp80 @llvm.atan.f80(x86_fp80) [[NUW_RNI]]
+// CHECK-NO: declare float @llvm.atan.f32(float) [[NUW_RNI]]
 // CHECK-YES: declare double @atan(double noundef) [[NUW:#[0-9]+]]
 // CHECK-YES: declare x86_fp80 @atanl(x86_fp80 noundef) [[NUW]]
 // CHECK-YES: declare float @atanf(float noundef) [[NUW]]
@@ -95,7 +95,7 @@ void test_builtins(double d, float f, long double ld) {
   double atan2_ = atan2(d, 2);
   long double atan2l_ = atan2l(ld, ld);
   float atan2f_ = atan2f(f, f);
-// CHECK-NO: declare double @atan2(double noundef, double noundef) [[NUW_RN]]
+// CHECK-NO: declare double @atan2(double noundef, double noundef) [[NUW_RN:#[0-9]+]]
 // CHECK-NO: declare x86_fp80 @atan2l(x86_fp80 noundef, x86_fp80 noundef) [[NUW_RN]]
 // CHECK-NO: declare float @atan2f(float noundef, float noundef) [[NUW_RN]]
 // CHECK-YES: declare double @atan2(double noundef, double noundef) [[NUW]]
diff --git a/clang/test/CodeGen/math-libcalls.c b/clang/test/CodeGen/math-libcalls.c
index a249182692762..5b23a4a3faef3 100644
--- a/clang/test/CodeGen/math-libcalls.c
+++ b/clang/test/CodeGen/math-libcalls.c
@@ -121,15 +121,15 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
   /* math */
   acos(f);       acosf(f);      acosl(f);
 
-// NO__ERRNO: declare double @acos(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @acosf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @acosl(x86_fp80 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.acos.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.acos.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.acos.f80(x86_fp80) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @acos(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @acosf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @acosl(x86_fp80 noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare double @acos(double noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare float @acosf(float noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare x86_fp80 @acosl(x86_fp80 noundef) [[NOT_READNONE]]
+// HAS_MAYTRAP: declare double @llvm.experimental.constrained.acos.f64(
+// HAS_MAYTRAP: declare float  @llvm.experimental.constrained.acos.f32(
+// HAS_MAYTRAP: declare x86_fp80  @llvm.experimental.constrained.acos.f80(
 
 
   acosh(f);      acoshf(f);     acoshl(f);
@@ -146,15 +146,15 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
 
   asin(f);       asinf(f);      asinl(f);
 
-// NO__ERRNO: declare double @asin(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @asinf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @asinl(x86_fp80 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.asin.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.asin.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.asin.f80(x86_fp80) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @asin(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @asinf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @asinl(x86_fp80 noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare double @asin(double noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare float @asinf(float noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare x86_fp80 @asinl(x86_fp80 noundef) [[NOT_READNONE]]
+// HAS_MAYTRAP: declare double @llvm.experimental.constrained.asin.f64(
+// HAS_MAYTRAP: declare float @llvm.experimental.constrained.asin.f32(
+// HAS_MAYTRAP: declare x86_fp80 @llvm.experimental.constrained.asin.f80(
 
   asinh(f);      asinhf(f);     asinhl(f);
 
@@ -170,15 +170,15 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
 
   atan(f);       atanf(f);      atanl(f);
 
-// NO__ERRNO: declare double @atan(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @atanf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @atanl(x86_fp80 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.atan.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.atan.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.atan.f80(x86_fp80) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @atan(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @atanf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @atanl(x86_fp80 noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare double @atan(double noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare float @atanf(float noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare x86_fp80 @atanl(x86_fp80 noundef) [[NOT_READNONE]]
+// HAS_MAYTRAP: declare double @llvm.experimental.constrained.atan.f64(
+// HAS_MAYTRAP: declare float @llvm.experimental.constrained.atan.f32(
+// HAS_MAYTRAP: declare x86_fp80 @llvm.experimental.constrained.atan.f80(
 
   atanh(f);      atanhf(f);     atanhl(f);
 
@@ -230,15 +230,15 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
 
   cosh(f);       coshf(f);      coshl(f);
 
-// NO__ERRNO: declare double @cosh(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @coshf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @coshl(x86_fp80 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.cosh.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.cosh.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.cosh.f80(x86_fp80) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @cosh(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @coshf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @coshl(x86_fp80 noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare double @cosh(double noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare float @coshf(float noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare x86_fp80 @coshl(x86_fp80 noundef) [[NOT_READNONE]]
+// HAS_MAYTRAP: declare double @llvm.experimental.constrained.cosh.f64(
+// HAS_MAYTRAP: declare float @llvm.experimental.constrained.cosh.f32(
+// HAS_MAYTRAP: declare x86_fp80 @llvm.experimental.constrained.cosh.f80(
 
   erf(f);        erff(f);       erfl(f);
 
@@ -638,15 +638,16 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
 
   sinh(f);       sinhf(f);      sinhl(f);
 
-// NO__ERRNO: declare double @sinh(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @sinhf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @sinhl(x86_fp80 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.sinh.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.sinh.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.sinh.f80(x86_fp80) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @sinh(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @sinhf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @sinhl(x86_fp80 noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare double @sinh(double noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare float @sinhf(float noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare x86_fp80 @sinhl(x86_fp80 noundef) [[NOT_READNONE]]
+// HAS_MAYTRAP: declare double @llvm.experimental.constrained.sinh.f64(
+// HAS_MAYTRAP: declare float @llvm.experimental.constrained.sinh.f32(
+// HAS_MAYTRAP: declare x86_fp80 @llvm.experimental.constrained.sinh.f80(
+
 
   sqrt(f);       sqrtf(f);      sqrtl(f);
 
@@ -674,15 +675,15 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
 
   tanh(f);       tanhf(f);      tanhl(f);
 
-// NO__ERRNO: declare double @tanh(double noundef) [[READNONE]]
-// NO__ERRNO: declare float @tanhf(float noundef) [[READNONE]]
-// NO__ERRNO: declare x86_fp80 @tanhl(x86_fp80 noundef) [[READNONE]]
+// NO__ERRNO: declare double @llvm.tanh.f64(double) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare float @llvm.tanh.f32(float) [[READNONE_INTRINSIC]]
+// NO__ERRNO: declare x86_fp80 @llvm.tanh.f80(x86_fp80) [[READNONE_INTRINSIC]]
 // HAS_ERRNO: declare double @tanh(double noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare float @tanhf(float noundef) [[NOT_READNONE]]
 // HAS_ERRNO: declare x86_fp80 @tanhl(x86_fp80 noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare double @tanh(double noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare float @tanhf(float noundef) [[NOT_READNONE]]
-// HAS_MAYTRAP: declare x86_fp80 @tanhl(x86_fp80 noundef) [[NOT_READNONE]]
+// HAS_MAYTRAP: declare double @llvm.experimental.constrained.tanh.f64(
+// HAS_MAYTRAP: declare float @llvm.experimental.constrained.tanh.f32(
+// HAS_MAYTRAP: declare x86_fp80 @llvm.experimental.constrained.tanh.f80(
 
   tgamma(f);     tgammaf(f);    tgammal(f);
 
diff --git a/clang/test/CodeGenOpenCL/builtins-f16.cl b/clang/test/CodeGenOpenCL/builtins-f16.cl
index d7bffdad5c548..8150bc1ac9e2d 100644
--- a/clang/test/CodeGenOpenCL/builtins-f16.cl
+++ b/clang/test/CodeGenOpenCL/builtins-f16.cl
@@ -6,6 +6,15 @@
 void test_half_builtins(half h0, half h1, half h2, int i0) {
   volatile half res;
 
+  // CHECK: call half @llvm.acos.f16(half %h0)
+  res = __builtin_acosf16(h0);
+
+  // CHECK: call half @llvm.asin.f16(half %h0)
+  res = __builtin_asinf16(h0);
+
+  // CHECK: call half @llvm.atan.f16(half %h0)
+  res = __builtin_atanf16(h0);
+
   // CHECK: call half @llvm.copysign.f16(half %h0, half %h1)
   res = __builtin_copysignf16(h0, h1);
 
@@ -18,6 +27,9 @@ void test_half_builtins(half h0, half h1, half h2, int i0) {
   // CHECK: call half @llvm.cos.f16(half %h0)
   res = __builtin_cosf16(h0);
 
+  // CHECK: call half @llvm.cosh.f16(half %h0)
+  res = __builtin_coshf16(h0);
+
   // CHECK: call half @llvm.exp.f16(half %h0)
   res = __builtin_expf16(h0);
 
@@ -63,12 +75,18 @@ void test_half_builtins(half h0, half h1, half h2, int i0) {
   // CHECK: call half @llvm.sin.f16(half %h0)
   res = __builtin_sinf16(h0);
 
+  // CHECK: call half @llvm.sinh.f16(half %h0)
+  res = __builtin_sinhf16(h0);
+
   // CHECK: call half @llvm.sqrt.f16(half %h0)
   res = __builtin_sqrtf16(h0);
 
   // CHECK: call half @llvm.tan.f16(half %h0)
   res = __builtin_tanf16(h0);
 
+  // CHECK: call half @llvm.tanh.f16(half %h0)
+  res = __builtin_tanhf16(h0);
+
   // CHECK: call half @llvm.trunc.f16(half %h0)
   res = __builtin_truncf16(h0);
 

From 7e37d021022b0786a704a89abae2014692158b2f Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 19 Jul 2024 09:26:35 -0500
Subject: [PATCH 201/486] [libc] Fix headers for statvfs implementation

Summry:
@lntue
---
 libc/include/llvm-libc-types/struct_statvfs.h | 4 ++--
 libc/src/sys/statvfs/fstatvfs.h               | 2 +-
 libc/src/sys/statvfs/linux/statfs_utils.h     | 2 +-
 libc/src/sys/statvfs/statvfs.h                | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/libc/include/llvm-libc-types/struct_statvfs.h b/libc/include/llvm-libc-types/struct_statvfs.h
index f467cfd936bdb..9c649af151ff6 100644
--- a/libc/include/llvm-libc-types/struct_statvfs.h
+++ b/libc/include/llvm-libc-types/struct_statvfs.h
@@ -9,8 +9,8 @@
 #ifndef LLVM_LIBC_TYPES_STRUCT_STATVFS_H
 #define LLVM_LIBC_TYPES_STRUCT_STATVFS_H
 
-#include <llvm-libc-types/fsblkcnt_t.h>
-#include <llvm-libc-types/fsfilcnt_t.h>
+#include "fsblkcnt_t.h"
+#include "fsfilcnt_t.h"
 
 struct statvfs {
   unsigned long f_bsize;   /* Filesystem block size */
diff --git a/libc/src/sys/statvfs/fstatvfs.h b/libc/src/sys/statvfs/fstatvfs.h
index 60c271c253d32..dfff757d29fee 100644
--- a/libc/src/sys/statvfs/fstatvfs.h
+++ b/libc/src/sys/statvfs/fstatvfs.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_SYS_STATVFS_FSTATVFS_H
 #define LLVM_LIBC_SRC_SYS_STATVFS_FSTATVFS_H
 
-#include "llvm-libc-types/struct_statvfs.h"
+#include "include/llvm-libc-types/struct_statvfs.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/sys/statvfs/linux/statfs_utils.h b/libc/src/sys/statvfs/linux/statfs_utils.h
index ca981c1364748..1e5be51531012 100644
--- a/libc/src/sys/statvfs/linux/statfs_utils.h
+++ b/libc/src/sys/statvfs/linux/statfs_utils.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_SYS_STATVFS_LINUX_STATFS_TO_STATVFS_H
 #define LLVM_LIBC_SRC_SYS_STATVFS_LINUX_STATFS_TO_STATVFS_H
 
-#include "llvm-libc-types/struct_statvfs.h"
+#include "include/llvm-libc-types/struct_statvfs.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/macros/attributes.h"
diff --git a/libc/src/sys/statvfs/statvfs.h b/libc/src/sys/statvfs/statvfs.h
index 674a079b593c4..1350880c4ad9f 100644
--- a/libc/src/sys/statvfs/statvfs.h
+++ b/libc/src/sys/statvfs/statvfs.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_SYS_STATVFS_STATVFS_H
 #define LLVM_LIBC_SRC_SYS_STATVFS_STATVFS_H
 
-#include "llvm-libc-types/struct_statvfs.h"
+#include "include/llvm-libc-types/struct_statvfs.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {

From f6e01b9ece1e73f6eda6e1dbff3aa72e917f4007 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Fri, 19 Jul 2024 07:17:48 -0700
Subject: [PATCH 202/486] [SLP]Do not trunc bv nodes, if the user is vectorized
 an requires wider type.

If at least a single user of the gathered trunc'ed instruction is
vectorized and requires wider type, than the trunc node, such
gathers/buildvectors should not be optimized for better bitwidth.
---
 .../lib/Transforms/Vectorize/SLPVectorizer.cpp | 18 +++++++++++++++++-
 .../SLPVectorizer/RISCV/trunc-bv-multi-uses.ll |  3 +--
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index bf2f2c334e4c1..667c4eb311c22 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15529,7 +15529,23 @@ void BoUpSLP::computeMinimumValueSizes() {
     // Check if the root is trunc and the next node is gather/buildvector, then
     // keep trunc in scalars, which is free in most cases.
     if (E.isGather() && IsTruncRoot && E.UserTreeIndices.size() == 1 &&
-        E.Idx > (IsStoreOrInsertElt ? 2 : 1)) {
+        E.Idx > (IsStoreOrInsertElt ? 2 : 1) &&
+        all_of(E.Scalars, [&](Value *V) {
+          return V->hasOneUse() || isa<Constant>(V) ||
+                 (!V->hasNUsesOrMore(UsesLimit) &&
+                  none_of(V->users(), [&](User *U) {
+                    const TreeEntry *TE = getTreeEntry(U);
+                    const TreeEntry *UserTE = E.UserTreeIndices.back().UserTE;
+                    if (TE == UserTE || !TE)
+                      return false;
+                    unsigned UserTESz = DL->getTypeSizeInBits(
+                        UserTE->Scalars.front()->getType());
+                    auto It = MinBWs.find(TE);
+                    if (It != MinBWs.end() && It->second.first > UserTESz)
+                      return true;
+                    return DL->getTypeSizeInBits(U->getType()) > UserTESz;
+                  }));
+        })) {
       ToDemote.push_back(E.Idx);
       const TreeEntry *UserTE = E.UserTreeIndices.back().UserTE;
       auto It = MinBWs.find(UserTE);
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-bv-multi-uses.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-bv-multi-uses.ll
index 53acc37155c9f..3ca5733a6bad0 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-bv-multi-uses.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-bv-multi-uses.ll
@@ -8,8 +8,7 @@ define i32 @test(i64 %v1, i64 %v2) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[V1]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i64> [[TMP0]], i64 [[V2]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i32>
-; CHECK-NEXT:    [[TMP3:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr <2 x i64> [[TMP3]], <i64 32, i64 32>
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr <2 x i64> [[TMP1]], <i64 32, i64 32>
 ; CHECK-NEXT:    [[TMP5:%.*]] = trunc <2 x i64> [[TMP4]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i32> [[TMP2]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0

From f1c27a9b269ead54d348e19b78b608c961c9e647 Mon Sep 17 00:00:00 2001
From: Utkarsh Saxena <usx@google.com>
Date: Fri, 19 Jul 2024 16:40:28 +0200
Subject: [PATCH 203/486] Reapply "Add source file name for template
 instantiations in -ftime-trace" (#99545)

Fix the Windows test.
---
 a-abfdec1d.o.tmp                              |   0
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/Driver/Options.td         |   4 +
 .../include/clang/Frontend/FrontendOptions.h  |   8 +-
 clang/lib/Driver/ToolChains/Clang.cpp         |   1 +
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  11 +-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |  11 +-
 clang/test/Driver/ftime-trace-sections.cpp    |   2 +-
 clang/test/Driver/ftime-trace.cpp             |  39 +++---
 clang/tools/driver/cc1_main.cpp               |   3 +-
 clang/unittests/Support/TimeProfilerTest.cpp  | 123 ++++++++++++++----
 llvm/include/llvm/Support/TimeProfiler.h      |  23 +++-
 llvm/lib/Support/TimeProfiler.cpp             |  61 +++++++--
 13 files changed, 225 insertions(+), 64 deletions(-)
 create mode 100644 a-abfdec1d.o.tmp

diff --git a/a-abfdec1d.o.tmp b/a-abfdec1d.o.tmp
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index aa91663260755..7ac6ed934290d 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -750,6 +750,9 @@ Improvements to Clang's time-trace
 - Clang now specifies that using ``auto`` in a lambda parameter is a C++14 extension when
   appropriate. (`#46059: <https://github.com/llvm/llvm-project/issues/46059>`_).
 
+- Clang now adds source file infomation for template instantiations as ``event["args"]["filename"]``. This
+  added behind an option ``-ftime-trace-verbose``. This is expected to increase the size of trace by 2-3 times.
+
 Improvements to Coverage Mapping
 --------------------------------
 
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8707e71f2a319..9c6cebd77ff0a 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3998,6 +3998,10 @@ def ftime_trace_granularity_EQ : Joined<["-"], "ftime-trace-granularity=">, Grou
   HelpText<"Minimum time granularity (in microseconds) traced by time profiler">,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
   MarshallingInfoInt<FrontendOpts<"TimeTraceGranularity">, "500u">;
+def ftime_trace_verbose : Joined<["-"], "ftime-trace-verbose">, Group<f_Group>,
+  HelpText<"Make time trace capture verbose event details (e.g. source filenames). This can increase the size of the output by 2-3 times">,
+  Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
+  MarshallingInfoFlag<FrontendOpts<"TimeTraceVerbose">>;
 def ftime_trace_EQ : Joined<["-"], "ftime-trace=">, Group<f_Group>,
   HelpText<"Similar to -ftime-trace. Specify the JSON file or a directory which will contain the JSON file">,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 5e5034fe01eb5..8241925c98476 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -580,6 +580,11 @@ class FrontendOptions {
   /// Minimum time granularity (in microseconds) traced by time profiler.
   unsigned TimeTraceGranularity;
 
+  /// Make time trace capture verbose event details (e.g. source filenames).
+  /// This can increase the size of the output by 2-3 times.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned TimeTraceVerbose : 1;
+
   /// Path which stores the output files for -ftime-trace
   std::string TimeTracePath;
 
@@ -601,7 +606,8 @@ class FrontendOptions {
         EmitSymbolGraph(false), EmitExtensionSymbolGraphs(false),
         EmitSymbolGraphSymbolLabelsForTesting(false),
         EmitPrettySymbolGraphs(false), GenReducedBMI(false),
-        UseClangIRPipeline(false), TimeTraceGranularity(500) {}
+        UseClangIRPipeline(false), TimeTraceGranularity(500),
+        TimeTraceVerbose(false) {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file
   /// extension. For example, "c" would return Language::C.
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index c9f0fe0a3d60f..f7b987bf810c1 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6757,6 +6757,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (const char *Name = C.getTimeTraceFile(&JA)) {
     CmdArgs.push_back(Args.MakeArgString("-ftime-trace=" + Twine(Name)));
     Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_granularity_EQ);
+    Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_verbose);
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_ftrapv_handler_EQ)) {
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index a7bc6749c5852..725b62db5e80a 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -3426,11 +3426,16 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
     return true;
 
   llvm::TimeTraceScope TimeScope("InstantiateClass", [&]() {
-    std::string Name;
-    llvm::raw_string_ostream OS(Name);
+    llvm::TimeTraceMetadata M;
+    llvm::raw_string_ostream OS(M.Detail);
     Instantiation->getNameForDiagnostic(OS, getPrintingPolicy(),
                                         /*Qualified=*/true);
-    return Name;
+    if (llvm::isTimeTraceVerbose()) {
+      auto Loc = SourceMgr.getExpansionLoc(Instantiation->getLocation());
+      M.File = SourceMgr.getFilename(Loc);
+      M.Line = SourceMgr.getExpansionLineNumber(Loc);
+    }
+    return M;
   });
 
   Pattern = PatternDef;
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 97161febc15f7..a12d2eff1d2c8 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -4966,11 +4966,16 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   }
 
   llvm::TimeTraceScope TimeScope("InstantiateFunction", [&]() {
-    std::string Name;
-    llvm::raw_string_ostream OS(Name);
+    llvm::TimeTraceMetadata M;
+    llvm::raw_string_ostream OS(M.Detail);
     Function->getNameForDiagnostic(OS, getPrintingPolicy(),
                                    /*Qualified=*/true);
-    return Name;
+    if (llvm::isTimeTraceVerbose()) {
+      auto Loc = SourceMgr.getExpansionLoc(Function->getLocation());
+      M.File = SourceMgr.getFilename(Loc);
+      M.Line = SourceMgr.getExpansionLineNumber(Loc);
+    }
+    return M;
   });
 
   // If we're performing recursive template instantiation, create our own
diff --git a/clang/test/Driver/ftime-trace-sections.cpp b/clang/test/Driver/ftime-trace-sections.cpp
index 0c16052bc0c3a..da7109b9d81a6 100644
--- a/clang/test/Driver/ftime-trace-sections.cpp
+++ b/clang/test/Driver/ftime-trace-sections.cpp
@@ -1,5 +1,5 @@
 // RUN: rm -rf %t && mkdir %t && cd %t
-// RUN: %clangxx -S -ftime-trace -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: %python %S/ftime-trace-sections.py < out.json
 
 template <typename T>
diff --git a/clang/test/Driver/ftime-trace.cpp b/clang/test/Driver/ftime-trace.cpp
index 5fe63de915a71..60c5885704b58 100644
--- a/clang/test/Driver/ftime-trace.cpp
+++ b/clang/test/Driver/ftime-trace.cpp
@@ -1,18 +1,18 @@
 // RUN: rm -rf %t && mkdir -p %t && cd %t
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=new-name.json -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=new-name.json -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat new-name.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
 // RUN: mkdir dir1 dir2
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir1 -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir1 -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat dir1/out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir2/ -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir2/ -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat dir2/out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
@@ -34,32 +34,33 @@
 // RUN: mkdir d e f && cp %s d/a.cpp && touch d/b.c
 
 /// TODO: Support -fno-integrated-as.
-// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -fintegrated-as d/a.cpp -o e/a.o 2>&1 | FileCheck %s --check-prefix=COMPILE1
-// COMPILE1: -cc1{{.*}} "-ftime-trace=e/a.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -fintegrated-as d/a.cpp -o e/a.o 2>&1 | FileCheck %s --check-prefix=COMPILE1
+// COMPILE1: -cc1{{.*}} "-ftime-trace=e/a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
-// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=COMPILE2
-// COMPILE2: -cc1{{.*}} "-ftime-trace=f/a.json" "-ftime-trace-granularity=0"
-// COMPILE2: -cc1{{.*}} "-ftime-trace=f/b.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=COMPILE2
+// COMPILE2: -cc1{{.*}} "-ftime-trace=f/a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// COMPILE2: -cc1{{.*}} "-ftime-trace=f/b.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
 /// -o specifies the link output. Create ${output}-${basename}.json.
-// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -o e/x 2>&1 | FileCheck %s --check-prefix=LINK1
-// LINK1: -cc1{{.*}} "-ftime-trace=e/x-a.json" "-ftime-trace-granularity=0"
-// LINK1: -cc1{{.*}} "-ftime-trace=e/x-b.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o e/x 2>&1 | FileCheck %s --check-prefix=LINK1
+// LINK1: -cc1{{.*}} "-ftime-trace=e/x-a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// LINK1: -cc1{{.*}} "-ftime-trace=e/x-b.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
 /// -dumpdir is f/g, not ending with a path separator. We create f/g${basename}.json.
-// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -o e/x -dumpdir f/g 2>&1 | FileCheck %s --check-prefix=LINK2
-// LINK2: -cc1{{.*}} "-ftime-trace=f/ga.json" "-ftime-trace-granularity=0"
-// LINK2: -cc1{{.*}} "-ftime-trace=f/gb.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o e/x -dumpdir f/g 2>&1 | FileCheck %s --check-prefix=LINK2
+// LINK2: -cc1{{.*}} "-ftime-trace=f/ga.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// LINK2: -cc1{{.*}} "-ftime-trace=f/gb.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
-// RUN: %clang -### -ftime-trace=e -ftime-trace-granularity=0 d/a.cpp d/b.c -o f/x -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=LINK3
-// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}a-{{[^.]*}}.json" "-ftime-trace-granularity=0"
-// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}b-{{[^.]*}}.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -ftime-trace=e -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o f/x -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=LINK3
+// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}a-{{[^.]*}}.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}b-{{[^.]*}}.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
-// RUN: %clang -### -ftime-trace -ftime-trace=e -ftime-trace-granularity=1 -xassembler d/a.cpp 2>&1 | \
+// RUN: %clang -### -ftime-trace -ftime-trace=e -ftime-trace-granularity=1 -ftime-trace-verbose -xassembler d/a.cpp 2>&1 | \
 // RUN:   FileCheck %s --check-prefix=UNUSED
 // UNUSED:      warning: argument unused during compilation: '-ftime-trace'
 // UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace=e'
 // UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace-granularity=1'
+// UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace-verbose'
 // UNUSED-NOT:  warning:
 
 template <typename T>
diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp
index c2ccb47a15bc8..f5e5fad36573e 100644
--- a/clang/tools/driver/cc1_main.cpp
+++ b/clang/tools/driver/cc1_main.cpp
@@ -241,7 +241,8 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
 
   if (!Clang->getFrontendOpts().TimeTracePath.empty()) {
     llvm::timeTraceProfilerInitialize(
-        Clang->getFrontendOpts().TimeTraceGranularity, Argv0);
+        Clang->getFrontendOpts().TimeTraceGranularity, Argv0,
+        Clang->getFrontendOpts().TimeTraceVerbose);
   }
   // --print-supported-cpus takes priority over the actual compilation.
   if (Clang->getFrontendOpts().PrintSupportedCPUs)
diff --git a/clang/unittests/Support/TimeProfilerTest.cpp b/clang/unittests/Support/TimeProfilerTest.cpp
index 5f3950ff033f1..56d880cffde61 100644
--- a/clang/unittests/Support/TimeProfilerTest.cpp
+++ b/clang/unittests/Support/TimeProfilerTest.cpp
@@ -10,11 +10,15 @@
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Lex/PreprocessorOptions.h"
 
+#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/JSON.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/TimeProfiler.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include <stack>
 
 #include "gtest/gtest.h"
+#include <tuple>
 
 using namespace clang;
 using namespace llvm;
@@ -23,7 +27,8 @@ namespace {
 
 // Should be called before testing.
 void setupProfiler() {
-  timeTraceProfilerInitialize(/*TimeTraceGranularity=*/0, "test");
+  timeTraceProfilerInitialize(/*TimeTraceGranularity=*/0, "test",
+                              /*TimeTraceVerbose=*/true);
 }
 
 // Should be called after `compileFromString()`.
@@ -38,14 +43,24 @@ std::string teardownProfiler() {
 
 // Returns true if code compiles successfully.
 // We only parse AST here. This is enough for constexpr evaluation.
-bool compileFromString(StringRef Code, StringRef Standard, StringRef FileName) {
+bool compileFromString(StringRef Code, StringRef Standard, StringRef File,
+                       llvm::StringMap<std::string> Headers = {}) {
   CompilerInstance Compiler;
   Compiler.createDiagnostics();
 
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> FS(
+      new llvm::vfs::InMemoryFileSystem());
+  FS->addFile(File, 0, MemoryBuffer::getMemBuffer(Code));
+  for (const auto &Header : Headers) {
+    FS->addFile(Header.getKey(), 0,
+                MemoryBuffer::getMemBuffer(Header.getValue()));
+  }
+  llvm::IntrusiveRefCntPtr<FileManager> Files(
+      new FileManager(FileSystemOptions(), FS));
+  Compiler.setFileManager(Files.get());
+
   auto Invocation = std::make_shared<CompilerInvocation>();
-  Invocation->getPreprocessorOpts().addRemappedFile(
-      FileName, MemoryBuffer::getMemBuffer(Code).release());
-  const char *Args[] = {Standard.data(), FileName.data()};
+  std::vector<const char *> Args = {Standard.data(), File.data()};
   CompilerInvocation::CreateFromArgs(*Invocation, Args,
                                      Compiler.getDiagnostics());
   Compiler.setInvocation(std::move(Invocation));
@@ -60,13 +75,28 @@ bool compileFromString(StringRef Code, StringRef Standard, StringRef FileName) {
   return Compiler.ExecuteAction(Action);
 }
 
+std::string GetMetadata(json::Object *Event) {
+  std::string Metadata;
+  llvm::raw_string_ostream OS(Metadata);
+  if (json::Object *Args = Event->getObject("args")) {
+    if (auto Detail = Args->getString("detail"))
+      OS << Detail;
+    // Use only filename to not include os-specific path separators.
+    if (auto File = Args->getString("file"))
+      OS << ", " << llvm::sys::path::filename(*File);
+    if (auto Line = Args->getInteger("line"))
+      OS << ":" << *Line;
+  }
+  return Metadata;
+}
+
 // Returns pretty-printed trace graph.
 std::string buildTraceGraph(StringRef Json) {
   struct EventRecord {
     int64_t TimestampBegin;
     int64_t TimestampEnd;
-    StringRef Name;
-    StringRef Detail;
+    std::string Name;
+    std::string Metadata;
   };
   std::vector<EventRecord> Events;
 
@@ -81,10 +111,13 @@ std::string buildTraceGraph(StringRef Json) {
     int64_t TimestampBegin = TraceEventObj->getInteger("ts").value_or(0);
     int64_t TimestampEnd =
         TimestampBegin + TraceEventObj->getInteger("dur").value_or(0);
-    StringRef Name = TraceEventObj->getString("name").value_or("");
-    StringRef Detail = "";
-    if (json::Object *Args = TraceEventObj->getObject("args"))
-      Detail = Args->getString("detail").value_or("");
+    std::string Name = TraceEventObj->getString("name").value_or("").str();
+    std::string Metadata = GetMetadata(TraceEventObj);
+
+    // Source events are asynchronous events and may not perfectly nest the
+    // synchronous events. Skip testing them.
+    if (Name == "Source")
+      continue;
 
     // This is a "summary" event, like "Total PerformPendingInstantiations",
     // skip it
@@ -92,7 +125,7 @@ std::string buildTraceGraph(StringRef Json) {
       continue;
 
     Events.emplace_back(
-        EventRecord{TimestampBegin, TimestampEnd, Name, Detail});
+        EventRecord{TimestampBegin, TimestampEnd, Name, Metadata});
   }
 
   // There can be nested events that are very fast, for example:
@@ -132,9 +165,9 @@ std::string buildTraceGraph(StringRef Json) {
       Stream << "| ";
     }
     Stream.write(Event.Name.data(), Event.Name.size());
-    if (!Event.Detail.empty()) {
+    if (!Event.Metadata.empty()) {
       Stream << " (";
-      Stream.write(Event.Detail.data(), Event.Detail.size());
+      Stream.write(Event.Metadata.data(), Event.Metadata.size());
       Stream << ")";
     }
     Stream << "\n";
@@ -145,7 +178,7 @@ std::string buildTraceGraph(StringRef Json) {
 } // namespace
 
 TEST(TimeProfilerTest, ConstantEvaluationCxx20) {
-  constexpr StringRef Code = R"(
+  std::string Code = R"(
 void print(double value);
 
 namespace slow_namespace {
@@ -175,8 +208,7 @@ constexpr int slow_init_list[] = {1, 1, 2, 3, 5, 8, 13, 21}; // 25th line
   setupProfiler();
   ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc"));
   std::string Json = teardownProfiler();
-  std::string TraceGraph = buildTraceGraph(Json);
-  ASSERT_TRUE(TraceGraph == R"(
+  ASSERT_EQ(R"(
 Frontend
 | ParseDeclarationOrFunctionDefinition (test.cc:2:1)
 | ParseDeclarationOrFunctionDefinition (test.cc:6:1)
@@ -202,14 +234,54 @@ Frontend
 | ParseDeclarationOrFunctionDefinition (test.cc:25:1)
 | | EvaluateAsInitializer (slow_init_list)
 | PerformPendingInstantiations
-)");
+)",
+            buildTraceGraph(Json));
+}
+
+TEST(TimeProfilerTest, TemplateInstantiations) {
+  std::string B_H = R"(
+    template <typename T>
+    T fooB(T t) {
+      return T();
+    }
 
-  // NOTE: If this test is failing, run this test with
-  // `llvm::errs() << TraceGraph;` and change the assert above.
+    #define MacroTemp(x) template <typename T> void foo##x(T) { T(); }
+  )";
+
+  std::string A_H = R"(
+    #include "b.h"
+
+    MacroTemp(MTA)
+
+    template <typename T>
+    void fooA(T t) { fooB(t); fooMTA(t); }
+  )";
+  std::string Code = R"(
+    #include "a.h"
+    void user() { fooA(0); }
+  )";
+
+  setupProfiler();
+  ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc",
+                                /*Headers=*/{{"a.h", A_H}, {"b.h", B_H}}));
+  std::string Json = teardownProfiler();
+  ASSERT_EQ(R"(
+Frontend
+| ParseFunctionDefinition (fooB)
+| ParseFunctionDefinition (fooMTA)
+| ParseFunctionDefinition (fooA)
+| ParseDeclarationOrFunctionDefinition (test.cc:3:5)
+| | ParseFunctionDefinition (user)
+| PerformPendingInstantiations
+| | InstantiateFunction (fooA<int>, a.h:7)
+| | | InstantiateFunction (fooB<int>, b.h:3)
+| | | InstantiateFunction (fooMTA<int>, a.h:4)
+)",
+            buildTraceGraph(Json));
 }
 
 TEST(TimeProfilerTest, ConstantEvaluationC99) {
-  constexpr StringRef Code = R"(
+  std::string Code = R"(
 struct {
   short quantval[4]; // 3rd line
 } value;
@@ -218,15 +290,12 @@ struct {
   setupProfiler();
   ASSERT_TRUE(compileFromString(Code, "-std=c99", "test.c"));
   std::string Json = teardownProfiler();
-  std::string TraceGraph = buildTraceGraph(Json);
-  ASSERT_TRUE(TraceGraph == R"(
+  ASSERT_EQ(R"(
 Frontend
 | ParseDeclarationOrFunctionDefinition (test.c:2:1)
 | | isIntegerConstantExpr (<test.c:3:18>)
 | | EvaluateKnownConstIntCheckOverflow (<test.c:3:18>)
 | PerformPendingInstantiations
-)");
-
-  // NOTE: If this test is failing, run this test with
-  // `llvm::errs() << TraceGraph;` and change the assert above.
+)",
+            buildTraceGraph(Json));
 }
diff --git a/llvm/include/llvm/Support/TimeProfiler.h b/llvm/include/llvm/Support/TimeProfiler.h
index 31f7df10916db..6eb92930b36fd 100644
--- a/llvm/include/llvm/Support/TimeProfiler.h
+++ b/llvm/include/llvm/Support/TimeProfiler.h
@@ -83,16 +83,28 @@ namespace llvm {
 
 class raw_pwrite_stream;
 
+struct TimeTraceMetadata {
+  std::string Detail;
+  // Source file and line number information for the event.
+  std::string File;
+  int Line;
+
+  bool isEmpty() const { return Detail.empty() && File.empty(); }
+};
+
 struct TimeTraceProfiler;
 TimeTraceProfiler *getTimeTraceProfilerInstance();
 
+bool isTimeTraceVerbose();
+
 struct TimeTraceProfilerEntry;
 
 /// Initialize the time trace profiler.
 /// This sets up the global \p TimeTraceProfilerInstance
 /// variable to be the profiler instance.
 void timeTraceProfilerInitialize(unsigned TimeTraceGranularity,
-                                 StringRef ProcName);
+                                 StringRef ProcName,
+                                 bool TimeTraceVerbose = false);
 
 /// Cleanup the time trace profiler, if it was initialized.
 void timeTraceProfilerCleanup();
@@ -128,6 +140,10 @@ TimeTraceProfilerEntry *
 timeTraceProfilerBegin(StringRef Name,
                        llvm::function_ref<std::string()> Detail);
 
+TimeTraceProfilerEntry *
+timeTraceProfilerBegin(StringRef Name,
+                       llvm::function_ref<TimeTraceMetadata()> MetaData);
+
 /// Manually begin a time section, with the given \p Name and \p Detail.
 /// This starts Async Events having \p Name as a category which is shown
 /// separately from other traces. See
@@ -164,6 +180,11 @@ class TimeTraceScope {
     if (getTimeTraceProfilerInstance() != nullptr)
       Entry = timeTraceProfilerBegin(Name, Detail);
   }
+  TimeTraceScope(StringRef Name,
+                 llvm::function_ref<TimeTraceMetadata()> Metadata) {
+    if (getTimeTraceProfilerInstance() != nullptr)
+      Entry = timeTraceProfilerBegin(Name, Metadata);
+  }
   ~TimeTraceScope() {
     if (getTimeTraceProfilerInstance() != nullptr)
       timeTraceProfilerEnd(Entry);
diff --git a/llvm/lib/Support/TimeProfiler.cpp b/llvm/lib/Support/TimeProfiler.cpp
index 9612db7d30f98..c2014028ddadc 100644
--- a/llvm/lib/Support/TimeProfiler.cpp
+++ b/llvm/lib/Support/TimeProfiler.cpp
@@ -73,12 +73,20 @@ struct llvm::TimeTraceProfilerEntry {
   const TimePointType Start;
   TimePointType End;
   const std::string Name;
-  const std::string Detail;
+  TimeTraceMetadata Metadata;
+
   const bool AsyncEvent = false;
   TimeTraceProfilerEntry(TimePointType &&S, TimePointType &&E, std::string &&N,
                          std::string &&Dt, bool Ae)
+      : Start(std::move(S)), End(std::move(E)), Name(std::move(N)), Metadata(),
+        AsyncEvent(Ae) {
+    Metadata.Detail = std::move(Dt);
+  }
+
+  TimeTraceProfilerEntry(TimePointType &&S, TimePointType &&E, std::string &&N,
+                         TimeTraceMetadata &&Mt, bool Ae)
       : Start(std::move(S)), End(std::move(E)), Name(std::move(N)),
-        Detail(std::move(Dt)), AsyncEvent(Ae) {}
+        Metadata(std::move(Mt)), AsyncEvent(Ae) {}
 
   // Calculate timings for FlameGraph. Cast time points to microsecond precision
   // rather than casting duration. This avoids truncation issues causing inner
@@ -97,10 +105,12 @@ struct llvm::TimeTraceProfilerEntry {
 };
 
 struct llvm::TimeTraceProfiler {
-  TimeTraceProfiler(unsigned TimeTraceGranularity = 0, StringRef ProcName = "")
+  TimeTraceProfiler(unsigned TimeTraceGranularity = 0, StringRef ProcName = "",
+                    bool TimeTraceVerbose = false)
       : BeginningOfTime(system_clock::now()), StartTime(ClockType::now()),
         ProcName(ProcName), Pid(sys::Process::getProcessId()),
-        Tid(llvm::get_threadid()), TimeTraceGranularity(TimeTraceGranularity) {
+        Tid(llvm::get_threadid()), TimeTraceGranularity(TimeTraceGranularity),
+        TimeTraceVerbose(TimeTraceVerbose) {
     llvm::get_thread_name(ThreadName);
   }
 
@@ -113,6 +123,15 @@ struct llvm::TimeTraceProfiler {
     return Stack.back().get();
   }
 
+  TimeTraceProfilerEntry *
+  begin(std::string Name, llvm::function_ref<TimeTraceMetadata()> Metadata,
+        bool AsyncEvent = false) {
+    Stack.emplace_back(std::make_unique<TimeTraceProfilerEntry>(
+        ClockType::now(), TimePointType(), std::move(Name), Metadata(),
+        AsyncEvent));
+    return Stack.back().get();
+  }
+
   void end() {
     assert(!Stack.empty() && "Must call begin() first");
     end(*Stack.back());
@@ -184,8 +203,15 @@ struct llvm::TimeTraceProfiler {
           J.attribute("dur", DurUs);
         }
         J.attribute("name", E.Name);
-        if (!E.Detail.empty()) {
-          J.attributeObject("args", [&] { J.attribute("detail", E.Detail); });
+        if (!E.Metadata.isEmpty()) {
+          J.attributeObject("args", [&] {
+            if (!E.Metadata.Detail.empty())
+              J.attribute("detail", E.Metadata.Detail);
+            if (!E.Metadata.File.empty())
+              J.attribute("file", E.Metadata.File);
+            if (E.Metadata.Line > 0)
+              J.attribute("line", E.Metadata.Line);
+          });
         }
       });
 
@@ -307,14 +333,25 @@ struct llvm::TimeTraceProfiler {
 
   // Minimum time granularity (in microseconds)
   const unsigned TimeTraceGranularity;
+
+  // Make time trace capture verbose event details (e.g. source filenames). This
+  // can increase the size of the output by 2-3 times.
+  const bool TimeTraceVerbose;
 };
 
+bool llvm::isTimeTraceVerbose() {
+  return getTimeTraceProfilerInstance() &&
+         getTimeTraceProfilerInstance()->TimeTraceVerbose;
+}
+
 void llvm::timeTraceProfilerInitialize(unsigned TimeTraceGranularity,
-                                       StringRef ProcName) {
+                                       StringRef ProcName,
+                                       bool TimeTraceVerbose) {
   assert(TimeTraceProfilerInstance == nullptr &&
          "Profiler should not be initialized");
   TimeTraceProfilerInstance = new TimeTraceProfiler(
-      TimeTraceGranularity, llvm::sys::path::filename(ProcName));
+      TimeTraceGranularity, llvm::sys::path::filename(ProcName),
+      TimeTraceVerbose);
 }
 
 // Removes all TimeTraceProfilerInstances.
@@ -381,6 +418,14 @@ llvm::timeTraceProfilerBegin(StringRef Name,
   return nullptr;
 }
 
+TimeTraceProfilerEntry *
+llvm::timeTraceProfilerBegin(StringRef Name,
+                             llvm::function_ref<TimeTraceMetadata()> Metadata) {
+  if (TimeTraceProfilerInstance != nullptr)
+    return TimeTraceProfilerInstance->begin(std::string(Name), Metadata, false);
+  return nullptr;
+}
+
 TimeTraceProfilerEntry *llvm::timeTraceAsyncProfilerBegin(StringRef Name,
                                                           StringRef Detail) {
   if (TimeTraceProfilerInstance != nullptr)

From 9da9127fec7b0a252b80d60b09b8c0ccedb41672 Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Fri, 19 Jul 2024 10:40:44 -0400
Subject: [PATCH 204/486] [libc][math] Fix signaling nan handling of hypot(f)
 and improve hypotf performance. (#99432)

The errors were reported by Paul Zimmermann with the CORE-MATH project's
test suites:
```
zimmerma@tartine:/tmp/core-math$ CORE_MATH_CHECK_STD=true LIBM=$L ./check.sh hypot
Running worst cases check in --rndn mode...
FAIL x=snan y=inf ref=qnan z=inf
Running worst cases check in --rndz mode...
FAIL x=snan y=inf ref=qnan z=inf
Running worst cases check in --rndu mode...
FAIL x=snan y=inf ref=qnan z=inf
Running worst cases check in --rndd mode...
Spurious inexact exception for x=0x1.ffffffffffffep+24 y=0x1p+0 (z=0x1.0000000000001p+25)
```
---
 libc/src/__support/FPUtil/Hypot.h             |  63 ++++++-----
 libc/src/math/generic/CMakeLists.txt          |   5 +-
 libc/src/math/generic/hypotf.cpp              | 100 +++++++++++-------
 libc/test/src/math/smoke/CMakeLists.txt       |   4 +
 libc/test/src/math/smoke/HypotTest.h          |  31 ++----
 .../llvm-project-overlay/libc/BUILD.bazel     |   1 +
 6 files changed, 110 insertions(+), 94 deletions(-)

diff --git a/libc/src/__support/FPUtil/Hypot.h b/libc/src/__support/FPUtil/Hypot.h
index a5a991e75ed01..6aa808446d6d9 100644
--- a/libc/src/__support/FPUtil/Hypot.h
+++ b/libc/src/__support/FPUtil/Hypot.h
@@ -109,45 +109,39 @@ LIBC_INLINE T hypot(T x, T y) {
   using StorageType = typename FPBits<T>::StorageType;
   using DStorageType = typename DoubleLength<StorageType>::Type;
 
-  FPBits_t x_bits(x), y_bits(y);
+  FPBits_t x_abs = FPBits_t(x).abs();
+  FPBits_t y_abs = FPBits_t(y).abs();
 
-  if (x_bits.is_inf() || y_bits.is_inf()) {
-    return FPBits_t::inf().get_val();
-  }
-  if (x_bits.is_nan()) {
-    return x;
-  }
-  if (y_bits.is_nan()) {
+  bool x_abs_larger = x_abs.uintval() >= y_abs.uintval();
+
+  FPBits_t a_bits = x_abs_larger ? x_abs : y_abs;
+  FPBits_t b_bits = x_abs_larger ? y_abs : x_abs;
+
+  if (LIBC_UNLIKELY(a_bits.is_inf_or_nan())) {
+    if (x_abs.is_signaling_nan() || y_abs.is_signaling_nan()) {
+      fputil::raise_except_if_required(FE_INVALID);
+      return FPBits_t::quiet_nan().get_val();
+    }
+    if (x_abs.is_inf() || y_abs.is_inf())
+      return FPBits_t::inf().get_val();
+    if (x_abs.is_nan())
+      return x;
+    // y is nan
     return y;
   }
 
-  uint16_t x_exp = x_bits.get_biased_exponent();
-  uint16_t y_exp = y_bits.get_biased_exponent();
-  uint16_t exp_diff = (x_exp > y_exp) ? (x_exp - y_exp) : (y_exp - x_exp);
+  uint16_t a_exp = a_bits.get_biased_exponent();
+  uint16_t b_exp = b_bits.get_biased_exponent();
 
-  if ((exp_diff >= FPBits_t::FRACTION_LEN + 2) || (x == 0) || (y == 0)) {
-    return abs(x) + abs(y);
-  }
+  if ((a_exp - b_exp >= FPBits_t::FRACTION_LEN + 2) || (x == 0) || (y == 0))
+    return x_abs.get_val() + y_abs.get_val();
 
-  uint16_t a_exp, b_exp, out_exp;
-  StorageType a_mant, b_mant;
+  uint64_t out_exp = a_exp;
+  StorageType a_mant = a_bits.get_mantissa();
+  StorageType b_mant = b_bits.get_mantissa();
   DStorageType a_mant_sq, b_mant_sq;
   bool sticky_bits;
 
-  if (abs(x) >= abs(y)) {
-    a_exp = x_exp;
-    a_mant = x_bits.get_mantissa();
-    b_exp = y_exp;
-    b_mant = y_bits.get_mantissa();
-  } else {
-    a_exp = y_exp;
-    a_mant = y_bits.get_mantissa();
-    b_exp = x_exp;
-    b_mant = x_bits.get_mantissa();
-  }
-
-  out_exp = a_exp;
-
   // Add an extra bit to simplify the final rounding bit computation.
   constexpr StorageType ONE = StorageType(1) << (FPBits_t::FRACTION_LEN + 1);
 
@@ -165,11 +159,10 @@ LIBC_INLINE T hypot(T x, T y) {
     a_exp = 1;
   }
 
-  if (b_exp != 0) {
+  if (b_exp != 0)
     b_mant |= ONE;
-  } else {
+  else
     b_exp = 1;
-  }
 
   a_mant_sq = static_cast<DStorageType>(a_mant) * a_mant;
   b_mant_sq = static_cast<DStorageType>(b_mant) * b_mant;
@@ -260,6 +253,10 @@ LIBC_INLINE T hypot(T x, T y) {
   }
 
   y_new |= static_cast<StorageType>(out_exp) << FPBits_t::FRACTION_LEN;
+
+  if (!(round_bit || sticky_bits || (r != 0)))
+    fputil::clear_except_if_required(FE_INEXACT);
+
   return cpp::bit_cast<T>(y_new);
 }
 
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 9c86bac4a0cb7..d775026fabb3e 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -2830,9 +2830,12 @@ add_entrypoint_object(
   HDRS
     ../hypotf.h
   DEPENDS
-    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.double_double
+    libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.sqrt
+    libc.src.__support.macros.optimization
   COMPILE_OPTIONS
     -O3
 )
diff --git a/libc/src/math/generic/hypotf.cpp b/libc/src/math/generic/hypotf.cpp
index 75c55ed846726..959c0420ae149 100644
--- a/libc/src/math/generic/hypotf.cpp
+++ b/libc/src/math/generic/hypotf.cpp
@@ -6,11 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 #include "src/math/hypotf.h"
-#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/double_double.h"
+#include "src/__support/FPUtil/multiply_add.h"
 #include "src/__support/FPUtil/sqrt.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -18,54 +21,75 @@ LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) {
   using DoubleBits = fputil::FPBits<double>;
   using FPBits = fputil::FPBits<float>;
 
-  FPBits x_bits(x), y_bits(y);
+  FPBits x_abs = FPBits(x).abs();
+  FPBits y_abs = FPBits(y).abs();
 
-  uint16_t x_exp = x_bits.get_biased_exponent();
-  uint16_t y_exp = y_bits.get_biased_exponent();
-  uint16_t exp_diff = (x_exp > y_exp) ? (x_exp - y_exp) : (y_exp - x_exp);
+  bool x_abs_larger = x_abs.uintval() >= y_abs.uintval();
 
-  if (exp_diff >= FPBits::FRACTION_LEN + 2) {
-    return fputil::abs(x) + fputil::abs(y);
-  }
+  FPBits a_bits = x_abs_larger ? x_abs : y_abs;
+  FPBits b_bits = x_abs_larger ? y_abs : x_abs;
 
-  double xd = static_cast<double>(x);
-  double yd = static_cast<double>(y);
+  uint32_t a_u = a_bits.uintval();
+  uint32_t b_u = b_bits.uintval();
 
-  // These squares are exact.
-  double x_sq = xd * xd;
-  double y_sq = yd * yd;
+  // Note: replacing `a_u >= FPBits::EXP_MASK` with `a_bits.is_inf_or_nan()`
+  // generates extra exponent bit masking instructions on x86-64.
+  if (LIBC_UNLIKELY(a_u >= FPBits::EXP_MASK)) {
+    // x or y is inf or nan
+    if (a_bits.is_signaling_nan() || b_bits.is_signaling_nan()) {
+      fputil::raise_except_if_required(FE_INVALID);
+      return FPBits::quiet_nan().get_val();
+    }
+    if (a_bits.is_inf() || b_bits.is_inf())
+      return FPBits::inf().get_val();
+    return a_bits.get_val();
+  }
 
-  // Compute the sum of squares.
-  double sum_sq = x_sq + y_sq;
+  if (LIBC_UNLIKELY(a_u - b_u >=
+                    static_cast<uint32_t>((FPBits::FRACTION_LEN + 2)
+                                          << FPBits::FRACTION_LEN)))
+    return x_abs.get_val() + y_abs.get_val();
 
-  // Compute the rounding error with Fast2Sum algorithm:
-  // x_sq + y_sq = sum_sq - err
-  double err = (x_sq >= y_sq) ? (sum_sq - x_sq) - y_sq : (sum_sq - y_sq) - x_sq;
+  double ad = static_cast<double>(a_bits.get_val());
+  double bd = static_cast<double>(b_bits.get_val());
+
+  // These squares are exact.
+  double a_sq = ad * ad;
+#ifdef LIBC_TARGET_CPU_HAS_FMA
+  double sum_sq = fputil::multiply_add(bd, bd, a_sq);
+#else
+  double b_sq = bd * bd;
+  double sum_sq = a_sq + b_sq;
+#endif
 
   // Take sqrt in double precision.
   DoubleBits result(fputil::sqrt<double>(sum_sq));
+  uint64_t r_u = result.uintval();
 
-  if (!DoubleBits(sum_sq).is_inf_or_nan()) {
-    // Correct rounding.
-    double r_sq = result.get_val() * result.get_val();
-    double diff = sum_sq - r_sq;
-    constexpr uint64_t MASK = 0x0000'0000'3FFF'FFFFULL;
-    uint64_t lrs = result.uintval() & MASK;
-
-    if (lrs == 0x0000'0000'1000'0000ULL && err < diff) {
-      result.set_uintval(result.uintval() | 1ULL);
-    } else if (lrs == 0x0000'0000'3000'0000ULL && err > diff) {
-      result.set_uintval(result.uintval() - 1ULL);
-    }
-  } else {
-    FPBits bits_x(x), bits_y(y);
-    if (bits_x.is_inf_or_nan() || bits_y.is_inf_or_nan()) {
-      if (bits_x.is_inf() || bits_y.is_inf())
-        return FPBits::inf().get_val();
-      if (bits_x.is_nan())
-        return x;
-      return y;
+  // If any of the sticky bits of the result are non-zero, except the LSB, then
+  // the rounded result is correct.
+  if (LIBC_UNLIKELY(((r_u + 1) & 0x0000'0000'0FFF'FFFE) == 0)) {
+    double r_d = result.get_val();
+
+    // Perform rounding correction.
+#ifdef LIBC_TARGET_CPU_HAS_FMA
+    double sum_sq_lo = fputil::multiply_add(bd, bd, a_sq - sum_sq);
+    double err = sum_sq_lo - fputil::multiply_add(r_d, r_d, -sum_sq);
+#else
+    fputil::DoubleDouble r_sq = fputil::exact_mult(r_d, r_d);
+    double sum_sq_lo = b_sq - (sum_sq - a_sq);
+    double err = (sum_sq - r_sq.hi) + (sum_sq_lo - r_sq.lo);
+#endif
+
+    if (err > 0) {
+      r_u |= 1;
+    } else if ((err < 0) && (r_u & 1) == 0) {
+      r_u -= 1;
+    } else if ((r_u & 0x0000'0000'1FFF'FFFF) == 0) {
+      // The rounded result is exact.
+      fputil::clear_except_if_required(FE_INEXACT);
     }
+    return static_cast<float>(DoubleBits(r_u).get_val());
   }
 
   return static_cast<float>(result.get_val());
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index c57aa9638ed30..9384df8f5290d 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -2733,6 +2733,8 @@ add_fp_unittest(
     libc-math-smoke-tests
   SRCS
     hypotf_test.cpp
+  HDRS
+    HypotTest.h
   DEPENDS
     libc.src.math.hypotf
     libc.src.__support.FPUtil.fp_bits
@@ -2744,6 +2746,8 @@ add_fp_unittest(
     libc-math-smoke-tests
   SRCS
     hypot_test.cpp
+  HDRS
+    HypotTest.h
   DEPENDS
     libc.src.math.hypot
     libc.src.__support.FPUtil.fp_bits
diff --git a/libc/test/src/math/smoke/HypotTest.h b/libc/test/src/math/smoke/HypotTest.h
index 80e9bb7366dfe..7921d531a3196 100644
--- a/libc/test/src/math/smoke/HypotTest.h
+++ b/libc/test/src/math/smoke/HypotTest.h
@@ -9,30 +9,15 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_HYPOTTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_HYPOTTEST_H
 
-#include "src/__support/FPUtil/FPBits.h"
-#include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include "hdr/math_macros.h"
-
 template <typename T>
-class HypotTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
+class HypotTestTemplate : public LIBC_NAMESPACE::testing::Test {
 private:
   using Func = T (*)(T, T);
-  using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
-  using StorageType = typename FPBits::StorageType;
-
-  const T nan = FPBits::quiet_nan().get_val();
-  const T inf = FPBits::inf(Sign::POS).get_val();
-  const T neg_inf = FPBits::inf(Sign::NEG).get_val();
-  const T zero = FPBits::zero(Sign::POS).get_val();
-  const T neg_zero = FPBits::zero(Sign::NEG).get_val();
 
-  const T max_normal = FPBits::max_normal().get_val();
-  const T min_normal = FPBits::min_normal().get_val();
-  const T max_subnormal = FPBits::max_subnormal().get_val();
-  const T min_subnormal = FPBits::min_subnormal().get_val();
+  DECLARE_SPECIAL_CONSTANTS(T)
 
 public:
   void test_special_numbers(Func func) {
@@ -40,11 +25,13 @@ class HypotTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     // Pythagorean triples.
     constexpr T PYT[N][3] = {{3, 4, 5}, {5, 12, 13}, {8, 15, 17}, {7, 24, 25}};
 
-    EXPECT_FP_EQ(func(inf, nan), inf);
-    EXPECT_FP_EQ(func(nan, neg_inf), inf);
-    EXPECT_FP_EQ(func(nan, nan), nan);
-    EXPECT_FP_EQ(func(nan, zero), nan);
-    EXPECT_FP_EQ(func(neg_zero, nan), nan);
+    EXPECT_FP_EQ(func(inf, sNaN), aNaN);
+    EXPECT_FP_EQ(func(sNaN, neg_inf), aNaN);
+    EXPECT_FP_EQ(func(inf, aNaN), inf);
+    EXPECT_FP_EQ(func(aNaN, neg_inf), inf);
+    EXPECT_FP_EQ(func(aNaN, aNaN), aNaN);
+    EXPECT_FP_EQ(func(aNaN, zero), aNaN);
+    EXPECT_FP_EQ(func(neg_zero, aNaN), aNaN);
 
     for (int i = 0; i < N; ++i) {
       EXPECT_FP_EQ_ALL_ROUNDING(PYT[i][2], func(PYT[i][0], PYT[i][1]));
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index f0c25e658fd18..63fd3faf38aef 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -2020,6 +2020,7 @@ libc_math_function(name = "hypot")
 libc_math_function(
     name = "hypotf",
     additional_deps = [
+        ":__support_fputil_double_double",
         ":__support_fputil_sqrt",
     ],
 )

From 71e2b8df30ad78f5f86bf0d6729296282e061a5c Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Fri, 19 Jul 2024 16:46:31 +0200
Subject: [PATCH 205/486] [clang] NFC, simplify the code in
 CheckExprLifetime.cpp (#99637)

No need to get the Owner/Pointer attr via the type. The Decl has this
attr information.
---
 clang/lib/Sema/CheckExprLifetime.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang/lib/Sema/CheckExprLifetime.cpp b/clang/lib/Sema/CheckExprLifetime.cpp
index d9031256f235f..5c8ef564f30aa 100644
--- a/clang/lib/Sema/CheckExprLifetime.cpp
+++ b/clang/lib/Sema/CheckExprLifetime.cpp
@@ -308,8 +308,7 @@ static bool shouldTrackFirstArgument(const FunctionDecl *FD) {
   const auto *RD = FD->getParamDecl(0)->getType()->getPointeeCXXRecordDecl();
   if (!FD->isInStdNamespace() || !RD || !RD->isInStdNamespace())
     return false;
-  if (!isRecordWithAttr<PointerAttr>(QualType(RD->getTypeForDecl(), 0)) &&
-      !isRecordWithAttr<OwnerAttr>(QualType(RD->getTypeForDecl(), 0)))
+  if (!RD->hasAttr<PointerAttr>() && !RD->hasAttr<OwnerAttr>())
     return false;
   if (FD->getReturnType()->isPointerType() ||
       isRecordWithAttr<PointerAttr>(FD->getReturnType())) {

From f9f6f5a9c7748fe485adee206748adf28a849b79 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 19 Jul 2024 15:48:50 +0100
Subject: [PATCH 206/486] [TBAA] Remove remaining entry BB in check lines for
 tbaa-pointers.c (2)

Missed out of 47c08fb8d79ec.
---
 clang/test/CodeGen/tbaa-pointers.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang/test/CodeGen/tbaa-pointers.c b/clang/test/CodeGen/tbaa-pointers.c
index c036110f45c70..75d8c3d501750 100644
--- a/clang/test/CodeGen/tbaa-pointers.c
+++ b/clang/test/CodeGen/tbaa-pointers.c
@@ -91,8 +91,7 @@ void p4char_const1(const char ****ptr) {
 void p4char_const2(const char **const **ptr) {
   // COMMON-LABEL: define void @p4char_const2(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
-  // COMMON-NEXT:  entry:
-  // COMMON-NEXT:    [[PTR_ADDR:%.+]] = alloca ptr, align 8
+  // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
   // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
   // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
   // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0]]

From 4b9fab591916eec9fd1942f37afe3b137b564089 Mon Sep 17 00:00:00 2001
From: David Truby <david.truby@arm.com>
Date: Fri, 19 Jul 2024 15:55:36 +0100
Subject: [PATCH 207/486] [flang][OpenMP] Implement lastprivate with collapse
 (#99500)

This patch enables the lastprivate clause to be used in the presence of
the collapse clause.

Note: the way we currently implement lastprivate means that this adds a
large number of compare instructions to the end of every iteration of
the loop. This is a clearly non-optimal thing to do, but lastprivate in
general will need re-implementing to prevent this. This is planned as
part of the delayed privatization work. This current implementation is
just a stop-gap measure as generating sub-optimal but working code is
better than crashing out.
---
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp |  55 ++--
 flang/lib/Lower/OpenMP/DataSharingProcessor.h |   7 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |   7 +-
 .../Lower/OpenMP/parallel-wsloop-lastpriv.f90 | 239 ++++++++++++++++++
 4 files changed, 276 insertions(+), 32 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90

diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 7df3905c29990..c1d4c089df3b2 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -139,7 +139,6 @@ void DataSharingProcessor::collectOmpObjectListSymbol(
 }
 
 void DataSharingProcessor::collectSymbolsForPrivatization() {
-  bool hasCollapse = false;
   for (const omp::Clause &clause : clauses) {
     if (const auto &privateClause =
             std::get_if<omp::clause::Private>(&clause.u)) {
@@ -153,16 +152,11 @@ void DataSharingProcessor::collectSymbolsForPrivatization() {
       const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t);
       collectOmpObjectListSymbol(objects, explicitlyPrivatizedSymbols);
       hasLastPrivateOp = true;
-    } else if (std::get_if<omp::clause::Collapse>(&clause.u)) {
-      hasCollapse = true;
     }
   }
 
   for (auto *sym : explicitlyPrivatizedSymbols)
     allPrivatizedSymbols.insert(sym);
-
-  if (hasCollapse && hasLastPrivateOp)
-    TODO(converter.getCurrentLocation(), "Collapse clause with lastprivate");
 }
 
 bool DataSharingProcessor::needBarrier() {
@@ -225,28 +219,39 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
       mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator();
       firOpBuilder.setInsertionPoint(lastOper);
 
-      mlir::Value iv = loopOp.getIVs()[0];
-      mlir::Value ub = loopOp.getUpperBound()[0];
-      mlir::Value step = loopOp.getStep()[0];
-
-      // v = iv + step
-      // cmp = step < 0 ? v < ub : v > ub
-      mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
-      mlir::Value zero =
-          firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
-      mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::slt, step, zero);
-      mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::slt, v, ub);
-      mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::sgt, v, ub);
-      mlir::Value cmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
-          loc, negativeStep, vLT, vGT);
+      mlir::Value cmpOp;
+      llvm::SmallVector<mlir::Value> vs;
+      vs.reserve(loopOp.getIVs().size());
+      for (auto [iv, ub, step] : llvm::zip_equal(
+               loopOp.getIVs(), loopOp.getUpperBound(), loopOp.getStep())) {
+        // v = iv + step
+        // cmp = step < 0 ? v < ub : v > ub
+        mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
+        vs.push_back(v);
+        mlir::Value zero =
+            firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
+        mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
+            loc, mlir::arith::CmpIPredicate::slt, step, zero);
+        mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
+            loc, mlir::arith::CmpIPredicate::slt, v, ub);
+        mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
+            loc, mlir::arith::CmpIPredicate::sgt, v, ub);
+        mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
+            loc, negativeStep, vLT, vGT);
+
+        if (cmpOp) {
+          cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp);
+        } else {
+          cmpOp = icmpOp;
+        }
+      }
 
       auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false);
       firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      assert(loopIV && "loopIV was not set");
-      firOpBuilder.createStoreWithConvert(loc, v, loopIV);
+      for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) {
+        assert(loopIV && "loopIV was not set");
+        firOpBuilder.createStoreWithConvert(loc, v, loopIV);
+      }
       lastPrivIP = firOpBuilder.saveInsertionPoint();
     } else if (mlir::isa<mlir::omp::SectionsOp>(op)) {
       // Already handled by genOMP()
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index 7c0c831ed43a8..1e9d07ec13857 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -71,7 +71,7 @@ class DataSharingProcessor {
   bool hasLastPrivateOp;
   mlir::OpBuilder::InsertPoint lastPrivIP;
   mlir::OpBuilder::InsertPoint insPt;
-  mlir::Value loopIV;
+  llvm::SmallVector<mlir::Value> loopIVs;
   // Symbols in private, firstprivate, and/or lastprivate clauses.
   llvm::SetVector<const semantics::Symbol *> explicitlyPrivatizedSymbols;
   llvm::SetVector<const semantics::Symbol *> defaultSymbols;
@@ -147,10 +147,7 @@ class DataSharingProcessor {
   void processStep1(mlir::omp::PrivateClauseOps *clauseOps = nullptr);
   void processStep2(mlir::Operation *op, bool isLoop);
 
-  void setLoopIV(mlir::Value iv) {
-    assert(!loopIV && "Loop iteration variable already set");
-    loopIV = iv;
-  }
+  void pushLoopIV(mlir::Value iv) { loopIVs.push_back(iv); }
 
   const llvm::SetVector<const semantics::Symbol *> &
   getAllSymbolsToPrivatize() const {
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 47ce2d8c8db87..625a8c83c369b 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -677,8 +677,11 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
         assert(tempDsp.has_value());
         tempDsp->processStep2(privatizationTopLevelOp, isLoop);
       } else {
-        if (isLoop && regionArgs.size() > 0)
-          info.dsp->setLoopIV(info.converter.getSymbolAddress(*regionArgs[0]));
+        if (isLoop && regionArgs.size() > 0) {
+          for (const auto &regionArg : regionArgs) {
+            info.dsp->pushLoopIV(info.converter.getSymbolAddress(*regionArg));
+          }
+        }
         info.dsp->processStep2(privatizationTopLevelOp, isLoop);
       }
     }
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
new file mode 100644
index 0000000000000..9b90ac28f693c
--- /dev/null
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
@@ -0,0 +1,239 @@
+! This test checks lowering of OpenMP parallel DO, with the loop bound being
+! a lastprivate variable
+
+! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
+
+! CHECK: func @_QPomp_do_lastprivate(%[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"})
+subroutine omp_do_lastprivate(a)
+  ! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_do_lastprivateEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  integer::a
+  integer::n
+  n = a+1
+  !$omp parallel do lastprivate(a)
+  ! CHECK:  omp.parallel {
+
+  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_do_lastprivateEa"}
+  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[LB:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: %[[UB:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[STEP:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: omp.wsloop {
+  ! CHECK-NEXT: omp.loop_nest (%[[ARG1:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
+  ! CHECK-NEXT: fir.store %[[ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+  ! CHECK:      %[[NEXT_ARG1:.*]] = arith.addi %[[ARG1]], %[[STEP]] : i32
+  ! CHECK:      %[[ZERO:.*]] = arith.constant 0 : i32
+  ! CHECK:      %[[STEP_DIR:.*]] = arith.cmpi slt, %[[STEP]], %[[ZERO]] : i32
+  ! CHECK:      %[[LT_UB:.*]] = arith.cmpi slt, %[[NEXT_ARG1]], %[[UB]] : i32
+  ! CHECK:      %[[GT_UB:.*]] = arith.cmpi sgt, %[[NEXT_ARG1]], %[[UB]] : i32
+  ! CHECK:      %[[SEL:.*]] = arith.select %[[STEP_DIR]], %[[LT_UB]], %[[GT_UB]] : i1
+  ! CHECK:      fir.if %[[SEL]] {
+  ! CHECK:        fir.store %[[NEXT_ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK:        %[[A_PVT_LOAD:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:        hlfir.assign %[[A_PVT_LOAD]] to %[[ARG0_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+  ! CHECK:      }
+
+  ! CHECK-NEXT: omp.yield
+  ! CHECK-NEXT: }
+  ! CHECK-NEXT: omp.terminator
+  ! CHECK-NEXT: }
+    do i=1, a
+      call foo(i, a)
+    end do
+  !$omp end parallel do
+  !CHECK: fir.call @_QPbar(%[[ARG0_DECL]]#1) {{.*}}: (!fir.ref<i32>) -> ()
+  call bar(a)
+end subroutine omp_do_lastprivate
+
+! CHECK: func @_QPomp_do_lastprivate2(%[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"})
+subroutine omp_do_lastprivate2(a, n)
+  ! CHECK:  %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_do_lastprivate2Ea"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:  %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_do_lastprivate2En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  integer::a
+  integer::n
+  n = a+1
+  !$omp parallel do lastprivate(a, n)
+  ! CHECK:  omp.parallel {
+
+  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, {{.*}}}
+  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[N_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "n", pinned, uniq_name = "_QFomp_do_lastprivate2En"}
+  ! CHECK: %[[N_PVT_DECL:.*]]:2 = hlfir.declare %[[N_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate2En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[LB:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: %[[UB:.*]] = fir.load %[[N_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32
+  ! CHECK: omp.wsloop {
+  ! CHECK-NEXT: omp.loop_nest (%[[ARG2:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
+  ! CHECK: fir.store %[[ARG2]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+  ! CHECK: %[[NEXT_ARG2:.*]] = arith.addi %[[ARG2]], %[[STEP]] : i32
+  ! CHECK: %[[ZERO:.*]] = arith.constant 0 : i32
+  ! CHECK: %[[STEP_DIR:.*]] = arith.cmpi slt, %[[STEP]], %[[ZERO]] : i32
+  ! CHECK: %[[LT_UB:.*]] = arith.cmpi slt, %[[NEXT_ARG2]], %[[UB]] : i32
+  ! CHECK: %[[GT_UB:.*]] = arith.cmpi sgt, %[[NEXT_ARG2]], %[[UB]] : i32
+  ! CHECK: %[[SEL:.*]] = arith.select %[[STEP_DIR]], %[[LT_UB]], %[[GT_UB]] : i1
+  ! CHECK: fir.if %[[SEL]] {
+  ! CHECK:   fir.store %[[NEXT_ARG2]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK:   %[[A_PVT_LOAD:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:   hlfir.assign %[[A_PVT_LOAD]] to %[[ARG0_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+  ! CHECK:   %[[N_PVT_LOAD:.*]] = fir.load %[[N_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:   hlfir.assign %[[N_PVT_LOAD]] to %[[ARG1_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+  ! CHECK: }
+
+  ! CHECK: omp.yield
+  ! CHECK: omp.terminator
+    do i= a, n
+      call foo(i, a)
+    end do
+  !$omp end parallel do
+  !CHECK: fir.call @_QPbar(%[[ARG1_DECL]]#1) {{.*}}: (!fir.ref<i32>) -> ()
+  call bar(n)
+end subroutine omp_do_lastprivate2
+
+! CHECK: func @_QPomp_do_lastprivate_collapse2(%[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"})
+subroutine omp_do_lastprivate_collapse2(a)
+  ! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_do_lastprivate_collapse2Ea"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  integer::a
+  !$omp parallel do lastprivate(a) collapse(2)
+  ! CHECK:  omp.parallel {
+
+  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_do_lastprivate_collapse2Ea"}
+  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  !
+  ! CHECK: %[[J_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "j", pinned, {{.*}}}
+  ! CHECK: %[[J_PVT_DECL:.*]]:2 = hlfir.declare %[[J_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse2Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[LB1:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: %[[UB1:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[STEP1:.*]] = arith.constant 1 : i32
+  ! CHECK: %[[LB2:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: %[[UB2:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[STEP2:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: omp.wsloop {
+  ! CHECK-NEXT: omp.loop_nest (%[[ARG1:.*]], %[[ARG2:.*]]) : i32 = (%[[LB1]], %[[LB2]]) to (%[[UB1]], %[[UB2]]) inclusive step (%[[STEP1]], %[[STEP2]]) {
+  ! CHECK-NEXT: fir.store %[[ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK-NEXT: fir.store %[[ARG2]] to %[[J_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+  ! CHECK:      %[[NEXT_ARG1:.*]] = arith.addi %[[ARG1]], %[[STEP1]] : i32
+  ! CHECK:      %[[ZERO1:.*]] = arith.constant 0 : i32
+  ! CHECK:      %[[STEP1_END:.*]] = arith.cmpi slt, %[[STEP1]], %[[ZERO1]] : i32
+  ! CHECK:      %[[LT_UB1:.*]] = arith.cmpi slt, %[[NEXT_ARG1]], %[[UB1]] : i32
+  ! CHECK:      %[[GT_UB1:.*]] = arith.cmpi sgt, %[[NEXT_ARG1]], %[[UB1]] : i32
+  ! CHECK:      %[[SEL1:.*]] = arith.select %[[STEP1_END]], %[[LT_UB1]], %[[GT_UB1]] : i1
+  ! CHECK:      %[[NEXT_ARG2:.*]] = arith.addi %[[ARG2]], %[[STEP2]] : i32
+  ! CHECK:      %[[ZERO2:.*]] = arith.constant 0 : i32
+  ! CHECK:      %[[STEP2_END:.*]] = arith.cmpi slt, %[[STEP2]], %[[ZERO2]] : i32
+  ! CHECK:      %[[LT_UB2:.*]] = arith.cmpi slt, %[[NEXT_ARG2]], %[[UB2]] : i32
+  ! CHECK:      %[[GT_UB2:.*]] = arith.cmpi sgt, %[[NEXT_ARG2]], %[[UB2]] : i32
+  ! CHECK:      %[[SEL2:.*]] = arith.select %[[STEP2_END]], %[[LT_UB2]], %[[GT_UB2]] : i1
+  ! CHECK:      %[[AND:.*]] = arith.andi %[[SEL1]], %[[SEL2]] : i1
+  ! CHECK:      fir.if %[[AND]] {
+  ! CHECK:        fir.store %[[NEXT_ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK:        fir.store %[[NEXT_ARG2]] to %[[J_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK:        %[[A_PVT_LOAD:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:        hlfir.assign %[[A_PVT_LOAD]] to %[[ARG0_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+  ! CHECK:      }
+
+  ! CHECK-NEXT: omp.yield
+  ! CHECK-NEXT: }
+  ! CHECK-NEXT: omp.terminator
+  ! CHECK-NEXT: }
+    do i=1, a
+      do j=1, a
+        call foo(i, a)
+      end do
+    end do
+  !$omp end parallel do
+  !CHECK: fir.call @_QPbar(%[[ARG0_DECL]]#1) {{.*}}: (!fir.ref<i32>) -> ()
+  call bar(a)
+end subroutine omp_do_lastprivate_collapse2
+
+! CHECK: func @_QPomp_do_lastprivate_collapse3(%[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"})
+subroutine omp_do_lastprivate_collapse3(a)
+  ! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_do_lastprivate_collapse3Ea"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  integer::a
+  !$omp parallel do lastprivate(a) collapse(3)
+  ! CHECK:  omp.parallel {
+
+  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_do_lastprivate_collapse3Ea"}
+  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[J_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "j", pinned, {{.*}}}
+  ! CHECK: %[[J_PVT_DECL:.*]]:2 = hlfir.declare %[[J_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[K_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "k", pinned, {{.*}}}
+  ! CHECK: %[[K_PVT_DECL:.*]]:2 = hlfir.declare %[[K_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  ! CHECK: %[[LB1:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: %[[UB1:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[STEP1:.*]] = arith.constant 1 : i32
+  ! CHECK: %[[LB2:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: %[[UB2:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[STEP2:.*]] = arith.constant 1 : i32
+  ! CHECK: %[[LB3:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: %[[UB3:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[STEP3:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: omp.wsloop {
+  ! CHECK-NEXT: omp.loop_nest (%[[ARG1:.*]], %[[ARG2:.*]], %[[ARG3:.*]]) : i32 = (%[[LB1]], %[[LB2]], %[[LB3]]) to (%[[UB1]], %[[UB2]], %[[UB3]]) inclusive step (%[[STEP1]], %[[STEP2]], %[[STEP3]]) {
+  ! CHECK-NEXT: fir.store %[[ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK-NEXT: fir.store %[[ARG2]] to %[[J_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK-NEXT: fir.store %[[ARG3]] to %[[K_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+  ! CHECK:      %[[NEXT_ARG1:.*]] = arith.addi %[[ARG1]], %[[STEP1]] : i32
+  ! CHECK:      %[[ZERO1:.*]] = arith.constant 0 : i32
+  ! CHECK:      %[[STEP1_END:.*]] = arith.cmpi slt, %[[STEP1]], %[[ZERO1]] : i32
+  ! CHECK:      %[[LT_UB1:.*]] = arith.cmpi slt, %[[NEXT_ARG1]], %[[UB1]] : i32
+  ! CHECK:      %[[GT_UB1:.*]] = arith.cmpi sgt, %[[NEXT_ARG1]], %[[UB1]] : i32
+  ! CHECK:      %[[SEL1:.*]] = arith.select %[[STEP1_END]], %[[LT_UB1]], %[[GT_UB1]] : i1
+  ! CHECK:      %[[NEXT_ARG2:.*]] = arith.addi %[[ARG2]], %[[STEP2]] : i32
+  ! CHECK:      %[[ZERO2:.*]] = arith.constant 0 : i32
+  ! CHECK:      %[[STEP2_END:.*]] = arith.cmpi slt, %[[STEP2]], %[[ZERO2]] : i32
+  ! CHECK:      %[[LT_UB2:.*]] = arith.cmpi slt, %[[NEXT_ARG2]], %[[UB2]] : i32
+  ! CHECK:      %[[GT_UB2:.*]] = arith.cmpi sgt, %[[NEXT_ARG2]], %[[UB2]] : i32
+  ! CHECK:      %[[SEL2:.*]] = arith.select %[[STEP2_END]], %[[LT_UB2]], %[[GT_UB2]] : i1
+  ! CHECK:      %[[AND1:.*]] = arith.andi %[[SEL1]], %[[SEL2]] : i1
+  ! CHECK:      %[[NEXT_ARG3:.*]] = arith.addi %[[ARG3]], %[[STEP3]] : i32
+  ! CHECK:      %[[ZERO3:.*]] = arith.constant 0 : i32
+  ! CHECK:      %[[STEP3_END:.*]] = arith.cmpi slt, %[[STEP3]], %[[ZERO3]] : i32
+  ! CHECK:      %[[LT_UB3:.*]] = arith.cmpi slt, %[[NEXT_ARG3]], %[[UB3]] : i32
+  ! CHECK:      %[[GT_UB3:.*]] = arith.cmpi sgt, %[[NEXT_ARG3]], %[[UB3]] : i32
+  ! CHECK:      %[[SEL3:.*]] = arith.select %[[STEP3_END]], %[[LT_UB3]], %[[GT_UB3]] : i1
+  ! CHECK:      %[[AND2:.*]] = arith.andi %[[AND1]], %[[SEL3]] : i1
+  ! CHECK:      fir.if %[[AND2]] {
+  ! CHECK:        fir.store %[[NEXT_ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK:        fir.store %[[NEXT_ARG2]] to %[[J_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK:        fir.store %[[NEXT_ARG3]] to %[[K_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK:        %[[A_PVT_LOAD:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:        hlfir.assign %[[A_PVT_LOAD]] to %[[ARG0_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+  ! CHECK:      }
+
+  ! CHECK-NEXT: omp.yield
+  ! CHECK-NEXT: }
+  ! CHECK-NEXT: omp.terminator
+  ! CHECK-NEXT: }
+    do i=1, a
+      do j=1, a
+        do k=1, a
+          call foo(i, a)
+        end do
+      end do
+    end do
+  !$omp end parallel do
+  !CHECK: fir.call @_QPbar(%[[ARG0_DECL]]#1) {{.*}}: (!fir.ref<i32>) -> ()
+  call bar(a)
+end subroutine omp_do_lastprivate_collapse3

From 9d86722eeecbb1dd6a53c10f7c44fdc317de6809 Mon Sep 17 00:00:00 2001
From: Dhruv Chauhan <dhruv.chauhan@arm.com>
Date: Fri, 19 Jul 2024 16:11:23 +0100
Subject: [PATCH 208/486] [TOSA] Add lowering for `tosa.sin` and `tosa.cos`
 (#99651)

Lower tosa ops to `mlir::math::SinOp` and `mlir::math::CosOp` as part of
the tosa to linalg conversion.
Added lit tests for conversion.
---
 mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp     | 10 ++++++++++
 mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir |  8 ++++++++
 2 files changed, 18 insertions(+)

diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 1442f2ad72255..ba259d4b84fce 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -298,6 +298,14 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   if (isa<tosa::ExpOp>(op) && isa<FloatType>(elementTy))
     return rewriter.create<mlir::math::ExpOp>(loc, resultTypes, args);
 
+  // tosa::SinOp
+  if (isa<tosa::SinOp>(op) && isa<FloatType>(elementTy))
+    return rewriter.create<mlir::math::SinOp>(loc, resultTypes, args);
+
+  // tosa::CosOp
+  if (isa<tosa::CosOp>(op) && isa<FloatType>(elementTy))
+    return rewriter.create<mlir::math::CosOp>(loc, resultTypes, args);
+
   // tosa::TanhOp
   if (isa<tosa::TanhOp>(op) && isa<FloatType>(elementTy))
     return rewriter.create<mlir::math::TanhOp>(loc, resultTypes, args);
@@ -2598,6 +2606,8 @@ void mlir::tosa::populateTosaToLinalgConversionPatterns(
       PointwiseConverter<tosa::LogOp>,
       PointwiseConverter<tosa::ExpOp>,
       PointwiseConverter<tosa::AbsOp>,
+      PointwiseConverter<tosa::SinOp>,
+      PointwiseConverter<tosa::CosOp>,
       PointwiseConverter<tosa::TanhOp>,
       PointwiseConverter<tosa::ErfOp>,
       PointwiseConverter<tosa::BitwiseAndOp>,
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
index 5187d79fd4c0b..0e35f8ea9d0cd 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
@@ -545,6 +545,14 @@ func.func @test_simple_f32(%arg0: tensor<1xf32>) -> () {
   // CHECK: math.erf
   %24 = tosa.erf %0 : (tensor<1xf32>) -> tensor<1xf32>
 
+  // CHECK: linalg.generic
+  // CHECK: math.sin
+  %25 = tosa.sin %arg0 : (tensor<1xf32>) -> tensor<1xf32>
+
+  // CHECK: linalg.generic
+  // CHECK: math.cos
+  %26 = tosa.cos %arg0 : (tensor<1xf32>) -> tensor<1xf32>
+
   return
 }
 

From ac11430983d0d89b7ccd8e10a1a3d02ad7e2208d Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Fri, 19 Jul 2024 17:20:16 +0200
Subject: [PATCH 209/486] [libc] Fix missing sysroot path for kernel headers
 when crosscompiling (#99588)

When crosscompiling, we need to search for the linux kernel headers in the sysroot but since #97486 the linux kernel headers were always searched in /usr/include.

This patch fixes this behaviour by prepending a '=' to where we search for the kernel headers. As per the gcc/clang's documentation a '=' before the path is replaced by the sysroot.

This patch also includes a fix for rv32, that fails to compile due to a missing definition of CLOCK_REALTIME after this change.
---
 libc/cmake/modules/LLVMLibCCompileOptionRules.cmake | 6 +++++-
 libc/src/time/linux/nanosleep.cpp                   | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index 62e16d52cb3ea..a722fe709af5f 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -88,7 +88,11 @@ function(_get_common_compile_options output_var flags)
          (LIBC_CC_SUPPORTS_NOSTDLIBINC OR COMPILER_RESOURCE_DIR))
         # We use -idirafter to avoid preempting libc's own headers in case the
         # directory (e.g. /usr/include) contains other headers.
-        list(APPEND compile_options "-idirafter${LIBC_KERNEL_HEADERS}")
+        if(CMAKE_CROSSCOMPILING)
+          list(APPEND compile_options "-idirafter=${LIBC_KERNEL_HEADERS}")
+        else()
+          list(APPEND compile_options "-idirafter${LIBC_KERNEL_HEADERS}")
+        endif()
       endif()
     endif()
 
diff --git a/libc/src/time/linux/nanosleep.cpp b/libc/src/time/linux/nanosleep.cpp
index b267c3238b01f..7a856376ffb20 100644
--- a/libc/src/time/linux/nanosleep.cpp
+++ b/libc/src/time/linux/nanosleep.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/time/nanosleep.h"
-
+#include "hdr/time_macros.h"
 #include "src/__support/OSUtil/syscall.h" // For syscall functions.
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"

From cf50a84dc0327c1ff732011d74d2c126a2b93af6 Mon Sep 17 00:00:00 2001
From: Akira Hatanaka <ahatanak@gmail.com>
Date: Fri, 19 Jul 2024 08:27:16 -0700
Subject: [PATCH 210/486] [PAC] Authenticate function pointers in UBSan type
 checks (#99590)

The function pointer needs to be authenticated before doing the type
checks.
---
 clang/lib/CodeGen/Address.h           | 4 +++-
 clang/lib/CodeGen/CGExpr.cpp          | 9 +++++++++
 clang/lib/CodeGen/CGPointerAuth.cpp   | 4 ++++
 clang/test/CodeGen/ubsan-function.cpp | 4 ++++
 4 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index 1c4d2e103b5e7..a18c7169af1eb 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -249,7 +249,9 @@ class Address {
   /// Return the pointer contained in this class after authenticating it and
   /// adding offset to it if necessary.
   llvm::Value *emitRawPointer(CodeGenFunction &CGF) const {
-    return getBasePointer();
+    if (!isSigned())
+      return getBasePointer();
+    return emitRawPointerSlow(CGF);
   }
 
   /// Return address with different pointer, but same element type and
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index c5341e4b53651..aa53f96358044 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -5837,6 +5837,15 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee
           CGM.getLLVMContext(), {PrefixSigType, Int32Ty}, /*isPacked=*/true);
 
       llvm::Value *CalleePtr = Callee.getFunctionPointer();
+      if (CGM.getCodeGenOpts().PointerAuth.FunctionPointers) {
+        // Use raw pointer since we are using the callee pointer as data here.
+        Address Addr =
+            Address(CalleePtr, CalleePtr->getType(),
+                    CharUnits::fromQuantity(
+                        CalleePtr->getPointerAlignment(CGM.getDataLayout())),
+                    Callee.getPointerAuthInfo(), nullptr);
+        CalleePtr = Addr.emitRawPointer(*this);
+      }
 
       // On 32-bit Arm, the low bit of a function pointer indicates whether
       // it's using the Arm or Thumb instruction set. The actual first
diff --git a/clang/lib/CodeGen/CGPointerAuth.cpp b/clang/lib/CodeGen/CGPointerAuth.cpp
index 7fe62c0788742..72aed9f24d595 100644
--- a/clang/lib/CodeGen/CGPointerAuth.cpp
+++ b/clang/lib/CodeGen/CGPointerAuth.cpp
@@ -566,6 +566,10 @@ Address Address::getResignedAddress(const CGPointerAuthInfo &NewInfo,
                  /*Offset=*/nullptr, isKnownNonNull());
 }
 
+llvm::Value *Address::emitRawPointerSlow(CodeGenFunction &CGF) const {
+  return CGF.getAsNaturalPointerTo(*this, QualType());
+}
+
 llvm::Value *LValue::getPointer(CodeGenFunction &CGF) const {
   assert(isSimple());
   return emitResignedPointer(getType(), CGF);
diff --git a/clang/test/CodeGen/ubsan-function.cpp b/clang/test/CodeGen/ubsan-function.cpp
index bc37a61c98ee3..8478f05a10b78 100644
--- a/clang/test/CodeGen/ubsan-function.cpp
+++ b/clang/test/CodeGen/ubsan-function.cpp
@@ -4,6 +4,8 @@
 // RUN: %clang_cc1 -triple aarch64_be-linux-gnu -emit-llvm -o - %s -fsanitize=function -fno-sanitize-recover=all | FileCheck %s --check-prefixes=CHECK,GNU,64
 // RUN: %clang_cc1 -triple arm-none-eabi -emit-llvm -o - %s -fsanitize=function -fno-sanitize-recover=all | FileCheck %s --check-prefixes=CHECK,ARM,GNU,32
 
+// RUN: %clang_cc1 -triple arm64e-apple-ios -emit-llvm -o - %s -fsanitize=function -fno-sanitize-recover=all -fptrauth-calls | FileCheck %s --check-prefixes=CHECK,GNU,64,64e
+
 // GNU:  define{{.*}} void @_Z3funv() #0 !func_sanitize ![[FUNCSAN:.*]] {
 // MSVC: define{{.*}} void @"?fun@@YAXXZ"() #0 !func_sanitize ![[FUNCSAN:.*]] {
 void fun() {}
@@ -13,6 +15,8 @@ void fun() {}
 // ARM:   ptrtoint ptr {{.*}} to i32, !nosanitize !5
 // ARM:   and i32 {{.*}}, -2, !nosanitize !5
 // ARM:   inttoptr i32 {{.*}} to ptr, !nosanitize !5
+// 64e:   %[[STRIPPED:.*]] = ptrtoint ptr {{.*}} to i64, !nosanitize
+// 64e:   call i64 @llvm.ptrauth.auth(i64 %[[STRIPPED]], i32 0, i64 0), !nosanitize
 // CHECK: getelementptr <{ i32, i32 }>, ptr {{.*}}, i32 -1, i32 0, !nosanitize
 // CHECK: load i32, ptr {{.*}}, align {{.*}}, !nosanitize
 // CHECK: icmp eq i32 {{.*}}, -1056584962, !nosanitize

From d463617d7772c5e3275cf8c4a8a21d17b3d03d86 Mon Sep 17 00:00:00 2001
From: Akira Hatanaka <ahatanak@gmail.com>
Date: Fri, 19 Jul 2024 08:28:29 -0700
Subject: [PATCH 211/486] [PAC] Fix a crash when signing a pointer to a
 function with an incomplete enum parameter (#99595)

Use int as the underlying type when the enum type is incomplete.
---
 clang/lib/AST/ASTContext.cpp                             | 6 ++++--
 clang/test/CodeGen/ptrauth-function-type-discriminator.c | 7 +++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 98ce88eb6a9fd..90bcbea072e39 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -3223,14 +3223,16 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
     OS << "<objc_object>";
     return;
 
-  case Type::Enum:
+  case Type::Enum: {
     // C11 6.7.2.2p4:
     //   Each enumerated type shall be compatible with char, a signed integer
     //   type, or an unsigned integer type.
     //
     // So we have to treat enum types as integers.
+    QualType UnderlyingType = cast<EnumType>(T)->getDecl()->getIntegerType();
     return encodeTypeForFunctionPointerAuth(
-        Ctx, OS, cast<EnumType>(T)->getDecl()->getIntegerType());
+        Ctx, OS, UnderlyingType.isNull() ? Ctx.IntTy : UnderlyingType);
+  }
 
   case Type::FunctionNoProto:
   case Type::FunctionProto: {
diff --git a/clang/test/CodeGen/ptrauth-function-type-discriminator.c b/clang/test/CodeGen/ptrauth-function-type-discriminator.c
index 58717015adb6c..0952c1abf6c07 100644
--- a/clang/test/CodeGen/ptrauth-function-type-discriminator.c
+++ b/clang/test/CodeGen/ptrauth-function-type-discriminator.c
@@ -30,6 +30,13 @@ void (*test_constant_null)(int) = 0;
 // CHECK: @test_constant_cast = global ptr ptrauth (ptr @f, i32 0, i64 2712)
 void (*test_constant_cast)(int) = (void (*)(int))f;
 
+#ifndef __cplusplus
+// CHECKC: @enum_func_ptr = global ptr ptrauth (ptr @enum_func, i32 0, i64 2712)
+enum Enum0;
+void enum_func(enum Enum0);
+void (*enum_func_ptr)(enum Enum0) = enum_func;
+#endif
+
 // CHECK: @test_opaque = global ptr ptrauth (ptr @f, i32 0)
 void *test_opaque =
 #ifdef __cplusplus

From 2d69c3628bb23a33e5fa36c6ec02fdc57c622dde Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 19 Jul 2024 08:28:12 -0700
Subject: [PATCH 212/486] [tsan] Consume leading zeroes in a test

Follow up to #98578
---
 compiler-rt/test/tsan/debug_alloc_stack.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/compiler-rt/test/tsan/debug_alloc_stack.cpp b/compiler-rt/test/tsan/debug_alloc_stack.cpp
index 5deed49b18b28..70e9759f0af63 100644
--- a/compiler-rt/test/tsan/debug_alloc_stack.cpp
+++ b/compiler-rt/test/tsan/debug_alloc_stack.cpp
@@ -61,11 +61,11 @@ int main() {
   fprintf(stderr, "thread os id = 0x%llx\n", thread_os_id);
   // CHECK: thread os id = [[THREAD_OS_ID]]
   fprintf(stderr, "%p\n", trace[0]);
-  // CHECK: [[ALLOC_FRAME_0:0x[0-9a-f]+]]
+  // CHECK: 0x{{0*}}[[ALLOC_FRAME_0:[0-9a-f]+]]
   fprintf(stderr, "%p\n", trace[1]);
-  // CHECK: [[ALLOC_FRAME_1:0x[0-9a-f]+]]
+  // CHECK: 0x{{0*}}[[ALLOC_FRAME_1:[0-9a-f]+]]
   fprintf(stderr, "%p\n", trace[2]);
-  // CHECK: [[ALLOC_FRAME_2:0x[0-9a-f]+]]
+  // CHECK: 0x{{0*}}[[ALLOC_FRAME_2:[0-9a-f]+]]
 
   pthread_create(&t, NULL, RaceThread, NULL);
   barrier_wait(&barrier);
@@ -79,6 +79,6 @@ int main() {
 
 // CHECK: WARNING: ThreadSanitizer: data race
 // CHECK: Location is heap block of size 10 at {{.*}} allocated by thread T1
-// CHECK: #0 [[ALLOC_FRAME_0]]
-// CHECK: #1 [[ALLOC_FRAME_1]] in alloc_func
-// CHECK: #2 [[ALLOC_FRAME_2]] in AllocThread
+// CHECK: #0 0x{{0*}}[[ALLOC_FRAME_0]]
+// CHECK: #1 0x{{0*}}[[ALLOC_FRAME_1]] in alloc_func
+// CHECK: #2 0x{{0*}}[[ALLOC_FRAME_2]] in AllocThread

From d1ca1d017fe2712dd53adb6072fe1c1e5e1c7cb1 Mon Sep 17 00:00:00 2001
From: Guillermo Callaghan <guillermo.callaghan@huawei.com>
Date: Fri, 19 Jul 2024 16:36:12 +0100
Subject: [PATCH 213/486] [mlir] Makes `zip_shortest` an optional keyword in
 `transform.foreach` (#98492)

This PR addresses a [comment] made by @ftynse about the syntax for
`ForeachOp`. The syntax was modified by @muneebkhan85 in #82792, where
the attribute dictionary was moved to the middle.
This patch moves it back to its original place at the end. And
introduces an optional keyword for `zip_shortest`.

[comment]:
https://github.com/llvm/llvm-project/pull/82792#pullrequestreview-2132814144
---
 mlir/include/mlir/Dialect/Transform/IR/TransformOps.td | 5 +++--
 mlir/lib/Dialect/Transform/IR/TransformOps.cpp         | 6 +++---
 mlir/test/Dialect/Linalg/continuous-tiling-full.mlir   | 2 +-
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
index 7a661c663e010..b946fc8875860 100644
--- a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
+++ b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
@@ -649,11 +649,12 @@ def ForeachOp : TransformDialectOp<"foreach",
   }];
 
   let arguments = (ins Variadic<Transform_AnyHandleOrParamType>:$targets,
-                       UnitAttr:$zip_shortest);
+                       UnitAttr:$with_zip_shortest);
   let results = (outs Variadic<Transform_AnyHandleOrParamType>:$results);
   let regions = (region SizedRegion<1>:$body);
   let assemblyFormat =
-    "$targets attr-dict `:` type($targets) (`->` type($results)^)? $body";
+    "$targets oilist(`with_zip_shortest` $with_zip_shortest) `:` "
+    "type($targets) (`->` type($results)^)? $body attr-dict";
   let hasVerifier = 1;
 
   let extraClassDeclaration = [{
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 3d6a9a18d9f41..c4238080533be 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -1396,11 +1396,11 @@ transform::ForeachOp::apply(transform::TransformRewriter &rewriter,
   SmallVector<SmallVector<MappedValue>> payloads;
   detail::prepareValueMappings(payloads, getTargets(), state);
   size_t numIterations = payloads.empty() ? 0 : payloads.front().size();
-  bool isZipShortest = getZipShortest();
+  bool withZipShortest = getWithZipShortest();
 
   // In case of `zip_shortest`, set the number of iterations to the
   // smallest payload in the targets.
-  if (isZipShortest) {
+  if (withZipShortest) {
     numIterations =
         llvm::min_element(payloads, [&](const SmallVector<MappedValue> &A,
                                         const SmallVector<MappedValue> &B) {
@@ -1414,7 +1414,7 @@ transform::ForeachOp::apply(transform::TransformRewriter &rewriter,
   // As we will be "zipping" over them, check all payloads have the same size.
   // `zip_shortest` adjusts all payloads to the same size, so skip this check
   // when true.
-  for (size_t argIdx = 1; !isZipShortest && argIdx < payloads.size();
+  for (size_t argIdx = 1; !withZipShortest && argIdx < payloads.size();
        argIdx++) {
     if (payloads[argIdx].size() != numIterations) {
       return emitSilenceableError()
diff --git a/mlir/test/Dialect/Linalg/continuous-tiling-full.mlir b/mlir/test/Dialect/Linalg/continuous-tiling-full.mlir
index b61c727f9cffc..7410ff593d01a 100644
--- a/mlir/test/Dialect/Linalg/continuous-tiling-full.mlir
+++ b/mlir/test/Dialect/Linalg/continuous-tiling-full.mlir
@@ -127,7 +127,7 @@ module attributes {transform.with_named_sequence} {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %tile_sizes, %chunk_sizes = transform.structured.continuous_tile_sizes %0 { dimension = 0, target_size = 9 } : (!transform.any_op) -> !transform.any_op
     %linalg_splits, %empty = transform.structured.split %0 after %chunk_sizes { dimension = 0, multiway } : !transform.any_op, !transform.any_op
-    transform.foreach %linalg_splits, %tile_sizes {zip_shortest} : !transform.any_op, !transform.any_op {
+    transform.foreach %linalg_splits, %tile_sizes with_zip_shortest : !transform.any_op, !transform.any_op {
     ^bb1(%linalg_split: !transform.any_op, %tile_size: !transform.any_op):
       %tiled_linalg_split, %dim0_loop = transform.structured.tile_using_for %linalg_split tile_sizes [%tile_size] : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
       transform.yield

From da44c0695c0c0a0865687b8c4b6fad7d786a7e40 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 19 Jul 2024 10:46:35 -0500
Subject: [PATCH 214/486] [libc++][docs] Add tip for developers running the
 test suite on macOS (#99544)

---
 libcxx/docs/TestingLibcxx.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst
index 6d3417cabfd61..2457ed3c78645 100644
--- a/libcxx/docs/TestingLibcxx.rst
+++ b/libcxx/docs/TestingLibcxx.rst
@@ -84,6 +84,12 @@ flags to use, and how to run an executable. This system is meant to be easily
 extended for custom needs, in particular when porting the libc++ test suite to
 new platforms.
 
+.. note::
+  If you run the test suite on Apple platforms, we recommend adding the terminal application
+  used to run the test suite to the list of "Developer Tools". This prevents the system from
+  trying to scan each individual test binary for malware and dramatically speeds up the test
+  suite.
+
 Using a Custom Site Configuration
 ---------------------------------
 

From 2f8c786846972f85a1967502050675c724a2904d Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 19 Jul 2024 10:47:34 -0500
Subject: [PATCH 215/486] [libc++] Refactor buildkite-pipeline.yml (#99483)

This patch removes unused stuff from the Buildkite pipeline definition.
---
 libcxx/utils/ci/buildkite-pipeline.yml | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml
index 71d211bfc287d..0a81268cbe7a1 100644
--- a/libcxx/utils/ci/buildkite-pipeline.yml
+++ b/libcxx/utils/ci/buildkite-pipeline.yml
@@ -17,11 +17,6 @@
 # goal being to reduce the load on testers when a commit is known to fail.
 #
 
-# The Linux CI runners use the nightly ToT build provided by the Docker image.
-# (Note the image isn't updated daily.) The LLVM_HEAD_VERSION contains that
-# version number. The Linux CI runners for GCC use the latest stable version.
-# Theses numbers are available in all runners, making it easier to update the
-# version number.
 env:
     # LLVM POST-BRANCH bump version
     # LLVM POST-BRANCH add compiler test for ToT - 1, e.g. "Clang 17"
@@ -41,20 +36,6 @@ definitions:
       - "**/*.abilist"
       - "**/crash_diagnostics/*"
 
-environment_definitions:
-  _common_env: &common_env
-      LLVM_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer-${LLVM_HEAD_VERSION}"
-      CLANG_CRASH_DIAGNOSTICS_DIR: "crash_diagnostics"
-      CC: clang-${LLVM_HEAD_VERSION}
-      CXX: clang++-${LLVM_HEAD_VERSION}
-
-  _absolute_path_clang: &absolute_path_clang
-    # Note modules require and absolute path for clang-scan-deps
-    # https://github.com/llvm/llvm-project/issues/61006
-    CC: /usr/lib/llvm-${LLVM_HEAD_VERSION}/bin/clang
-    CXX: /usr/lib/llvm-${LLVM_HEAD_VERSION}/bin/clang++
-
-
 steps:
 - group: ':mac: Apple'
   steps:

From 600d4937521210eb8c5fd3e1107e50ec5cb246f2 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Thu, 11 Jul 2024 20:19:39 +0100
Subject: [PATCH 216/486] [AArch64] Remove superfluous sxtw in peephole opt
 (#96293)

Across a basic-block we might have an i32 extract from a value that only
operates on upper bits (for example a sxtw). We can replace the COPY
with a new version skipping the sxtw.

This is a re-commit of 7f2a5dfe35f8bbaca2819644c7aa844f938befd6, with a fix for
removing all the intermediate COPY nodes (and some extra debug logging).
---
 .../Target/AArch64/AArch64MIPeepholeOpt.cpp   | 38 ++++++++
 llvm/lib/Target/AArch64/peephole-sxtw.mir     | 97 +++++++++++++++++++
 .../CodeGen/AArch64/aarch64-mull-masks.ll     | 30 ++++--
 3 files changed, 157 insertions(+), 8 deletions(-)
 create mode 100644 llvm/lib/Target/AArch64/peephole-sxtw.mir

diff --git a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
index bd11bc4dd6e3f..f61de8ff1a4a6 100644
--- a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
@@ -128,6 +128,7 @@ struct AArch64MIPeepholeOpt : public MachineFunctionPass {
   bool visitINSviGPR(MachineInstr &MI, unsigned Opc);
   bool visitINSvi64lane(MachineInstr &MI);
   bool visitFMOVDr(MachineInstr &MI);
+  bool visitCopy(MachineInstr &MI);
   bool runOnMachineFunction(MachineFunction &MF) override;
 
   StringRef getPassName() const override {
@@ -690,6 +691,40 @@ bool AArch64MIPeepholeOpt::visitFMOVDr(MachineInstr &MI) {
   return true;
 }
 
+// Across a basic-block we might have in i32 extract from a value that only
+// operates on upper bits (for example a sxtw). We can replace the COPY with a
+// new version skipping the sxtw.
+bool AArch64MIPeepholeOpt::visitCopy(MachineInstr &MI) {
+  Register InputReg = MI.getOperand(1).getReg();
+  if (MI.getOperand(1).getSubReg() != AArch64::sub_32 ||
+      !MRI->hasOneNonDBGUse(InputReg))
+    return false;
+
+  MachineInstr *SrcMI = MRI->getUniqueVRegDef(InputReg);
+  SmallPtrSet<MachineInstr *, 4> DeadInstrs;
+  DeadInstrs.insert(SrcMI);
+  while (SrcMI && SrcMI->isFullCopy() &&
+         MRI->hasOneNonDBGUse(SrcMI->getOperand(1).getReg())) {
+    SrcMI = MRI->getUniqueVRegDef(SrcMI->getOperand(1).getReg());
+    DeadInstrs.insert(SrcMI);
+  }
+
+  if (!SrcMI || SrcMI->getOpcode() != AArch64::SBFMXri ||
+      SrcMI->getOperand(2).getImm() != 0 || SrcMI->getOperand(3).getImm() != 31)
+    return false;
+
+  Register SrcReg = SrcMI->getOperand(1).getReg();
+  MRI->constrainRegClass(SrcReg, MRI->getRegClass(InputReg));
+  LLVM_DEBUG(dbgs() << "Optimizing: " << MI);
+  MI.getOperand(1).setReg(SrcReg);
+  LLVM_DEBUG(dbgs() << "        to: " << MI);
+  for (auto *DeadMI : DeadInstrs) {
+    LLVM_DEBUG(dbgs() << "  Removing: " << *DeadMI);
+    DeadMI->eraseFromParent();
+  }
+  return true;
+}
+
 bool AArch64MIPeepholeOpt::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
@@ -771,6 +806,9 @@ bool AArch64MIPeepholeOpt::runOnMachineFunction(MachineFunction &MF) {
       case AArch64::FMOVDr:
         Changed |= visitFMOVDr(MI);
         break;
+      case AArch64::COPY:
+        Changed |= visitCopy(MI);
+        break;
       }
     }
   }
diff --git a/llvm/lib/Target/AArch64/peephole-sxtw.mir b/llvm/lib/Target/AArch64/peephole-sxtw.mir
new file mode 100644
index 0000000000000..274d434bbec67
--- /dev/null
+++ b/llvm/lib/Target/AArch64/peephole-sxtw.mir
@@ -0,0 +1,97 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -run-pass=aarch64-mi-peephole-opt -o - -mtriple=aarch64-unknown-linux -verify-machineinstrs %s | FileCheck %s
+
+---
+name: removeSxtw
+tracksRegLiveness: true
+body: |
+  bb.0.entry:
+    liveins: $x0
+    ; CHECK-LABEL: name: removeSxtw
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32sp = COPY [[COPY]].sub_32
+    ; CHECK-NEXT: [[ADDWri:%[0-9]+]]:gpr32sp = ADDWri [[COPY1]], 1, 0
+    ; CHECK-NEXT: $w0 = COPY [[ADDWri]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr64 = COPY $x0
+    %1:gpr64 = SBFMXri %0:gpr64, 0, 31
+    %2:gpr32sp = COPY %1.sub_32:gpr64
+    %3:gpr32sp = ADDWri %2:gpr32sp, 1, 0
+    $w0 = COPY %3:gpr32sp
+    RET_ReallyLR implicit $w0
+...
+---
+name: extraCopy
+tracksRegLiveness: true
+body: |
+  bb.0.entry:
+    liveins: $x0
+    ; CHECK-LABEL: name: extraCopy
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32sp = COPY [[COPY]].sub_32
+    ; CHECK-NEXT: [[ADDWri:%[0-9]+]]:gpr32sp = ADDWri [[COPY1]], 1, 0
+    ; CHECK-NEXT: $w0 = COPY [[ADDWri]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr64 = COPY $x0
+    %1:gpr64 = SBFMXri %0:gpr64, 0, 31
+    %2:gpr64all = COPY %1:gpr64
+    %3:gpr32sp = COPY %2.sub_32:gpr64all
+    %4:gpr32sp = ADDWri %3:gpr32sp, 1, 0
+    $w0 = COPY %4:gpr32sp
+    RET_ReallyLR implicit $w0
+...
+---
+name: multipleCopies
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: multipleCopies
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gpr64all = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[INSERT_SUBREG:%[0-9]+]]:gpr64 = INSERT_SUBREG [[DEF]], [[COPY]], %subreg.sub_32
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32 = COPY [[INSERT_SUBREG]].sub_32
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32 = COPY $wzr
+  ; CHECK-NEXT:   [[SUBWrr:%[0-9]+]]:gpr32 = SUBWrr [[COPY2]], [[COPY1]]
+  ; CHECK-NEXT:   [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[SUBWrr]], %subreg.sub_32
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.1(0x04000000), %bb.2(0x7c000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   CBZX [[SUBREG_TO_REG]], %bb.1
+  ; CHECK-NEXT:   B %bb.2
+  bb.0.entry:
+    successors: %bb.1(0x80000000)
+    liveins: $w0
+
+    %2:gpr32 = COPY $w0
+    %4:gpr64all = IMPLICIT_DEF
+    %3:gpr64 = INSERT_SUBREG %4, %2, %subreg.sub_32
+    %5:gpr64 = SBFMXri killed %3, 0, 31
+    %0:gpr64all = COPY %5
+    %6:gpr64all = COPY %0
+    %7:gpr32 = COPY %6.sub_32
+    %8:gpr32 = COPY $wzr
+    %9:gpr32 = SUBWrr %8, %7
+    %10:gpr32 = ORRWrs $wzr, %9, 0
+    %1:gpr64 = SUBREG_TO_REG 0, %10, %subreg.sub_32
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+  bb.2:
+    successors: %bb.1(0x04000000), %bb.2(0x7c000000)
+
+    CBZX %1, %bb.1
+    B %bb.2
+
+...
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll b/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll
index 9b0701ab148dc..c57c95fd3b531 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll
@@ -281,8 +281,7 @@ define i64 @smull_ldrsw_shift(ptr %x0, i64 %x1) {
 ; CHECK-LABEL: smull_ldrsw_shift:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtw x9, w1
-; CHECK-NEXT:    smull x0, w8, w9
+; CHECK-NEXT:    smull x0, w8, w1
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
@@ -490,8 +489,7 @@ define i64 @smaddl_ldrsw_shift(ptr %x0, i64 %x1, i64 %x2) {
 ; CHECK-LABEL: smaddl_ldrsw_shift:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtw x9, w1
-; CHECK-NEXT:    smaddl x0, w8, w9, x2
+; CHECK-NEXT:    smaddl x0, w8, w1, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
@@ -654,8 +652,7 @@ define i64 @smnegl_ldrsw_shift(ptr %x0, i64 %x1) {
 ; CHECK-LABEL: smnegl_ldrsw_shift:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtw x9, w1
-; CHECK-NEXT:    smnegl x0, w8, w9
+; CHECK-NEXT:    smnegl x0, w8, w1
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
@@ -818,8 +815,7 @@ define i64 @smsubl_ldrsw_shift(ptr %x0, i64 %x1, i64 %x2) {
 ; CHECK-LABEL: smsubl_ldrsw_shift:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtw x9, w1
-; CHECK-NEXT:    smsubl x0, w8, w9, x2
+; CHECK-NEXT:    smsubl x0, w8, w1, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
@@ -1451,3 +1447,21 @@ define i64 @umaddl_and_and(i64 %x, i64 %y, i64 %a) {
     %add = add i64 %a, %mul
     ret i64 %add
 }
+
+; Check which can contain multiple copies that should all be removed.
+define i32 @f(i32 %0) {
+entry:
+  %1 = sext i32 %0 to i64
+  br label %A
+
+A:
+  %2 = trunc i64 %1 to i32
+  %a69.us = sub i32 0, %2
+  %a69.us.fr = freeze i32 %a69.us
+  %3 = zext i32 %a69.us.fr to i64
+  br label %B
+
+B:
+  %t = icmp eq i64 0, %3
+  br i1 %t, label %A, label %B
+}

From 1bdd761c4c12d016c90db82b36d93b09869703ae Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Fri, 19 Jul 2024 16:51:28 +0100
Subject: [PATCH 217/486] [AArch64] Additional sqxtn and uqxtn tests. NFC

There are also some sqxtun2 tests if I got them correct. The
arm64-sqxtn2-combine.ll test was moved to qmovn to keep them
in the same place.
---
 .../CodeGen/AArch64/arm64-sqxtn2-combine.ll   |  37 ---
 llvm/test/CodeGen/AArch64/qmovn.ll            | 219 +++++++++++++++++-
 2 files changed, 215 insertions(+), 41 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/arm64-sqxtn2-combine.ll

diff --git a/llvm/test/CodeGen/AArch64/arm64-sqxtn2-combine.ll b/llvm/test/CodeGen/AArch64/arm64-sqxtn2-combine.ll
deleted file mode 100644
index f09109c282a20..0000000000000
--- a/llvm/test/CodeGen/AArch64/arm64-sqxtn2-combine.ll
+++ /dev/null
@@ -1,37 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple aarch64-none-linux-gnu | FileCheck %s
-
-; Test the (concat_vectors (X), (trunc(smin(smax(Y, -2^n), 2^n-1))) pattern.
-
-define <16 x i8> @test_combine_v8i16_to_v16i8(<8 x i8> %x, <8 x i16> %y) {
-; CHECK-LABEL: test_combine_v8i16_to_v16i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    sqxtn2 v0.16b, v1.8h
-; CHECK-NEXT:    ret
-entry:
-  %min = call <8 x i16> @llvm.smin.v8i16(<8 x i16> %y, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>)
-  %max = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %min, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>)
-  %trunc = trunc <8 x i16> %max to <8 x i8>
-  %shuffle = shufflevector <8 x i8> %x, <8 x i8> %trunc, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-  ret <16 x i8> %shuffle
-}
-
-define <8 x i16> @test_combine_v4i32_to_v8i16(<4 x i16> %x, <4 x i32> %y) {
-; CHECK-LABEL: test_combine_v4i32_to_v8i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    sqxtn2 v0.8h, v1.4s
-; CHECK-NEXT:    ret
-entry:
-  %max = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %y, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>)
-  %min = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %max, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>)
-  %trunc = trunc <4 x i32> %min to <4 x i16>
-  %shuffle = shufflevector <4 x i16> %x, <4 x i16> %trunc, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  ret <8 x i16> %shuffle
-}
-
-declare <8 x i16> @llvm.smin.v8i16(<8 x i16>, <8 x i16>)
-declare <8 x i16> @llvm.smax.v8i16(<8 x i16>, <8 x i16>)
-declare <4 x i32> @llvm.smin.v4i32(<4 x i32>, <4 x i32>)
-declare <4 x i32> @llvm.smax.v4i32(<4 x i32>, <4 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/qmovn.ll b/llvm/test/CodeGen/AArch64/qmovn.ll
index f4fdb1b97e292..dbdf9c58f8aba 100644
--- a/llvm/test/CodeGen/AArch64/qmovn.ll
+++ b/llvm/test/CodeGen/AArch64/qmovn.ll
@@ -84,9 +84,9 @@ entry:
 define <2 x i32> @vqmovni64_smaxmin(<2 x i64> %s0) {
 ; CHECK-LABEL: vqmovni64_smaxmin:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #2147483647
+; CHECK-NEXT:    mov w8, #2147483647 // =0x7fffffff
 ; CHECK-NEXT:    dup v1.2d, x8
-; CHECK-NEXT:    mov x8, #-2147483648
+; CHECK-NEXT:    mov x8, #-2147483648 // =0xffffffff80000000
 ; CHECK-NEXT:    cmgt v2.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    dup v1.2d, x8
@@ -106,9 +106,9 @@ entry:
 define <2 x i32> @vqmovni64_sminmax(<2 x i64> %s0) {
 ; CHECK-LABEL: vqmovni64_sminmax:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov x8, #-2147483648
+; CHECK-NEXT:    mov x8, #-2147483648 // =0xffffffff80000000
 ; CHECK-NEXT:    dup v1.2d, x8
-; CHECK-NEXT:    mov w8, #2147483647
+; CHECK-NEXT:    mov w8, #2147483647 // =0x7fffffff
 ; CHECK-NEXT:    cmgt v2.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    dup v1.2d, x8
@@ -140,3 +140,214 @@ entry:
   %t = trunc <2 x i64> %s1 to <2 x i32>
   ret <2 x i32> %t
 }
+
+; Test the (concat_vectors (X), (trunc(smin(smax(Y, -2^n), 2^n-1))) pattern.
+
+define <16 x i8> @signed_minmax_v8i16_to_v16i8(<8 x i8> %x, <8 x i16> %y) {
+; CHECK-LABEL: signed_minmax_v8i16_to_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqxtn2 v0.16b, v1.8h
+; CHECK-NEXT:    ret
+entry:
+  %min = call <8 x i16> @llvm.smin.v8i16(<8 x i16> %y, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>)
+  %max = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %min, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>)
+  %trunc = trunc <8 x i16> %max to <8 x i8>
+  %shuffle = shufflevector <8 x i8> %x, <8 x i8> %trunc, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %shuffle
+}
+
+define <8 x i16> @signed_minmax_v4i32_to_v8i16(<4 x i16> %x, <4 x i32> %y) {
+; CHECK-LABEL: signed_minmax_v4i32_to_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqxtn2 v0.8h, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %min = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %y, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>)
+  %max = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %min, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>)
+  %trunc = trunc <4 x i32> %max to <4 x i16>
+  %shuffle = shufflevector <4 x i16> %x, <4 x i16> %trunc, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %shuffle
+}
+
+define <4 x i32> @signed_minmax_v2i64_to_v4i32(<2 x i32> %x, <2 x i64> %y) {
+; CHECK-LABEL: signed_minmax_v2i64_to_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #2147483647 // =0x7fffffff
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v2.2d, x8
+; CHECK-NEXT:    mov x8, #-2147483648 // =0xffffffff80000000
+; CHECK-NEXT:    cmgt v3.2d, v2.2d, v1.2d
+; CHECK-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-NEXT:    dup v2.2d, x8
+; CHECK-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-NEXT:    xtn2 v0.4s, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  %min = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %y, <2 x i64> <i64 2147483647, i64 2147483647>)
+  %max = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %min, <2 x i64> <i64 -2147483648, i64 -2147483648>)
+  %trunc = trunc <2 x i64> %max to <2 x i32>
+  %shuffle = shufflevector <2 x i32> %x, <2 x i32> %trunc, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %shuffle
+}
+
+; Test the (concat_vectors (X), (trunc(smax(smin(Y, 2^n-1), -2^n))) pattern.
+
+define <16 x i8> @signed_maxmin_v8i16_to_v16i8(<8 x i8> %x, <8 x i16> %y) {
+; CHECK-LABEL: signed_maxmin_v8i16_to_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqxtn2 v0.16b, v1.8h
+; CHECK-NEXT:    ret
+entry:
+  %max = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %y, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>)
+  %min = call <8 x i16> @llvm.smin.v8i16(<8 x i16> %max, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>)
+  %trunc = trunc <8 x i16> %min to <8 x i8>
+  %shuffle = shufflevector <8 x i8> %x, <8 x i8> %trunc, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %shuffle
+}
+
+define <8 x i16> @signed_maxmin_v4i32_to_v8i16(<4 x i16> %x, <4 x i32> %y) {
+; CHECK-LABEL: signed_maxmin_v4i32_to_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqxtn2 v0.8h, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %max = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %y, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>)
+  %min = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %max, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>)
+  %trunc = trunc <4 x i32> %min to <4 x i16>
+  %shuffle = shufflevector <4 x i16> %x, <4 x i16> %trunc, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %shuffle
+}
+
+define <4 x i32> @signed_maxmin_v2i64_to_v4i32(<2 x i32> %x, <2 x i64> %y) {
+; CHECK-LABEL: signed_maxmin_v2i64_to_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x8, #-2147483648 // =0xffffffff80000000
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v2.2d, x8
+; CHECK-NEXT:    mov w8, #2147483647 // =0x7fffffff
+; CHECK-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-NEXT:    dup v2.2d, x8
+; CHECK-NEXT:    cmgt v3.2d, v2.2d, v1.2d
+; CHECK-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-NEXT:    xtn2 v0.4s, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  %max = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %y, <2 x i64> <i64 -2147483648, i64 -2147483648>)
+  %min = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %max, <2 x i64> <i64 2147483647, i64 2147483647>)
+  %trunc = trunc <2 x i64> %min to <2 x i32>
+  %shuffle = shufflevector <2 x i32> %x, <2 x i32> %trunc, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %shuffle
+}
+
+; Test the (concat_vectors (X), (trunc(umin(Y, 2^n)))) pattern.
+
+define <16 x i8> @unsigned_v8i16_to_v16i8(<8 x i8> %x, <8 x i16> %y) {
+; CHECK-LABEL: unsigned_v8i16_to_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v2.2d, #0xff00ff00ff00ff
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    umin v1.8h, v1.8h, v2.8h
+; CHECK-NEXT:    xtn2 v0.16b, v1.8h
+; CHECK-NEXT:    ret
+entry:
+  %min = call <8 x i16> @llvm.umin.v8i16(<8 x i16> %y, <8 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
+  %trunc = trunc <8 x i16> %min to <8 x i8>
+  %shuffle = shufflevector <8 x i8> %x, <8 x i8> %trunc, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %shuffle
+}
+
+define <8 x i16> @unsigned_v4i32_to_v8i16(<4 x i16> %x, <4 x i32> %y) {
+; CHECK-LABEL: unsigned_v4i32_to_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v2.2d, #0x00ffff0000ffff
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    umin v1.4s, v1.4s, v2.4s
+; CHECK-NEXT:    xtn2 v0.8h, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %min = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %y, <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>)
+  %trunc = trunc <4 x i32> %min to <4 x i16>
+  %shuffle = shufflevector <4 x i16> %x, <4 x i16> %trunc, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %shuffle
+}
+
+define <4 x i32> @unsigned_v2i64_to_v4i32(<2 x i32> %x, <2 x i64> %y) {
+; CHECK-LABEL: unsigned_v2i64_to_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v2.2d, #0x000000ffffffff
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    cmhi v2.2d, v2.2d, v1.2d
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    orn v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    xtn2 v0.4s, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  %min = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %y, <2 x i64> <i64 4294967295, i64 4294967295>)
+  %trunc = trunc <2 x i64> %min to <2 x i32>
+  %shuffle = shufflevector <2 x i32> %x, <2 x i32> %trunc, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %shuffle
+}
+
+; Test the (concat_vectors (X), (trunc(umin(smax(Y, 0), 2^n))))) pattern.
+
+define <16 x i8> @us_maxmin_v8i16_to_v16i8(<8 x i8> %x, <8 x i16> %y) {
+; CHECK-LABEL: us_maxmin_v8i16_to_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    movi v3.2d, #0xff00ff00ff00ff
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smax v1.8h, v1.8h, v2.8h
+; CHECK-NEXT:    smin v1.8h, v1.8h, v3.8h
+; CHECK-NEXT:    xtn2 v0.16b, v1.8h
+; CHECK-NEXT:    ret
+entry:
+  %max = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %y, <8 x i16> zeroinitializer)
+  %min = call <8 x i16> @llvm.umin.v8i16(<8 x i16> %max, <8 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
+  %trunc = trunc <8 x i16> %min to <8 x i8>
+  %shuffle = shufflevector <8 x i8> %x, <8 x i8> %trunc, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %shuffle
+}
+
+define <8 x i16> @us_maxmin_v4i32_to_v8i16(<4 x i16> %x, <4 x i32> %y) {
+; CHECK-LABEL: us_maxmin_v4i32_to_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smax v1.4s, v1.4s, v2.4s
+; CHECK-NEXT:    movi v2.2d, #0x00ffff0000ffff
+; CHECK-NEXT:    smin v1.4s, v1.4s, v2.4s
+; CHECK-NEXT:    xtn2 v0.8h, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %max = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %y, <4 x i32> zeroinitializer)
+  %min = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %max, <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>)
+  %trunc = trunc <4 x i32> %min to <4 x i16>
+  %shuffle = shufflevector <4 x i16> %x, <4 x i16> %trunc, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %shuffle
+}
+
+define <4 x i32> @us_maxmin_v2i64_to_v4i32(<2 x i32> %x, <2 x i64> %y) {
+; CHECK-LABEL: us_maxmin_v2i64_to_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmgt v2.2d, v1.2d, #0
+; CHECK-NEXT:    movi v3.2d, #0x000000ffffffff
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    cmgt v2.2d, v3.2d, v1.2d
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    orn v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    xtn2 v0.4s, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  %max = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %y, <2 x i64> zeroinitializer)
+  %min = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %max, <2 x i64> <i64 4294967295, i64 4294967295>)
+  %trunc = trunc <2 x i64> %min to <2 x i32>
+  %shuffle = shufflevector <2 x i32> %x, <2 x i32> %trunc, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %shuffle
+}

From 6e68b75e6634e9809e2af5d68c87a4eb4282217e Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes@amd.com>
Date: Mon, 15 Jul 2024 19:46:07 -0700
Subject: [PATCH 218/486] [AMDGPU] Reland: Do not use original PHIs in coercion
 chains

Change-Id: I579b5c69a85997f168ed35354b326524b6f84ef7
---
 .../AMDGPU/AMDGPULateCodeGenPrepare.cpp       |  51 +++-
 llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll | 219 ++++++++++++++++++
 2 files changed, 260 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
index 2cc95f81d2f94..fe7731b9550bf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
@@ -365,29 +365,60 @@ bool LiveRegOptimizer::optimizeLiveType(
       else
         MissingIncVal = true;
     }
-    Instruction *DeadInst = Phi;
     if (MissingIncVal) {
-      DeadInst = cast<Instruction>(ValMap[Phi]);
-      // Do not use the dead phi
-      ValMap[Phi] = Phi;
+      Value *DeadVal = ValMap[Phi];
+      // The coercion chain of the PHI is broken. Delete the Phi
+      // from the ValMap and any connected / user Phis.
+      SmallVector<Value *, 4> PHIWorklist;
+      SmallPtrSet<Value *, 4> Visited;
+      PHIWorklist.push_back(DeadVal);
+      while (!PHIWorklist.empty()) {
+        Value *NextDeadValue = PHIWorklist.pop_back_val();
+        Visited.insert(NextDeadValue);
+        auto OriginalPhi =
+            std::find_if(PhiNodes.begin(), PhiNodes.end(),
+                         [this, &NextDeadValue](PHINode *CandPhi) {
+                           return ValMap[CandPhi] == NextDeadValue;
+                         });
+        // This PHI may have already been removed from maps when
+        // unwinding a previous Phi
+        if (OriginalPhi != PhiNodes.end())
+          ValMap.erase(*OriginalPhi);
+
+        DeadInsts.emplace_back(cast<Instruction>(NextDeadValue));
+
+        for (User *U : NextDeadValue->users()) {
+          if (!Visited.contains(cast<PHINode>(U)))
+            PHIWorklist.push_back(U);
+        }
+      }
+    } else {
+      DeadInsts.emplace_back(cast<Instruction>(Phi));
     }
-    DeadInsts.emplace_back(DeadInst);
   }
   // Coerce back to the original type and replace the uses.
   for (Instruction *U : Uses) {
     // Replace all converted operands for a use.
     for (auto [OpIdx, Op] : enumerate(U->operands())) {
-      if (ValMap.contains(Op)) {
+      if (ValMap.contains(Op) && ValMap[Op]) {
         Value *NewVal = nullptr;
         if (BBUseValMap.contains(U->getParent()) &&
             BBUseValMap[U->getParent()].contains(ValMap[Op]))
           NewVal = BBUseValMap[U->getParent()][ValMap[Op]];
         else {
           BasicBlock::iterator InsertPt = U->getParent()->getFirstNonPHIIt();
-          NewVal =
-              convertFromOptType(Op->getType(), cast<Instruction>(ValMap[Op]),
-                                 InsertPt, U->getParent());
-          BBUseValMap[U->getParent()][ValMap[Op]] = NewVal;
+          // We may pick up ops that were previously converted for users in
+          // other blocks. If there is an originally typed definition of the Op
+          // already in this block, simply reuse it.
+          if (isa<Instruction>(Op) && !isa<PHINode>(Op) &&
+              U->getParent() == cast<Instruction>(Op)->getParent()) {
+            NewVal = Op;
+          } else {
+            NewVal =
+                convertFromOptType(Op->getType(), cast<Instruction>(ValMap[Op]),
+                                   InsertPt, U->getParent());
+            BBUseValMap[U->getParent()][ValMap[Op]] = NewVal;
+          }
         }
         assert(NewVal);
         U->setOperand(OpIdx, NewVal);
diff --git a/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll b/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll
index 5d2e299aa854a..1e60261df1304 100644
--- a/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll
@@ -349,4 +349,223 @@ bb.2:
   ret void
 }
 
+; Should not produce a broken phi
+
+define void @broken_phi() {
+; GFX906-LABEL: define void @broken_phi(
+; GFX906-SAME: ) #[[ATTR0]] {
+; GFX906-NEXT:  bb:
+; GFX906-NEXT:    br label [[BB1:%.*]]
+; GFX906:       bb1:
+; GFX906-NEXT:    [[I:%.*]] = phi <4 x i8> [ <i8 1, i8 1, i8 1, i8 1>, [[BB:%.*]] ], [ [[I8:%.*]], [[BB7:%.*]] ]
+; GFX906-NEXT:    br i1 false, label [[BB3:%.*]], label [[BB2:%.*]]
+; GFX906:       bb2:
+; GFX906-NEXT:    br label [[BB3]]
+; GFX906:       bb3:
+; GFX906-NEXT:    [[I4:%.*]] = phi <4 x i8> [ zeroinitializer, [[BB2]] ], [ [[I]], [[BB1]] ]
+; GFX906-NEXT:    br i1 false, label [[BB7]], label [[BB5:%.*]]
+; GFX906:       bb5:
+; GFX906-NEXT:    [[I6:%.*]] = call <4 x i8> @llvm.smax.v4i8(<4 x i8> [[I4]], <4 x i8> zeroinitializer)
+; GFX906-NEXT:    br label [[BB7]]
+; GFX906:       bb7:
+; GFX906-NEXT:    [[I8]] = phi <4 x i8> [ zeroinitializer, [[BB5]] ], [ zeroinitializer, [[BB3]] ]
+; GFX906-NEXT:    br label [[BB1]]
+;
+bb:
+  br label %bb1
+bb1:
+  %i = phi <4 x i8> [ <i8 1, i8 1, i8 1, i8 1>, %bb ], [ %i8, %bb7 ]
+  br i1 false, label %bb3, label %bb2
+bb2:
+  br label %bb3
+bb3:
+  %i4 = phi <4 x i8> [ zeroinitializer, %bb2 ], [ %i, %bb1 ]
+  br i1 false, label %bb7, label %bb5
+bb5:
+  %i6 = call <4 x i8> @llvm.smax.v4i8(<4 x i8> %i4, <4 x i8> zeroinitializer)
+  br label %bb7
+bb7:
+  %i8 = phi <4 x i8> [ zeroinitializer, %bb5 ], [ zeroinitializer, %bb3 ]
+  br label %bb1
+}
+
+; %sel1 should just use %sel0 instead of trying to convert back the
+; converted version of %sel0
+
+define amdgpu_kernel void @reuseOp() {
+; GFX906-LABEL: define amdgpu_kernel void @reuseOp(
+; GFX906-SAME: ) #[[ATTR0]] {
+; GFX906-NEXT:  entry:
+; GFX906-NEXT:    [[VEC1:%.*]] = insertelement <16 x i8> zeroinitializer, i8 0, i64 0
+; GFX906-NEXT:    [[VEC1_BC:%.*]] = bitcast <16 x i8> [[VEC1]] to <4 x i32>
+; GFX906-NEXT:    br label [[BB_1:%.*]]
+; GFX906:       bb.1:
+; GFX906-NEXT:    [[VEC1_BC_BC:%.*]] = bitcast <4 x i32> [[VEC1_BC]] to <16 x i8>
+; GFX906-NEXT:    [[SEL0:%.*]] = select i1 false, <16 x i8> zeroinitializer, <16 x i8> zeroinitializer
+; GFX906-NEXT:    [[SEL0_BC:%.*]] = bitcast <16 x i8> [[SEL0]] to <4 x i32>
+; GFX906-NEXT:    [[SEL1:%.*]] = select i1 false, <16 x i8> [[VEC1_BC_BC]], <16 x i8> [[SEL0]]
+; GFX906-NEXT:    br label [[BB_2:%.*]]
+; GFX906:       bb.2:
+; GFX906-NEXT:    [[SEL0_BC_BC:%.*]] = bitcast <4 x i32> [[SEL0_BC]] to <16 x i8>
+; GFX906-NEXT:    [[VAL:%.*]] = extractelement <16 x i8> [[SEL0_BC_BC]], i64 0
+; GFX906-NEXT:    ret void
+;
+entry:
+  %vec1 = insertelement <16 x i8> zeroinitializer, i8 0, i64 0
+  br label %bb.1
+
+bb.1:
+  %sel0 = select i1 false, <16 x i8> zeroinitializer, <16 x i8> zeroinitializer
+  %sel1 = select i1 false, <16 x i8> %vec1, <16 x i8> %sel0
+  br label %bb.2
+
+bb.2:
+  %val = extractelement <16 x i8> %sel0, i64 0
+  ret void
+}
+
+
+define amdgpu_kernel void @deletedPHI(i32 %in0, i1 %cmp, <10 x i8> %invec0) {
+; GFX906-LABEL: define amdgpu_kernel void @deletedPHI(
+; GFX906-SAME: i32 [[IN0:%.*]], i1 [[CMP:%.*]], <10 x i8> [[INVEC0:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:  entry:
+; GFX906-NEXT:    br label [[BB_1:%.*]]
+; GFX906:       bb.1:
+; GFX906-NEXT:    [[PHI0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB_11:%.*]] ]
+; GFX906-NEXT:    [[PHI1:%.*]] = phi <10 x i8> [ <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, [[ENTRY]] ], [ [[VEC1:%.*]], [[BB_11]] ]
+; GFX906-NEXT:    br i1 [[CMP]], label [[BB_3:%.*]], label [[BB_2:%.*]]
+; GFX906:       bb.2:
+; GFX906-NEXT:    br label [[BB_3]]
+; GFX906:       bb.3:
+; GFX906-NEXT:    [[PHI2:%.*]] = phi <10 x i8> [ zeroinitializer, [[BB_2]] ], [ [[PHI1]], [[BB_1]] ]
+; GFX906-NEXT:    br i1 [[CMP]], label [[BB_5:%.*]], label [[BB_4:%.*]]
+; GFX906:       bb.4:
+; GFX906-NEXT:    [[VEC0:%.*]] = insertelement <10 x i8> [[PHI2]], i8 0, i64 0
+; GFX906-NEXT:    br label [[BB_5]]
+; GFX906:       bb.5:
+; GFX906-NEXT:    [[PHI3:%.*]] = phi <10 x i8> [ [[VEC0]], [[BB_4]] ], [ [[PHI2]], [[BB_3]] ]
+; GFX906-NEXT:    br i1 [[CMP]], label [[BB_7:%.*]], label [[BB_6:%.*]]
+; GFX906:       bb.6:
+; GFX906-NEXT:    br label [[BB_7]]
+; GFX906:       bb.7:
+; GFX906-NEXT:    [[PHI4:%.*]] = phi <10 x i8> [ [[INVEC0]], [[BB_6]] ], [ [[PHI3]], [[BB_5]] ]
+; GFX906-NEXT:    br i1 [[CMP]], label [[BB_9:%.*]], label [[BB_8:%.*]]
+; GFX906:       bb.8:
+; GFX906-NEXT:    br label [[BB_9]]
+; GFX906:       bb.9:
+; GFX906-NEXT:    [[PHI5:%.*]] = phi <10 x i8> [ [[INVEC0]], [[BB_8]] ], [ [[PHI4]], [[BB_7]] ]
+; GFX906-NEXT:    br i1 [[CMP]], label [[BB_11]], label [[BB_10:%.*]]
+; GFX906:       bb.10:
+; GFX906-NEXT:    br label [[BB_11]]
+; GFX906:       bb.11:
+; GFX906-NEXT:    [[PHI6:%.*]] = phi <10 x i8> [ zeroinitializer, [[BB_10]] ], [ [[PHI5]], [[BB_9]] ]
+; GFX906-NEXT:    [[VEC1]] = shufflevector <10 x i8> [[PHI6]], <10 x i8> zeroinitializer, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 15, i32 16, i32 17, i32 18, i32 19>
+; GFX906-NEXT:    br label [[BB_1]]
+;
+entry:
+  br label %bb.1
+
+bb.1:
+  %phi0 = phi i32 [ 0, %entry ], [ 1, %bb.11 ]
+  %phi1 = phi <10 x i8> [ <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, %entry ], [ %vec1, %bb.11 ]
+  br i1 %cmp, label %bb.3, label %bb.2
+
+bb.2:
+  br label %bb.3
+
+bb.3:
+  %phi2 = phi <10 x i8> [ zeroinitializer, %bb.2 ], [ %phi1, %bb.1 ]
+  br i1 %cmp, label %bb.5, label %bb.4
+
+bb.4:
+  %vec0 = insertelement <10 x i8> %phi2, i8 0, i64 0
+  br label %bb.5
+
+bb.5:                               ; preds = %bb.4, %bb.3
+  %phi3 = phi <10 x i8> [ %vec0, %bb.4 ], [ %phi2, %bb.3 ]
+  br i1 %cmp, label %bb.7, label %bb.6
+
+bb.6:
+  br label %bb.7
+
+bb.7:                               ; preds = %bb.6, %bb.5
+  %phi4 = phi <10 x i8> [ %invec0, %bb.6 ], [ %phi3, %bb.5 ]
+  br i1 %cmp, label %bb.9, label %bb.8
+
+bb.8:
+  br label %bb.9
+
+bb.9:
+  %phi5 = phi <10 x i8> [ %invec0, %bb.8 ], [ %phi4, %bb.7 ]
+  br i1 %cmp, label %bb.11, label %bb.10
+
+bb.10:
+  br label %bb.11
+
+bb.11:
+  %phi6 = phi <10 x i8> [ zeroinitializer, %bb.10 ], [ %phi5, %bb.9 ]
+  %vec1 = shufflevector <10 x i8> %phi6, <10 x i8> zeroinitializer, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 15, i32 16, i32 17, i32 18, i32 19>
+  br label %bb.1
+}
+
+define amdgpu_kernel void @multiple_unwind(i1 %cmp, <10 x i8> %invec) {
+; GFX906-LABEL: define amdgpu_kernel void @multiple_unwind(
+; GFX906-SAME: i1 [[CMP:%.*]], <10 x i8> [[INVEC:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:  entry:
+; GFX906-NEXT:    br label [[BB_1:%.*]]
+; GFX906:       bb.1:
+; GFX906-NEXT:    [[PHI0:%.*]] = phi <10 x i8> [ <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, [[ENTRY:%.*]] ], [ [[PHI3:%.*]], [[BB_8:%.*]] ]
+; GFX906-NEXT:    br i1 [[CMP]], label [[BB_3:%.*]], label [[BB_2:%.*]]
+; GFX906:       bb.2:
+; GFX906-NEXT:    br label [[BB_3]]
+; GFX906:       bb.3:
+; GFX906-NEXT:    [[PHI1:%.*]] = phi <10 x i8> [ zeroinitializer, [[BB_2]] ], [ [[PHI0]], [[BB_1]] ]
+; GFX906-NEXT:    br i1 [[CMP]], label [[BB_5:%.*]], label [[BB_4:%.*]]
+; GFX906:       bb.4:
+; GFX906-NEXT:    br label [[BB_5]]
+; GFX906:       bb.5:
+; GFX906-NEXT:    [[PHI2:%.*]] = phi <10 x i8> [ [[PHI0]], [[BB_4]] ], [ [[PHI1]], [[BB_3]] ]
+; GFX906-NEXT:    br i1 [[CMP]], label [[BB_7:%.*]], label [[BB_6:%.*]]
+; GFX906:       bb.6:
+; GFX906-NEXT:    br label [[BB_7]]
+; GFX906:       bb.7:
+; GFX906-NEXT:    [[PHI3]] = phi <10 x i8> [ [[INVEC]], [[BB_6]] ], [ [[PHI2]], [[BB_5]] ]
+; GFX906-NEXT:    br label [[BB_8]]
+; GFX906:       bb.8:
+; GFX906-NEXT:    br label [[BB_1]]
+;
+entry:
+  br label %bb.1
+
+bb.1:
+  %phi0 = phi <10 x i8> [ <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, %entry ], [ %phi3, %bb.8 ]
+  br i1 %cmp, label %bb.3, label %bb.2
+
+bb.2:
+  br label %bb.3
+
+bb.3:
+  %phi1 = phi <10 x i8> [ zeroinitializer, %bb.2 ], [ %phi0, %bb.1 ]
+  br i1 %cmp, label %bb.5, label %bb.4
+
+bb.4:
+  br label %bb.5
+
+bb.5:
+  %phi2 = phi <10 x i8> [ %phi0, %bb.4 ], [ %phi1, %bb.3 ]
+  br i1 %cmp, label %bb.7, label %bb.6
+
+bb.6:                              ; preds = %bb.5
+  br label %bb.7
+
+bb.7:
+  %phi3 = phi <10 x i8> [ %invec, %bb.6 ], [ %phi2, %bb.5 ]
+  br label %bb.8
+
+bb.8:
+  br label %bb.1
+}
+
+
+
 declare i32 @llvm.amdgcn.workitem.id.x()

From 09fec4688234d899c82a5526874177c734dcc91b Mon Sep 17 00:00:00 2001
From: Ivan Kosarev <ivan.kosarev@amd.com>
Date: Fri, 19 Jul 2024 19:05:26 +0300
Subject: [PATCH 219/486] [AMDGPU][RFC] Combine asm and disasm tests. (#92895)

Eliminates the need to replicate the same instructions in MC and
MC/Disassembler tests and synchronize changes in them. Also highlights
differences between disassembled, reassembled and original instructions.
---
 llvm/test/MC/AMDGPU/gfx12_asm_vop1.s          | 2403 ++++++------
 llvm/test/MC/AMDGPU/lit.local.cfg             |    2 +
 .../Disassembler/AMDGPU/gfx12_dasm_vop1.txt   | 3338 -----------------
 3 files changed, 1207 insertions(+), 4536 deletions(-)
 delete mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1.txt

diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
index 7021bd4dcdcc9..55709fa365ff8 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
@@ -1,3590 +1,3597 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-ASM %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-DIS %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-ASM %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-DIS %s
 
 v_bfrev_b32_e32 v5, v1
-// GFX12: encoding: [0x01,0x71,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
 
 v_bfrev_b32 v5, v255
-// GFX12: encoding: [0xff,0x71,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, v255                ; encoding: [0xff,0x71,0x0a,0x7e]
 
 v_bfrev_b32 v5, s1
-// GFX12: encoding: [0x01,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, s1                  ; encoding: [0x01,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, s105
-// GFX12: encoding: [0x69,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, s105                ; encoding: [0x69,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, vcc_lo              ; encoding: [0x6a,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, vcc_hi              ; encoding: [0x6b,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, ttmp15              ; encoding: [0x7b,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, m0
-// GFX12: encoding: [0x7d,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, m0                  ; encoding: [0x7d,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, exec_lo             ; encoding: [0x7e,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, exec_hi             ; encoding: [0x7f,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, null
-// GFX12: encoding: [0x7c,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, null                ; encoding: [0x7c,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, -1
-// GFX12: encoding: [0xc1,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, -1                  ; encoding: [0xc1,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, 0.5
-// GFX12: encoding: [0xf0,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, 0.5                 ; encoding: [0xf0,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, src_scc
-// GFX12: encoding: [0xfd,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, src_scc             ; encoding: [0xfd,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x70,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_bfrev_b32_e32 v255, 0xaf123456        ; encoding: [0xff,0x70,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ceil_f16 v5, v1
-// GFX12: encoding: [0x01,0xb9,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, v1                   ; encoding: [0x01,0xb9,0x0a,0x7e]
 
 v_ceil_f16 v5, v127
-// GFX12: encoding: [0x7f,0xb9,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, v127                 ; encoding: [0x7f,0xb9,0x0a,0x7e]
 
 v_ceil_f16 v5, s1
-// GFX12: encoding: [0x01,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, s1                   ; encoding: [0x01,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, s105
-// GFX12: encoding: [0x69,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, s105                 ; encoding: [0x69,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, vcc_lo               ; encoding: [0x6a,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, vcc_hi               ; encoding: [0x6b,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, ttmp15               ; encoding: [0x7b,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, m0
-// GFX12: encoding: [0x7d,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, m0                   ; encoding: [0x7d,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, exec_lo              ; encoding: [0x7e,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, exec_hi              ; encoding: [0x7f,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, null
-// GFX12: encoding: [0x7c,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, null                 ; encoding: [0x7c,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, -1
-// GFX12: encoding: [0xc1,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, -1                   ; encoding: [0xc1,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, 0.5                  ; encoding: [0xf0,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, src_scc              ; encoding: [0xfd,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xb8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_ceil_f16_e32 v127, 0xfe0b             ; encoding: [0xff,0xb8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_ceil_f32 v5, v1
-// GFX12: encoding: [0x01,0x45,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, v1                   ; encoding: [0x01,0x45,0x0a,0x7e]
 
 v_ceil_f32 v5, v255
-// GFX12: encoding: [0xff,0x45,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, v255                 ; encoding: [0xff,0x45,0x0a,0x7e]
 
 v_ceil_f32 v5, s1
-// GFX12: encoding: [0x01,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, s1                   ; encoding: [0x01,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, s105
-// GFX12: encoding: [0x69,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, s105                 ; encoding: [0x69,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, vcc_lo               ; encoding: [0x6a,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, vcc_hi               ; encoding: [0x6b,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, ttmp15               ; encoding: [0x7b,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, m0
-// GFX12: encoding: [0x7d,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, m0                   ; encoding: [0x7d,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, exec_lo              ; encoding: [0x7e,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, exec_hi              ; encoding: [0x7f,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, null
-// GFX12: encoding: [0x7c,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, null                 ; encoding: [0x7c,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, -1
-// GFX12: encoding: [0xc1,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, -1                   ; encoding: [0xc1,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, 0.5                  ; encoding: [0xf0,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, src_scc              ; encoding: [0xfd,0x44,0x0a,0x7e]
 
 v_ceil_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x44,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ceil_f32_e32 v255, 0xaf123456         ; encoding: [0xff,0x44,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ceil_f64 v[5:6], v[1:2]
-// GFX12: encoding: [0x01,0x31,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], v[1:2]           ; encoding: [0x01,0x31,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], v[254:255]
-// GFX12: encoding: [0xfe,0x31,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], v[254:255]       ; encoding: [0xfe,0x31,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], s[2:3]
-// GFX12: encoding: [0x02,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], s[2:3]           ; encoding: [0x02,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], s[104:105]
-// GFX12: encoding: [0x68,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], s[104:105]       ; encoding: [0x68,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], vcc
-// GFX12: encoding: [0x6a,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], vcc              ; encoding: [0x6a,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], ttmp[14:15]      ; encoding: [0x7a,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], exec
-// GFX12: encoding: [0x7e,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], exec             ; encoding: [0x7e,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], null
-// GFX12: encoding: [0x7c,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], null             ; encoding: [0x7c,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], -1
-// GFX12: encoding: [0xc1,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], -1               ; encoding: [0xc1,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], 0.5              ; encoding: [0xf0,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], src_scc          ; encoding: [0xfd,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x30,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ceil_f64_e32 v[254:255], 0xaf123456   ; encoding: [0xff,0x30,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cls_i32 v5, v1
-// GFX12: encoding: [0x01,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v1                    ; encoding: [0x01,0x77,0x0a,0x7e]
 
 v_cls_i32 v5, v255
-// GFX12: encoding: [0xff,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v255                  ; encoding: [0xff,0x77,0x0a,0x7e]
 
 v_cls_i32 v5, s1
-// GFX12: encoding: [0x01,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s1                    ; encoding: [0x01,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, s105
-// GFX12: encoding: [0x69,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s105                  ; encoding: [0x69,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_lo                ; encoding: [0x6a,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_hi                ; encoding: [0x6b,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, ttmp15                ; encoding: [0x7b,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, m0
-// GFX12: encoding: [0x7d,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, m0                    ; encoding: [0x7d,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_lo               ; encoding: [0x7e,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_hi               ; encoding: [0x7f,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, null
-// GFX12: encoding: [0x7c,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, null                  ; encoding: [0x7c,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, -1
-// GFX12: encoding: [0xc1,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, -1                    ; encoding: [0xc1,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, 0.5
-// GFX12: encoding: [0xf0,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, 0.5                   ; encoding: [0xf0,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, src_scc
-// GFX12: encoding: [0xfd,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, src_scc               ; encoding: [0xfd,0x76,0x0a,0x7e]
 
 v_cls_i32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cls_i32_e32 v255, 0xaf123456          ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_clz_i32_u32 v5, v1
-// GFX12: encoding: [0x01,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v1                ; encoding: [0x01,0x73,0x0a,0x7e]
 
 v_clz_i32_u32 v5, v255
-// GFX12: encoding: [0xff,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v255              ; encoding: [0xff,0x73,0x0a,0x7e]
 
 v_clz_i32_u32 v5, s1
-// GFX12: encoding: [0x01,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s1                ; encoding: [0x01,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, s105
-// GFX12: encoding: [0x69,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s105              ; encoding: [0x69,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, m0
-// GFX12: encoding: [0x7d,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, m0                ; encoding: [0x7d,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, null
-// GFX12: encoding: [0x7c,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, null              ; encoding: [0x7c,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, -1
-// GFX12: encoding: [0xc1,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, -1                ; encoding: [0xc1,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, 0.5
-// GFX12: encoding: [0xf0,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, src_scc
-// GFX12: encoding: [0xfd,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_clz_i32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cos_f16 v5, v1
-// GFX12: encoding: [0x01,0xc3,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, v1                    ; encoding: [0x01,0xc3,0x0a,0x7e]
 
 v_cos_f16 v5, v127
-// GFX12: encoding: [0x7f,0xc3,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, v127                  ; encoding: [0x7f,0xc3,0x0a,0x7e]
 
 v_cos_f16 v5, s1
-// GFX12: encoding: [0x01,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, s1                    ; encoding: [0x01,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, s105
-// GFX12: encoding: [0x69,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, s105                  ; encoding: [0x69,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, m0
-// GFX12: encoding: [0x7d,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, m0                    ; encoding: [0x7d,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, null
-// GFX12: encoding: [0x7c,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, null                  ; encoding: [0x7c,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, -1
-// GFX12: encoding: [0xc1,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, -1                    ; encoding: [0xc1,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, src_scc               ; encoding: [0xfd,0xc2,0x0a,0x7e]
 
 v_cos_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xc2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cos_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xc2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cos_f32 v5, v1
-// GFX12: encoding: [0x01,0x6d,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, v1                    ; encoding: [0x01,0x6d,0x0a,0x7e]
 
 v_cos_f32 v5, v255
-// GFX12: encoding: [0xff,0x6d,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, v255                  ; encoding: [0xff,0x6d,0x0a,0x7e]
 
 v_cos_f32 v5, s1
-// GFX12: encoding: [0x01,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, s1                    ; encoding: [0x01,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, s105
-// GFX12: encoding: [0x69,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, s105                  ; encoding: [0x69,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, m0
-// GFX12: encoding: [0x7d,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, m0                    ; encoding: [0x7d,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, null
-// GFX12: encoding: [0x7c,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, null                  ; encoding: [0x7c,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, -1
-// GFX12: encoding: [0xc1,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, -1                    ; encoding: [0xc1,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, src_scc               ; encoding: [0xfd,0x6c,0x0a,0x7e]
 
 v_cos_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x6c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cos_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ctz_i32_b32 v5, v1
-// GFX12: encoding: [0x01,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v1                ; encoding: [0x01,0x75,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, v255
-// GFX12: encoding: [0xff,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v255              ; encoding: [0xff,0x75,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, s1
-// GFX12: encoding: [0x01,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s1                ; encoding: [0x01,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, s105
-// GFX12: encoding: [0x69,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s105              ; encoding: [0x69,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, m0
-// GFX12: encoding: [0x7d,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, m0                ; encoding: [0x7d,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, null
-// GFX12: encoding: [0x7c,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, null              ; encoding: [0x7c,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, -1
-// GFX12: encoding: [0xc1,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, -1                ; encoding: [0xc1,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, 0.5
-// GFX12: encoding: [0xf0,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, 0.5               ; encoding: [0xf0,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, src_scc
-// GFX12: encoding: [0xfd,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, src_scc           ; encoding: [0xfd,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_bf8_e32 v1, s3
-// GFX12: encoding: [0x03,0xda,0x02,0x7e]
+// GFX12: v_cvt_f32_bf8_e32 v1, s3                ; encoding: [0x03,0xda,0x02,0x7e]
 
 v_cvt_f32_bf8_e32 v1, 3
-// GFX12: encoding: [0x83,0xda,0x02,0x7e]
+// GFX12: v_cvt_f32_bf8_e32 v1, 3                 ; encoding: [0x83,0xda,0x02,0x7e]
 
 v_cvt_f32_bf8_e32 v1, v3
-// GFX12: encoding: [0x03,0xdb,0x02,0x7e]
+// GFX12: v_cvt_f32_bf8_e32 v1, v3                ; encoding: [0x03,0xdb,0x02,0x7e]
 
 v_cvt_f32_fp8_e32 v1, s3
-// GFX12: encoding: [0x03,0xd8,0x02,0x7e]
+// GFX12: v_cvt_f32_fp8_e32 v1, s3                ; encoding: [0x03,0xd8,0x02,0x7e]
 
 v_cvt_f32_fp8_e32 v1, 3
-// GFX12: encoding: [0x83,0xd8,0x02,0x7e]
+// GFX12: v_cvt_f32_fp8_e32 v1, 3                 ; encoding: [0x83,0xd8,0x02,0x7e]
 
 v_cvt_f32_fp8_e32 v1, v3
-// GFX12: encoding: [0x03,0xd9,0x02,0x7e]
+// GFX12: v_cvt_f32_fp8_e32 v1, v3                ; encoding: [0x03,0xd9,0x02,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[2:3], s3
-// GFX12: encoding: [0x03,0xde,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], s3         ; encoding: [0x03,0xde,0x04,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[3:4], s5
-// GFX12: encoding: [0x05,0xde,0x06,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], s5         ; encoding: [0x05,0xde,0x06,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[2:3], 3
-// GFX12: encoding: [0x83,0xde,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], 3          ; encoding: [0x83,0xde,0x04,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[3:4], 3
-// GFX12: encoding: [0x83,0xde,0x06,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], 3          ; encoding: [0x83,0xde,0x06,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[2:3], v3
-// GFX12: encoding: [0x03,0xdf,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], v3         ; encoding: [0x03,0xdf,0x04,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[3:4], v3
-// GFX12: encoding: [0x03,0xdf,0x06,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], v3         ; encoding: [0x03,0xdf,0x06,0x7e]
 
 v_cvt_pk_f32_fp8_e32 v[2:3], s3
-// GFX12: encoding: [0x03,0xdc,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], s3         ; encoding: [0x03,0xdc,0x04,0x7e]
 
 v_cvt_pk_f32_fp8_e32 v[2:3], 3
-// GFX12: encoding: [0x83,0xdc,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], 3          ; encoding: [0x83,0xdc,0x04,0x7e]
 
 v_cvt_pk_f32_fp8_e32 v[2:3], v3
-// GFX12: encoding: [0x03,0xdd,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], v3         ; encoding: [0x03,0xdd,0x04,0x7e]
 
 v_cvt_f16_f32 v5, v1
-// GFX12: encoding: [0x01,0x15,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, v1                ; encoding: [0x01,0x15,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, v255
-// GFX12: encoding: [0xff,0x15,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, v255              ; encoding: [0xff,0x15,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, s1
-// GFX12: encoding: [0x01,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, s1                ; encoding: [0x01,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, s105
-// GFX12: encoding: [0x69,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, s105              ; encoding: [0x69,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, m0
-// GFX12: encoding: [0x7d,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, m0                ; encoding: [0x7d,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, null
-// GFX12: encoding: [0x7c,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, null              ; encoding: [0x7c,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, -1
-// GFX12: encoding: [0xc1,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, -1                ; encoding: [0xc1,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, 0.5               ; encoding: [0xf0,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, src_scc           ; encoding: [0xfd,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v127, 0xaf123456
-// GFX12: encoding: [0xff,0x14,0xfe,0x7e,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f16_f32_e32 v127, 0xaf123456      ; encoding: [0xff,0x14,0xfe,0x7e,0x56,0x34,0x12,0xaf]
 
 v_cvt_f16_i16 v5, v1
-// GFX12: encoding: [0x01,0xa3,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, v1                ; encoding: [0x01,0xa3,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, v127
-// GFX12: encoding: [0x7f,0xa3,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, v127              ; encoding: [0x7f,0xa3,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, s1
-// GFX12: encoding: [0x01,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, s1                ; encoding: [0x01,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, s105
-// GFX12: encoding: [0x69,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, s105              ; encoding: [0x69,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, ttmp15            ; encoding: [0x7b,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, m0
-// GFX12: encoding: [0x7d,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, m0                ; encoding: [0x7d,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, exec_lo           ; encoding: [0x7e,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, exec_hi           ; encoding: [0x7f,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, null
-// GFX12: encoding: [0x7c,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, null              ; encoding: [0x7c,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, -1
-// GFX12: encoding: [0xc1,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, -1                ; encoding: [0xc1,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// GFX12: encoding: [0xf0,0xa2,0x0a,0x7e]
+// GFX12-ASM: v_cvt_f16_i16_e32 v5, 0.5               ; encoding: [0xf0,0xa2,0x0a,0x7e]
+// GFX12-DIS: v_cvt_f16_i16_e32 v5, 0x3800            ; encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_f16_i16 v5, src_scc
-// GFX12: encoding: [0xfd,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, src_scc           ; encoding: [0xfd,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xa2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_f16_i16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f16_u16 v5, v1
-// GFX12: encoding: [0x01,0xa1,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, v1                ; encoding: [0x01,0xa1,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, v127
-// GFX12: encoding: [0x7f,0xa1,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, v127              ; encoding: [0x7f,0xa1,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, s1
-// GFX12: encoding: [0x01,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, s1                ; encoding: [0x01,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, s105
-// GFX12: encoding: [0x69,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, s105              ; encoding: [0x69,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, ttmp15            ; encoding: [0x7b,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, m0
-// GFX12: encoding: [0x7d,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, m0                ; encoding: [0x7d,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, exec_lo           ; encoding: [0x7e,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, exec_hi           ; encoding: [0x7f,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, null
-// GFX12: encoding: [0x7c,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, null              ; encoding: [0x7c,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, -1
-// GFX12: encoding: [0xc1,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, -1                ; encoding: [0xc1,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// GFX12: encoding: [0xf0,0xa0,0x0a,0x7e]
+// GFX12-ASM: v_cvt_f16_u16_e32 v5, 0.5               ; encoding: [0xf0,0xa0,0x0a,0x7e]
+// GFX12-DIS: v_cvt_f16_u16_e32 v5, 0x3800            ; encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_f16_u16 v5, src_scc
-// GFX12: encoding: [0xfd,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, src_scc           ; encoding: [0xfd,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xa0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_f16_u16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f32_f16 v5, v1
-// GFX12: encoding: [0x01,0x17,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, v1                ; encoding: [0x01,0x17,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, v127
-// GFX12: encoding: [0x7f,0x17,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, v127              ; encoding: [0x7f,0x17,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, s1
-// GFX12: encoding: [0x01,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, s1                ; encoding: [0x01,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, s105
-// GFX12: encoding: [0x69,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, s105              ; encoding: [0x69,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, ttmp15            ; encoding: [0x7b,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, m0
-// GFX12: encoding: [0x7d,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, m0                ; encoding: [0x7d,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, exec_lo           ; encoding: [0x7e,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, exec_hi           ; encoding: [0x7f,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, null
-// GFX12: encoding: [0x7c,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, null              ; encoding: [0x7c,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, -1
-// GFX12: encoding: [0xc1,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, -1                ; encoding: [0xc1,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, 0.5               ; encoding: [0xf0,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, src_scc           ; encoding: [0xfd,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v255, 0xfe0b
-// GFX12: encoding: [0xff,0x16,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_f32_f16_e32 v255, 0xfe0b          ; encoding: [0xff,0x16,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f32_f64 v5, v[1:2]
-// GFX12: encoding: [0x01,0x1f,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x1f,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, v[254:255]
-// GFX12: encoding: [0xfe,0x1f,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x1f,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, s[2:3]
-// GFX12: encoding: [0x02,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, s[104:105]
-// GFX12: encoding: [0x68,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, vcc
-// GFX12: encoding: [0x6a,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, vcc               ; encoding: [0x6a,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, ttmp[14:15]
-// GFX12: encoding: [0x7a,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, exec
-// GFX12: encoding: [0x7e,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, exec              ; encoding: [0x7e,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, null
-// GFX12: encoding: [0x7c,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, null              ; encoding: [0x7c,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, -1
-// GFX12: encoding: [0xc1,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, -1                ; encoding: [0xc1,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, 0.5
-// GFX12: encoding: [0xf0,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, src_scc
-// GFX12: encoding: [0xfd,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x1e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x1e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_i32 v5, v1
-// GFX12: encoding: [0x01,0x0b,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, v1                ; encoding: [0x01,0x0b,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, v255
-// GFX12: encoding: [0xff,0x0b,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, v255              ; encoding: [0xff,0x0b,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, s1
-// GFX12: encoding: [0x01,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, s1                ; encoding: [0x01,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, s105
-// GFX12: encoding: [0x69,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, s105              ; encoding: [0x69,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, ttmp15            ; encoding: [0x7b,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, m0
-// GFX12: encoding: [0x7d,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, m0                ; encoding: [0x7d,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, exec_lo           ; encoding: [0x7e,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, exec_hi           ; encoding: [0x7f,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, null
-// GFX12: encoding: [0x7c,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, null              ; encoding: [0x7c,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, -1
-// GFX12: encoding: [0xc1,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, -1                ; encoding: [0xc1,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, 0.5
-// GFX12: encoding: [0xf0,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, 0.5               ; encoding: [0xf0,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, src_scc
-// GFX12: encoding: [0xfd,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, src_scc           ; encoding: [0xfd,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x0a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_i32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_u32 v5, v1
-// GFX12: encoding: [0x01,0x0d,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, v1                ; encoding: [0x01,0x0d,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, v255
-// GFX12: encoding: [0xff,0x0d,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, v255              ; encoding: [0xff,0x0d,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, s1
-// GFX12: encoding: [0x01,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, s1                ; encoding: [0x01,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, s105
-// GFX12: encoding: [0x69,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, s105              ; encoding: [0x69,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, m0
-// GFX12: encoding: [0x7d,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, m0                ; encoding: [0x7d,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, null
-// GFX12: encoding: [0x7c,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, null              ; encoding: [0x7c,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, -1
-// GFX12: encoding: [0xc1,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, -1                ; encoding: [0xc1,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, 0.5
-// GFX12: encoding: [0xf0,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, src_scc
-// GFX12: encoding: [0xfd,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x0c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte0 v5, v1
-// GFX12: encoding: [0x01,0x23,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, v1             ; encoding: [0x01,0x23,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, v255
-// GFX12: encoding: [0xff,0x23,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, v255           ; encoding: [0xff,0x23,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, s1
-// GFX12: encoding: [0x01,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, s1             ; encoding: [0x01,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, s105
-// GFX12: encoding: [0x69,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, s105           ; encoding: [0x69,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_lo         ; encoding: [0x6a,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_hi         ; encoding: [0x6b,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, ttmp15
-// GFX12: encoding: [0x7b,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, ttmp15         ; encoding: [0x7b,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, m0
-// GFX12: encoding: [0x7d,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, m0             ; encoding: [0x7d,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, exec_lo
-// GFX12: encoding: [0x7e,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, exec_lo        ; encoding: [0x7e,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, exec_hi
-// GFX12: encoding: [0x7f,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, exec_hi        ; encoding: [0x7f,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, null
-// GFX12: encoding: [0x7c,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, null           ; encoding: [0x7c,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, -1
-// GFX12: encoding: [0xc1,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, -1             ; encoding: [0xc1,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, 0.5
-// GFX12: encoding: [0xf0,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, 0.5            ; encoding: [0xf0,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, src_scc
-// GFX12: encoding: [0xfd,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, src_scc        ; encoding: [0xfd,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x22,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte0_e32 v255, 0xaf123456   ; encoding: [0xff,0x22,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte1 v5, v1
-// GFX12: encoding: [0x01,0x25,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, v1             ; encoding: [0x01,0x25,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, v255
-// GFX12: encoding: [0xff,0x25,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, v255           ; encoding: [0xff,0x25,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, s1
-// GFX12: encoding: [0x01,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, s1             ; encoding: [0x01,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, s105
-// GFX12: encoding: [0x69,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, s105           ; encoding: [0x69,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_lo         ; encoding: [0x6a,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_hi         ; encoding: [0x6b,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, ttmp15
-// GFX12: encoding: [0x7b,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, ttmp15         ; encoding: [0x7b,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, m0
-// GFX12: encoding: [0x7d,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, m0             ; encoding: [0x7d,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, exec_lo
-// GFX12: encoding: [0x7e,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, exec_lo        ; encoding: [0x7e,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, exec_hi
-// GFX12: encoding: [0x7f,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, exec_hi        ; encoding: [0x7f,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, null
-// GFX12: encoding: [0x7c,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, null           ; encoding: [0x7c,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, -1
-// GFX12: encoding: [0xc1,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, -1             ; encoding: [0xc1,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, 0.5
-// GFX12: encoding: [0xf0,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, 0.5            ; encoding: [0xf0,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, src_scc
-// GFX12: encoding: [0xfd,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, src_scc        ; encoding: [0xfd,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x24,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte1_e32 v255, 0xaf123456   ; encoding: [0xff,0x24,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte2 v5, v1
-// GFX12: encoding: [0x01,0x27,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, v1             ; encoding: [0x01,0x27,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, v255
-// GFX12: encoding: [0xff,0x27,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, v255           ; encoding: [0xff,0x27,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, s1
-// GFX12: encoding: [0x01,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, s1             ; encoding: [0x01,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, s105
-// GFX12: encoding: [0x69,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, s105           ; encoding: [0x69,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_lo         ; encoding: [0x6a,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_hi         ; encoding: [0x6b,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, ttmp15
-// GFX12: encoding: [0x7b,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, ttmp15         ; encoding: [0x7b,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, m0
-// GFX12: encoding: [0x7d,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, m0             ; encoding: [0x7d,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, exec_lo
-// GFX12: encoding: [0x7e,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, exec_lo        ; encoding: [0x7e,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, exec_hi
-// GFX12: encoding: [0x7f,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, exec_hi        ; encoding: [0x7f,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, null
-// GFX12: encoding: [0x7c,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, null           ; encoding: [0x7c,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, -1
-// GFX12: encoding: [0xc1,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, -1             ; encoding: [0xc1,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, 0.5
-// GFX12: encoding: [0xf0,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, 0.5            ; encoding: [0xf0,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, src_scc
-// GFX12: encoding: [0xfd,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, src_scc        ; encoding: [0xfd,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x26,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte2_e32 v255, 0xaf123456   ; encoding: [0xff,0x26,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte3 v5, v1
-// GFX12: encoding: [0x01,0x29,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, v1             ; encoding: [0x01,0x29,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, v255
-// GFX12: encoding: [0xff,0x29,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, v255           ; encoding: [0xff,0x29,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, s1
-// GFX12: encoding: [0x01,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, s1             ; encoding: [0x01,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, s105
-// GFX12: encoding: [0x69,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, s105           ; encoding: [0x69,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_lo         ; encoding: [0x6a,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_hi         ; encoding: [0x6b,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, ttmp15
-// GFX12: encoding: [0x7b,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, ttmp15         ; encoding: [0x7b,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, m0
-// GFX12: encoding: [0x7d,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, m0             ; encoding: [0x7d,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, exec_lo
-// GFX12: encoding: [0x7e,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, exec_lo        ; encoding: [0x7e,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, exec_hi
-// GFX12: encoding: [0x7f,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, exec_hi        ; encoding: [0x7f,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, null
-// GFX12: encoding: [0x7c,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, null           ; encoding: [0x7c,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, -1
-// GFX12: encoding: [0xc1,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, -1             ; encoding: [0xc1,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, 0.5
-// GFX12: encoding: [0xf0,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, 0.5            ; encoding: [0xf0,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, src_scc
-// GFX12: encoding: [0xfd,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, src_scc        ; encoding: [0xfd,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x28,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte3_e32 v255, 0xaf123456   ; encoding: [0xff,0x28,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f64_f32 v[5:6], v1
-// GFX12: encoding: [0x01,0x21,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], v1            ; encoding: [0x01,0x21,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], v255
-// GFX12: encoding: [0xff,0x21,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], v255          ; encoding: [0xff,0x21,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], s1
-// GFX12: encoding: [0x01,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], s1            ; encoding: [0x01,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], s105
-// GFX12: encoding: [0x69,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], s105          ; encoding: [0x69,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], vcc_lo
-// GFX12: encoding: [0x6a,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], vcc_hi
-// GFX12: encoding: [0x6b,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], ttmp15
-// GFX12: encoding: [0x7b,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], m0
-// GFX12: encoding: [0x7d,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], m0            ; encoding: [0x7d,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], exec_lo
-// GFX12: encoding: [0x7e,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], exec_hi
-// GFX12: encoding: [0x7f,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], null
-// GFX12: encoding: [0x7c,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], null          ; encoding: [0x7c,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], -1
-// GFX12: encoding: [0xc1,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], -1            ; encoding: [0xc1,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x20,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f64_f32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x20,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f64_i32 v[5:6], v1
-// GFX12: encoding: [0x01,0x09,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], v1            ; encoding: [0x01,0x09,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], v255
-// GFX12: encoding: [0xff,0x09,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], v255          ; encoding: [0xff,0x09,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], s1
-// GFX12: encoding: [0x01,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], s1            ; encoding: [0x01,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], s105
-// GFX12: encoding: [0x69,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], s105          ; encoding: [0x69,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], vcc_lo
-// GFX12: encoding: [0x6a,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], vcc_hi
-// GFX12: encoding: [0x6b,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], ttmp15
-// GFX12: encoding: [0x7b,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], m0
-// GFX12: encoding: [0x7d,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], m0            ; encoding: [0x7d,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], exec_lo
-// GFX12: encoding: [0x7e,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], exec_hi
-// GFX12: encoding: [0x7f,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], null
-// GFX12: encoding: [0x7c,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], null          ; encoding: [0x7c,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], -1
-// GFX12: encoding: [0xc1,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], -1            ; encoding: [0xc1,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x08,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f64_i32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x08,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f64_u32 v[5:6], v1
-// GFX12: encoding: [0x01,0x2d,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], v1            ; encoding: [0x01,0x2d,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], v255
-// GFX12: encoding: [0xff,0x2d,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], v255          ; encoding: [0xff,0x2d,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], s1
-// GFX12: encoding: [0x01,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], s1            ; encoding: [0x01,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], s105
-// GFX12: encoding: [0x69,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], s105          ; encoding: [0x69,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], vcc_lo
-// GFX12: encoding: [0x6a,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], vcc_hi
-// GFX12: encoding: [0x6b,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], ttmp15
-// GFX12: encoding: [0x7b,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], m0
-// GFX12: encoding: [0x7d,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], m0            ; encoding: [0x7d,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], exec_lo
-// GFX12: encoding: [0x7e,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], exec_hi
-// GFX12: encoding: [0x7f,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], null
-// GFX12: encoding: [0x7c,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], null          ; encoding: [0x7c,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], -1
-// GFX12: encoding: [0xc1,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], -1            ; encoding: [0xc1,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x2c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f64_u32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x2c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_floor_i32_f32 v5, v1
-// GFX12: encoding: [0x01,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v1          ; encoding: [0x01,0x1b,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, v255
-// GFX12: encoding: [0xff,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v255        ; encoding: [0xff,0x1b,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, s1
-// GFX12: encoding: [0x01,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s1          ; encoding: [0x01,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, s105
-// GFX12: encoding: [0x69,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s105        ; encoding: [0x69,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, m0
-// GFX12: encoding: [0x7d,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, null
-// GFX12: encoding: [0x7c,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, null        ; encoding: [0x7c,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, -1
-// GFX12: encoding: [0xc1,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x1a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_floor_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x1a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_flr_i32_f32 v5, v1
-// GFX12: encoding: [0x01,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v1          ; encoding: [0x01,0x1b,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, v255
-// GFX12: encoding: [0xff,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v255        ; encoding: [0xff,0x1b,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, s1
-// GFX12: encoding: [0x01,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s1          ; encoding: [0x01,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, s105
-// GFX12: encoding: [0x69,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s105        ; encoding: [0x69,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, m0
-// GFX12: encoding: [0x7d,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, null
-// GFX12: encoding: [0x7c,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, null        ; encoding: [0x7c,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, -1
-// GFX12: encoding: [0xc1,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x1a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_floor_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x1a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_i16_f16 v5, v1
-// GFX12: encoding: [0x01,0xa7,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, v1                ; encoding: [0x01,0xa7,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, v127
-// GFX12: encoding: [0x7f,0xa7,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, v127              ; encoding: [0x7f,0xa7,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, s1
-// GFX12: encoding: [0x01,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, s1                ; encoding: [0x01,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, s105
-// GFX12: encoding: [0x69,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, s105              ; encoding: [0x69,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, ttmp15            ; encoding: [0x7b,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, m0
-// GFX12: encoding: [0x7d,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, m0                ; encoding: [0x7d,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, exec_lo           ; encoding: [0x7e,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, exec_hi           ; encoding: [0x7f,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, null
-// GFX12: encoding: [0x7c,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, null              ; encoding: [0x7c,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, -1
-// GFX12: encoding: [0xc1,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, -1                ; encoding: [0xc1,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, 0.5               ; encoding: [0xf0,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, src_scc           ; encoding: [0xfd,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xa6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_i16_f16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_i32_f32 v5, v1
-// GFX12: encoding: [0x01,0x11,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, v1                ; encoding: [0x01,0x11,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, v255
-// GFX12: encoding: [0xff,0x11,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, v255              ; encoding: [0xff,0x11,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, s1
-// GFX12: encoding: [0x01,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, s1                ; encoding: [0x01,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, s105
-// GFX12: encoding: [0x69,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, s105              ; encoding: [0x69,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, m0
-// GFX12: encoding: [0x7d,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, m0                ; encoding: [0x7d,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, null
-// GFX12: encoding: [0x7c,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, null              ; encoding: [0x7c,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, -1
-// GFX12: encoding: [0xc1,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, -1                ; encoding: [0xc1,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, 0.5               ; encoding: [0xf0,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, src_scc           ; encoding: [0xfd,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x10,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_i32_f32_e32 v255, 0xaf123456      ; encoding: [0xff,0x10,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_i32_f64 v5, v[1:2]
-// GFX12: encoding: [0x01,0x07,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x07,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, v[254:255]
-// GFX12: encoding: [0xfe,0x07,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x07,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, s[2:3]
-// GFX12: encoding: [0x02,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, s[104:105]
-// GFX12: encoding: [0x68,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, vcc
-// GFX12: encoding: [0x6a,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, vcc               ; encoding: [0x6a,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, ttmp[14:15]
-// GFX12: encoding: [0x7a,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, exec
-// GFX12: encoding: [0x7e,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, exec              ; encoding: [0x7e,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, null
-// GFX12: encoding: [0x7c,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, null              ; encoding: [0x7c,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, -1
-// GFX12: encoding: [0xc1,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, -1                ; encoding: [0xc1,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, 0.5
-// GFX12: encoding: [0xf0,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, src_scc
-// GFX12: encoding: [0xfd,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x06,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_i32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x06,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_i32_i16 v5, v1
-// GFX12: encoding: [0x01,0xd5,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, v1                ; encoding: [0x01,0xd5,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, v127
-// GFX12: encoding: [0x7f,0xd5,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, v127              ; encoding: [0x7f,0xd5,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, s1
-// GFX12: encoding: [0x01,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, s1                ; encoding: [0x01,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, s105
-// GFX12: encoding: [0x69,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, s105              ; encoding: [0x69,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, vcc_lo            ; encoding: [0x6a,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, vcc_hi            ; encoding: [0x6b,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, ttmp15            ; encoding: [0x7b,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, m0
-// GFX12: encoding: [0x7d,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, m0                ; encoding: [0x7d,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, exec_lo           ; encoding: [0x7e,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, exec_hi           ; encoding: [0x7f,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, null
-// GFX12: encoding: [0x7c,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, null              ; encoding: [0x7c,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, -1
-// GFX12: encoding: [0xc1,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, -1                ; encoding: [0xc1,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, 0.5
-// GFX12: encoding: [0xf0,0xd4,0x0a,0x7e]
+// GFX12-ASM: v_cvt_i32_i16_e32 v5, 0.5               ; encoding: [0xf0,0xd4,0x0a,0x7e]
+// GFX12-DIS: v_cvt_i32_i16_e32 v5, 0x3800            ; encoding: [0xff,0xd4,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_i32_i16 v5, src_scc
-// GFX12: encoding: [0xfd,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, src_scc           ; encoding: [0xfd,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v255, 0xfe0b
-// GFX12: encoding: [0xff,0xd4,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_i32_i16_e32 v255, 0xfe0b          ; encoding: [0xff,0xd4,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
 
 v_cvt_nearest_i32_f32 v5, v1
-// GFX12: encoding: [0x01,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v1        ; encoding: [0x01,0x19,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, v255
-// GFX12: encoding: [0xff,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v255      ; encoding: [0xff,0x19,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, s1
-// GFX12: encoding: [0x01,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s1        ; encoding: [0x01,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, s105
-// GFX12: encoding: [0x69,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s105      ; encoding: [0x69,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo    ; encoding: [0x6a,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi    ; encoding: [0x6b,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15    ; encoding: [0x7b,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, m0
-// GFX12: encoding: [0x7d,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, m0        ; encoding: [0x7d,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo   ; encoding: [0x7e,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi   ; encoding: [0x7f,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, null
-// GFX12: encoding: [0x7c,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, null      ; encoding: [0x7c,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, -1
-// GFX12: encoding: [0xc1,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, -1        ; encoding: [0xc1,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5       ; encoding: [0xf0,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc   ; encoding: [0xfd,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x18,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_nearest_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x18,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_norm_i16_f16 v5, v1
-// GFX12: encoding: [0x01,0xc7,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, v1           ; encoding: [0x01,0xc7,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, v127
-// GFX12: encoding: [0x7f,0xc7,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, v127         ; encoding: [0x7f,0xc7,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, s1
-// GFX12: encoding: [0x01,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, s1           ; encoding: [0x01,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, s105
-// GFX12: encoding: [0x69,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, s105         ; encoding: [0x69,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_lo       ; encoding: [0x6a,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_hi       ; encoding: [0x6b,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, ttmp15       ; encoding: [0x7b,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, m0
-// GFX12: encoding: [0x7d,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, m0           ; encoding: [0x7d,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, exec_lo      ; encoding: [0x7e,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, exec_hi      ; encoding: [0x7f,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, null
-// GFX12: encoding: [0x7c,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, null         ; encoding: [0x7c,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, -1
-// GFX12: encoding: [0xc1,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, -1           ; encoding: [0xc1,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, 0.5          ; encoding: [0xf0,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, src_scc      ; encoding: [0xfd,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xc6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_norm_i16_f16_e32 v127, 0xfe0b     ; encoding: [0xff,0xc6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_norm_u16_f16 v5, v1
-// GFX12: encoding: [0x01,0xc9,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, v1           ; encoding: [0x01,0xc9,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, v127
-// GFX12: encoding: [0x7f,0xc9,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, v127         ; encoding: [0x7f,0xc9,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, s1
-// GFX12: encoding: [0x01,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, s1           ; encoding: [0x01,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, s105
-// GFX12: encoding: [0x69,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, s105         ; encoding: [0x69,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_lo       ; encoding: [0x6a,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_hi       ; encoding: [0x6b,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, ttmp15       ; encoding: [0x7b,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, m0
-// GFX12: encoding: [0x7d,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, m0           ; encoding: [0x7d,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, exec_lo      ; encoding: [0x7e,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, exec_hi      ; encoding: [0x7f,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, null
-// GFX12: encoding: [0x7c,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, null         ; encoding: [0x7c,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, -1
-// GFX12: encoding: [0xc1,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, -1           ; encoding: [0xc1,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, 0.5          ; encoding: [0xf0,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, src_scc      ; encoding: [0xfd,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xc8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_norm_u16_f16_e32 v127, 0xfe0b     ; encoding: [0xff,0xc8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_off_f32_i4 v5, v1
-// GFX12: encoding: [0x01,0x1d,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, v1             ; encoding: [0x01,0x1d,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, v255
-// GFX12: encoding: [0xff,0x1d,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, v255           ; encoding: [0xff,0x1d,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, s1
-// GFX12: encoding: [0x01,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, s1             ; encoding: [0x01,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, s105
-// GFX12: encoding: [0x69,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, s105           ; encoding: [0x69,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, vcc_lo         ; encoding: [0x6a,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, vcc_hi         ; encoding: [0x6b,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, ttmp15
-// GFX12: encoding: [0x7b,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, ttmp15         ; encoding: [0x7b,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, m0
-// GFX12: encoding: [0x7d,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, m0             ; encoding: [0x7d,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, exec_lo
-// GFX12: encoding: [0x7e,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, exec_lo        ; encoding: [0x7e,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, exec_hi
-// GFX12: encoding: [0x7f,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, exec_hi        ; encoding: [0x7f,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, null
-// GFX12: encoding: [0x7c,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, null           ; encoding: [0x7c,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, -1
-// GFX12: encoding: [0xc1,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, -1             ; encoding: [0xc1,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, 0.5
-// GFX12: encoding: [0xf0,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, 0.5            ; encoding: [0xf0,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, src_scc
-// GFX12: encoding: [0xfd,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, src_scc        ; encoding: [0xfd,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v255, 0x4f
-// GFX12: encoding: [0xff,0x1c,0xfe,0x7f,0x4f,0x00,0x00,0x00]
+// GFX12: v_cvt_off_f32_i4_e32 v255, 0x4f         ; encoding: [0xff,0x1c,0xfe,0x7f,0x4f,0x00,0x00,0x00]
 
 v_cvt_rpi_i32_f32 v5, v1
-// GFX12: encoding: [0x01,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v1        ; encoding: [0x01,0x19,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, v255
-// GFX12: encoding: [0xff,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v255      ; encoding: [0xff,0x19,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, s1
-// GFX12: encoding: [0x01,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s1        ; encoding: [0x01,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, s105
-// GFX12: encoding: [0x69,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s105      ; encoding: [0x69,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo    ; encoding: [0x6a,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi    ; encoding: [0x6b,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15    ; encoding: [0x7b,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, m0
-// GFX12: encoding: [0x7d,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, m0        ; encoding: [0x7d,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo   ; encoding: [0x7e,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi   ; encoding: [0x7f,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, null
-// GFX12: encoding: [0x7c,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, null      ; encoding: [0x7c,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, -1
-// GFX12: encoding: [0xc1,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, -1        ; encoding: [0xc1,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5       ; encoding: [0xf0,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc   ; encoding: [0xfd,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x18,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_nearest_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x18,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_u16_f16 v5, v1
-// GFX12: encoding: [0x01,0xa5,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, v1                ; encoding: [0x01,0xa5,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, v127
-// GFX12: encoding: [0x7f,0xa5,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, v127              ; encoding: [0x7f,0xa5,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, s1
-// GFX12: encoding: [0x01,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, s1                ; encoding: [0x01,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, s105
-// GFX12: encoding: [0x69,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, s105              ; encoding: [0x69,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, ttmp15            ; encoding: [0x7b,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, m0
-// GFX12: encoding: [0x7d,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, m0                ; encoding: [0x7d,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, exec_lo           ; encoding: [0x7e,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, exec_hi           ; encoding: [0x7f,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, null
-// GFX12: encoding: [0x7c,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, null              ; encoding: [0x7c,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, -1
-// GFX12: encoding: [0xc1,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, -1                ; encoding: [0xc1,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, 0.5               ; encoding: [0xf0,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, src_scc           ; encoding: [0xfd,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xa4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_u16_f16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_u32_f32 v5, v1
-// GFX12: encoding: [0x01,0x0f,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, v1                ; encoding: [0x01,0x0f,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, v255
-// GFX12: encoding: [0xff,0x0f,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, v255              ; encoding: [0xff,0x0f,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, s1
-// GFX12: encoding: [0x01,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, s1                ; encoding: [0x01,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, s105
-// GFX12: encoding: [0x69,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, s105              ; encoding: [0x69,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, m0
-// GFX12: encoding: [0x7d,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, m0                ; encoding: [0x7d,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, null
-// GFX12: encoding: [0x7c,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, null              ; encoding: [0x7c,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, -1
-// GFX12: encoding: [0xc1,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, -1                ; encoding: [0xc1,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, 0.5               ; encoding: [0xf0,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, src_scc           ; encoding: [0xfd,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x0e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_u32_f32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_u32_f64 v5, v[1:2]
-// GFX12: encoding: [0x01,0x2b,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x2b,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, v[254:255]
-// GFX12: encoding: [0xfe,0x2b,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x2b,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, s[2:3]
-// GFX12: encoding: [0x02,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, s[104:105]
-// GFX12: encoding: [0x68,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, vcc
-// GFX12: encoding: [0x6a,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, vcc               ; encoding: [0x6a,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, ttmp[14:15]
-// GFX12: encoding: [0x7a,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, exec
-// GFX12: encoding: [0x7e,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, exec              ; encoding: [0x7e,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, null
-// GFX12: encoding: [0x7c,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, null              ; encoding: [0x7c,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, -1
-// GFX12: encoding: [0xc1,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, -1                ; encoding: [0xc1,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, 0.5
-// GFX12: encoding: [0xf0,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, src_scc
-// GFX12: encoding: [0xfd,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x2a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_u32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x2a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_u32_u16 v5, v1
-// GFX12: encoding: [0x01,0xd7,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, v1                ; encoding: [0x01,0xd7,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, v127
-// GFX12: encoding: [0x7f,0xd7,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, v127              ; encoding: [0x7f,0xd7,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, s1
-// GFX12: encoding: [0x01,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, s1                ; encoding: [0x01,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, s105
-// GFX12: encoding: [0x69,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, s105              ; encoding: [0x69,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, vcc_lo            ; encoding: [0x6a,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, vcc_hi            ; encoding: [0x6b,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, ttmp15            ; encoding: [0x7b,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, m0
-// GFX12: encoding: [0x7d,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, m0                ; encoding: [0x7d,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, exec_lo           ; encoding: [0x7e,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, exec_hi           ; encoding: [0x7f,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, null
-// GFX12: encoding: [0x7c,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, null              ; encoding: [0x7c,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, -1
-// GFX12: encoding: [0xc1,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, -1                ; encoding: [0xc1,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, 0.5
-// GFX12: encoding: [0xf0,0xd6,0x0a,0x7e]
+// GFX12-ASM: v_cvt_u32_u16_e32 v5, 0.5               ; encoding: [0xf0,0xd6,0x0a,0x7e]
+// GFX12-DIS: v_cvt_u32_u16_e32 v5, 0x3800            ; encoding: [0xff,0xd6,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_u32_u16 v5, src_scc
-// GFX12: encoding: [0xfd,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, src_scc           ; encoding: [0xfd,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v255, 0xfe0b
-// GFX12: encoding: [0xff,0xd6,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_u32_u16_e32 v255, 0xfe0b          ; encoding: [0xff,0xd6,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
 
 v_exp_f16 v5, v1
-// GFX12: encoding: [0x01,0xb1,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, v1                    ; encoding: [0x01,0xb1,0x0a,0x7e]
 
 v_exp_f16 v5, v127
-// GFX12: encoding: [0x7f,0xb1,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, v127                  ; encoding: [0x7f,0xb1,0x0a,0x7e]
 
 v_exp_f16 v5, s1
-// GFX12: encoding: [0x01,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, s1                    ; encoding: [0x01,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, s105
-// GFX12: encoding: [0x69,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, s105                  ; encoding: [0x69,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, m0
-// GFX12: encoding: [0x7d,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, m0                    ; encoding: [0x7d,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, null
-// GFX12: encoding: [0x7c,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, null                  ; encoding: [0x7c,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, -1
-// GFX12: encoding: [0xc1,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, -1                    ; encoding: [0xc1,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, src_scc               ; encoding: [0xfd,0xb0,0x0a,0x7e]
 
 v_exp_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xb0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_exp_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xb0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_exp_f32 v5, v1
-// GFX12: encoding: [0x01,0x4b,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, v1                    ; encoding: [0x01,0x4b,0x0a,0x7e]
 
 v_exp_f32 v5, v255
-// GFX12: encoding: [0xff,0x4b,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, v255                  ; encoding: [0xff,0x4b,0x0a,0x7e]
 
 v_exp_f32 v5, s1
-// GFX12: encoding: [0x01,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, s1                    ; encoding: [0x01,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, s105
-// GFX12: encoding: [0x69,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, s105                  ; encoding: [0x69,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, m0
-// GFX12: encoding: [0x7d,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, m0                    ; encoding: [0x7d,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, null
-// GFX12: encoding: [0x7c,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, null                  ; encoding: [0x7c,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, -1
-// GFX12: encoding: [0xc1,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, -1                    ; encoding: [0xc1,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, src_scc               ; encoding: [0xfd,0x4a,0x0a,0x7e]
 
 v_exp_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x4a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_exp_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x4a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ffbh_i32 v5, v1
-// GFX12: encoding: [0x01,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v1                    ; encoding: [0x01,0x77,0x0a,0x7e]
 
 v_ffbh_i32 v5, v255
-// GFX12: encoding: [0xff,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v255                  ; encoding: [0xff,0x77,0x0a,0x7e]
 
 v_ffbh_i32 v5, s1
-// GFX12: encoding: [0x01,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s1                    ; encoding: [0x01,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, s105
-// GFX12: encoding: [0x69,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s105                  ; encoding: [0x69,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_lo                ; encoding: [0x6a,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_hi                ; encoding: [0x6b,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, ttmp15                ; encoding: [0x7b,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, m0
-// GFX12: encoding: [0x7d,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, m0                    ; encoding: [0x7d,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_lo               ; encoding: [0x7e,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_hi               ; encoding: [0x7f,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, null
-// GFX12: encoding: [0x7c,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, null                  ; encoding: [0x7c,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, -1
-// GFX12: encoding: [0xc1,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, -1                    ; encoding: [0xc1,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, 0.5
-// GFX12: encoding: [0xf0,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, 0.5                   ; encoding: [0xf0,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, src_scc
-// GFX12: encoding: [0xfd,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, src_scc               ; encoding: [0xfd,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cls_i32_e32 v255, 0xaf123456          ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ffbh_u32 v5, v1
-// GFX12: encoding: [0x01,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v1                ; encoding: [0x01,0x73,0x0a,0x7e]
 
 v_ffbh_u32 v5, v255
-// GFX12: encoding: [0xff,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v255              ; encoding: [0xff,0x73,0x0a,0x7e]
 
 v_ffbh_u32 v5, s1
-// GFX12: encoding: [0x01,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s1                ; encoding: [0x01,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, s105
-// GFX12: encoding: [0x69,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s105              ; encoding: [0x69,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, m0
-// GFX12: encoding: [0x7d,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, m0                ; encoding: [0x7d,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, null
-// GFX12: encoding: [0x7c,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, null              ; encoding: [0x7c,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, -1
-// GFX12: encoding: [0xc1,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, -1                ; encoding: [0xc1,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, 0.5
-// GFX12: encoding: [0xf0,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, src_scc
-// GFX12: encoding: [0xfd,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_clz_i32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ffbl_b32 v5, v1
-// GFX12: encoding: [0x01,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v1                ; encoding: [0x01,0x75,0x0a,0x7e]
 
 v_ffbl_b32 v5, v255
-// GFX12: encoding: [0xff,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v255              ; encoding: [0xff,0x75,0x0a,0x7e]
 
 v_ffbl_b32 v5, s1
-// GFX12: encoding: [0x01,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s1                ; encoding: [0x01,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, s105
-// GFX12: encoding: [0x69,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s105              ; encoding: [0x69,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, m0
-// GFX12: encoding: [0x7d,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, m0                ; encoding: [0x7d,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, null
-// GFX12: encoding: [0x7c,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, null              ; encoding: [0x7c,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, -1
-// GFX12: encoding: [0xc1,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, -1                ; encoding: [0xc1,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, 0.5
-// GFX12: encoding: [0xf0,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, 0.5               ; encoding: [0xf0,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, src_scc
-// GFX12: encoding: [0xfd,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, src_scc           ; encoding: [0xfd,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_floor_f16 v5, v1
-// GFX12: encoding: [0x01,0xb7,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, v1                  ; encoding: [0x01,0xb7,0x0a,0x7e]
 
 v_floor_f16 v5, v127
-// GFX12: encoding: [0x7f,0xb7,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, v127                ; encoding: [0x7f,0xb7,0x0a,0x7e]
 
 v_floor_f16 v5, s1
-// GFX12: encoding: [0x01,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, s1                  ; encoding: [0x01,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, s105
-// GFX12: encoding: [0x69,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, s105                ; encoding: [0x69,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, m0
-// GFX12: encoding: [0x7d,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, m0                  ; encoding: [0x7d,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, null
-// GFX12: encoding: [0x7c,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, null                ; encoding: [0x7c,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, -1
-// GFX12: encoding: [0xc1,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, -1                  ; encoding: [0xc1,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, src_scc             ; encoding: [0xfd,0xb6,0x0a,0x7e]
 
 v_floor_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xb6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_floor_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xb6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_floor_f32 v5, v1
-// GFX12: encoding: [0x01,0x49,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, v1                  ; encoding: [0x01,0x49,0x0a,0x7e]
 
 v_floor_f32 v5, v255
-// GFX12: encoding: [0xff,0x49,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, v255                ; encoding: [0xff,0x49,0x0a,0x7e]
 
 v_floor_f32 v5, s1
-// GFX12: encoding: [0x01,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, s1                  ; encoding: [0x01,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, s105
-// GFX12: encoding: [0x69,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, s105                ; encoding: [0x69,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, m0
-// GFX12: encoding: [0x7d,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, m0                  ; encoding: [0x7d,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, null
-// GFX12: encoding: [0x7c,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, null                ; encoding: [0x7c,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, -1
-// GFX12: encoding: [0xc1,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, -1                  ; encoding: [0xc1,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, src_scc             ; encoding: [0xfd,0x48,0x0a,0x7e]
 
 v_floor_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x48,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_floor_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x48,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_floor_f64 v[5:6], v[1:2]
-// GFX12: encoding: [0x01,0x35,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x35,0x0a,0x7e]
 
 v_floor_f64 v[5:6], v[254:255]
-// GFX12: encoding: [0xfe,0x35,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x35,0x0a,0x7e]
 
 v_floor_f64 v[5:6], s[2:3]
-// GFX12: encoding: [0x02,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], s[104:105]
-// GFX12: encoding: [0x68,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], vcc
-// GFX12: encoding: [0x6a,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], exec
-// GFX12: encoding: [0x7e,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], null
-// GFX12: encoding: [0x7c,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], null            ; encoding: [0x7c,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], -1
-// GFX12: encoding: [0xc1,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x34,0x0a,0x7e]
 
 v_floor_f64 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x34,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_floor_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x34,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_fract_f16 v5, v1
-// GFX12: encoding: [0x01,0xbf,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, v1                  ; encoding: [0x01,0xbf,0x0a,0x7e]
 
 v_fract_f16 v5, v127
-// GFX12: encoding: [0x7f,0xbf,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, v127                ; encoding: [0x7f,0xbf,0x0a,0x7e]
 
 v_fract_f16 v5, s1
-// GFX12: encoding: [0x01,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, s1                  ; encoding: [0x01,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, s105
-// GFX12: encoding: [0x69,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, s105                ; encoding: [0x69,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, m0
-// GFX12: encoding: [0x7d,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, m0                  ; encoding: [0x7d,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, null
-// GFX12: encoding: [0x7c,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, null                ; encoding: [0x7c,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, -1
-// GFX12: encoding: [0xc1,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, -1                  ; encoding: [0xc1,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, src_scc             ; encoding: [0xfd,0xbe,0x0a,0x7e]
 
 v_fract_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xbe,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_fract_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xbe,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_fract_f32 v5, v1
-// GFX12: encoding: [0x01,0x41,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, v1                  ; encoding: [0x01,0x41,0x0a,0x7e]
 
 v_fract_f32 v5, v255
-// GFX12: encoding: [0xff,0x41,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, v255                ; encoding: [0xff,0x41,0x0a,0x7e]
 
 v_fract_f32 v5, s1
-// GFX12: encoding: [0x01,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, s1                  ; encoding: [0x01,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, s105
-// GFX12: encoding: [0x69,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, s105                ; encoding: [0x69,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, m0
-// GFX12: encoding: [0x7d,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, m0                  ; encoding: [0x7d,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, null
-// GFX12: encoding: [0x7c,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, null                ; encoding: [0x7c,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, -1
-// GFX12: encoding: [0xc1,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, -1                  ; encoding: [0xc1,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, src_scc             ; encoding: [0xfd,0x40,0x0a,0x7e]
 
 v_fract_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x40,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_fract_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x40,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_fract_f64 v[5:6], v[1:2]
-// GFX12: encoding: [0x01,0x7d,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x7d,0x0a,0x7e]
 
 v_fract_f64 v[5:6], v[254:255]
-// GFX12: encoding: [0xfe,0x7d,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x7d,0x0a,0x7e]
 
 v_fract_f64 v[5:6], s[2:3]
-// GFX12: encoding: [0x02,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], s[104:105]
-// GFX12: encoding: [0x68,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], vcc
-// GFX12: encoding: [0x6a,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], exec
-// GFX12: encoding: [0x7e,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], null
-// GFX12: encoding: [0x7c,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], null            ; encoding: [0x7c,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], -1
-// GFX12: encoding: [0xc1,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x7c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_fract_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x7c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_exp_i16_f16 v5, v1
-// GFX12: encoding: [0x01,0xb5,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, v1          ; encoding: [0x01,0xb5,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, v127
-// GFX12: encoding: [0x7f,0xb5,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, v127        ; encoding: [0x7f,0xb5,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, s1
-// GFX12: encoding: [0x01,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, s1          ; encoding: [0x01,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, s105
-// GFX12: encoding: [0x69,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, s105        ; encoding: [0x69,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_lo      ; encoding: [0x6a,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_hi      ; encoding: [0x6b,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, ttmp15      ; encoding: [0x7b,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, m0
-// GFX12: encoding: [0x7d,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, m0          ; encoding: [0x7d,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, exec_lo     ; encoding: [0x7e,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, exec_hi     ; encoding: [0x7f,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, null
-// GFX12: encoding: [0x7c,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, null        ; encoding: [0x7c,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, -1
-// GFX12: encoding: [0xc1,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, -1          ; encoding: [0xc1,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, 0.5         ; encoding: [0xf0,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, src_scc     ; encoding: [0xfd,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xb4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_frexp_exp_i16_f16_e32 v127, 0xfe0b    ; encoding: [0xff,0xb4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_frexp_exp_i32_f32 v5, v1
-// GFX12: encoding: [0x01,0x7f,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, v1          ; encoding: [0x01,0x7f,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, v255
-// GFX12: encoding: [0xff,0x7f,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, v255        ; encoding: [0xff,0x7f,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, s1
-// GFX12: encoding: [0x01,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, s1          ; encoding: [0x01,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, s105
-// GFX12: encoding: [0x69,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, s105        ; encoding: [0x69,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, m0
-// GFX12: encoding: [0x7d,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, null
-// GFX12: encoding: [0x7c,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, null        ; encoding: [0x7c,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, -1
-// GFX12: encoding: [0xc1,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x7e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_frexp_exp_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x7e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_exp_i32_f64 v5, v[1:2]
-// GFX12: encoding: [0x01,0x79,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, v[1:2]      ; encoding: [0x01,0x79,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, v[254:255]
-// GFX12: encoding: [0xfe,0x79,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, v[254:255]  ; encoding: [0xfe,0x79,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, s[2:3]
-// GFX12: encoding: [0x02,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, s[2:3]      ; encoding: [0x02,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, s[104:105]
-// GFX12: encoding: [0x68,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, s[104:105]  ; encoding: [0x68,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, vcc
-// GFX12: encoding: [0x6a,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, vcc         ; encoding: [0x6a,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, ttmp[14:15]
-// GFX12: encoding: [0x7a,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, ttmp[14:15] ; encoding: [0x7a,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, exec
-// GFX12: encoding: [0x7e,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, exec        ; encoding: [0x7e,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, null
-// GFX12: encoding: [0x7c,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, null        ; encoding: [0x7c,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, -1
-// GFX12: encoding: [0xc1,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, -1          ; encoding: [0xc1,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, 0.5
-// GFX12: encoding: [0xf0,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, 0.5         ; encoding: [0xf0,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, src_scc
-// GFX12: encoding: [0xfd,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, src_scc     ; encoding: [0xfd,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x78,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_frexp_exp_i32_f64_e32 v255, 0xaf123456 ; encoding: [0xff,0x78,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_mant_f16 v5, v1
-// GFX12: encoding: [0x01,0xb3,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, v1             ; encoding: [0x01,0xb3,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, v127
-// GFX12: encoding: [0x7f,0xb3,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, v127           ; encoding: [0x7f,0xb3,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, s1
-// GFX12: encoding: [0x01,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, s1             ; encoding: [0x01,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, s105
-// GFX12: encoding: [0x69,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, s105           ; encoding: [0x69,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, vcc_lo         ; encoding: [0x6a,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, vcc_hi         ; encoding: [0x6b,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, ttmp15         ; encoding: [0x7b,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, m0
-// GFX12: encoding: [0x7d,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, m0             ; encoding: [0x7d,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, exec_lo        ; encoding: [0x7e,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, exec_hi        ; encoding: [0x7f,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, null
-// GFX12: encoding: [0x7c,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, null           ; encoding: [0x7c,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, -1
-// GFX12: encoding: [0xc1,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, -1             ; encoding: [0xc1,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, 0.5            ; encoding: [0xf0,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, src_scc        ; encoding: [0xfd,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xb2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_frexp_mant_f16_e32 v127, 0xfe0b       ; encoding: [0xff,0xb2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_frexp_mant_f32 v5, v1
-// GFX12: encoding: [0x01,0x81,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, v1             ; encoding: [0x01,0x81,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, v255
-// GFX12: encoding: [0xff,0x81,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, v255           ; encoding: [0xff,0x81,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, s1
-// GFX12: encoding: [0x01,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, s1             ; encoding: [0x01,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, s105
-// GFX12: encoding: [0x69,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, s105           ; encoding: [0x69,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, vcc_lo         ; encoding: [0x6a,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, vcc_hi         ; encoding: [0x6b,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, ttmp15         ; encoding: [0x7b,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, m0
-// GFX12: encoding: [0x7d,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, m0             ; encoding: [0x7d,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, exec_lo        ; encoding: [0x7e,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, exec_hi        ; encoding: [0x7f,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, null
-// GFX12: encoding: [0x7c,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, null           ; encoding: [0x7c,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, -1
-// GFX12: encoding: [0xc1,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, -1             ; encoding: [0xc1,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, 0.5            ; encoding: [0xf0,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, src_scc        ; encoding: [0xfd,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x80,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_frexp_mant_f32_e32 v255, 0xaf123456   ; encoding: [0xff,0x80,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_mant_f64 v[5:6], v[1:2]
-// GFX12: encoding: [0x01,0x7b,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], v[1:2]     ; encoding: [0x01,0x7b,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], v[254:255]
-// GFX12: encoding: [0xfe,0x7b,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x7b,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], s[2:3]
-// GFX12: encoding: [0x02,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], s[2:3]     ; encoding: [0x02,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], s[104:105]
-// GFX12: encoding: [0x68,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], vcc
-// GFX12: encoding: [0x6a,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], vcc        ; encoding: [0x6a,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], exec
-// GFX12: encoding: [0x7e,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], exec       ; encoding: [0x7e,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], null
-// GFX12: encoding: [0x7c,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], null       ; encoding: [0x7c,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], -1
-// GFX12: encoding: [0xc1,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], -1         ; encoding: [0xc1,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], 0.5        ; encoding: [0xf0,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], src_scc    ; encoding: [0xfd,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x7a,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_frexp_mant_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x7a,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_log_f16 v5, v1
-// GFX12: encoding: [0x01,0xaf,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, v1                    ; encoding: [0x01,0xaf,0x0a,0x7e]
 
 v_log_f16 v5, v127
-// GFX12: encoding: [0x7f,0xaf,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, v127                  ; encoding: [0x7f,0xaf,0x0a,0x7e]
 
 v_log_f16 v5, s1
-// GFX12: encoding: [0x01,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, s1                    ; encoding: [0x01,0xae,0x0a,0x7e]
 
 v_log_f16 v5, s105
-// GFX12: encoding: [0x69,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, s105                  ; encoding: [0x69,0xae,0x0a,0x7e]
 
 v_log_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xae,0x0a,0x7e]
 
 v_log_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xae,0x0a,0x7e]
 
 v_log_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xae,0x0a,0x7e]
 
 v_log_f16 v5, m0
-// GFX12: encoding: [0x7d,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, m0                    ; encoding: [0x7d,0xae,0x0a,0x7e]
 
 v_log_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xae,0x0a,0x7e]
 
 v_log_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xae,0x0a,0x7e]
 
 v_log_f16 v5, null
-// GFX12: encoding: [0x7c,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, null                  ; encoding: [0x7c,0xae,0x0a,0x7e]
 
 v_log_f16 v5, -1
-// GFX12: encoding: [0xc1,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, -1                    ; encoding: [0xc1,0xae,0x0a,0x7e]
 
 v_log_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xae,0x0a,0x7e]
 
 v_log_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, src_scc               ; encoding: [0xfd,0xae,0x0a,0x7e]
 
 v_log_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xae,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_log_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xae,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_log_f32 v5, v1
-// GFX12: encoding: [0x01,0x4f,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, v1                    ; encoding: [0x01,0x4f,0x0a,0x7e]
 
 v_log_f32 v5, v255
-// GFX12: encoding: [0xff,0x4f,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, v255                  ; encoding: [0xff,0x4f,0x0a,0x7e]
 
 v_log_f32 v5, s1
-// GFX12: encoding: [0x01,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, s1                    ; encoding: [0x01,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, s105
-// GFX12: encoding: [0x69,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, s105                  ; encoding: [0x69,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, m0
-// GFX12: encoding: [0x7d,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, m0                    ; encoding: [0x7d,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, null
-// GFX12: encoding: [0x7c,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, null                  ; encoding: [0x7c,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, -1
-// GFX12: encoding: [0xc1,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, -1                    ; encoding: [0xc1,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, src_scc               ; encoding: [0xfd,0x4e,0x0a,0x7e]
 
 v_log_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x4e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_log_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x4e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_mov_b32 v5, v1
-// GFX12: encoding: [0x01,0x03,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, v1                    ; encoding: [0x01,0x03,0x0a,0x7e]
 
 v_mov_b32 v5, v255
-// GFX12: encoding: [0xff,0x03,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, v255                  ; encoding: [0xff,0x03,0x0a,0x7e]
 
 v_mov_b32 v5, s1
-// GFX12: encoding: [0x01,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, s1                    ; encoding: [0x01,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, s105
-// GFX12: encoding: [0x69,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, s105                  ; encoding: [0x69,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, vcc_lo                ; encoding: [0x6a,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, vcc_hi                ; encoding: [0x6b,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, ttmp15                ; encoding: [0x7b,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, m0
-// GFX12: encoding: [0x7d,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, m0                    ; encoding: [0x7d,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, exec_lo               ; encoding: [0x7e,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, exec_hi               ; encoding: [0x7f,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, null
-// GFX12: encoding: [0x7c,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, null                  ; encoding: [0x7c,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, -1
-// GFX12: encoding: [0xc1,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, -1                    ; encoding: [0xc1,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, 0.5
-// GFX12: encoding: [0xf0,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, 0.5                   ; encoding: [0xf0,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, src_scc
-// GFX12: encoding: [0xfd,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, src_scc               ; encoding: [0xfd,0x02,0x0a,0x7e]
 
 v_mov_b32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x02,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_mov_b32_e32 v255, 0xaf123456          ; encoding: [0xff,0x02,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_movreld_b32 v5, v1
-// GFX12: encoding: [0x01,0x85,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, v1                ; encoding: [0x01,0x85,0x0a,0x7e]
 
 v_movreld_b32 v5, v255
-// GFX12: encoding: [0xff,0x85,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, v255              ; encoding: [0xff,0x85,0x0a,0x7e]
 
 v_movreld_b32 v5, s1
-// GFX12: encoding: [0x01,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, s1                ; encoding: [0x01,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, s105
-// GFX12: encoding: [0x69,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, s105              ; encoding: [0x69,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, m0
-// GFX12: encoding: [0x7d,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, m0                ; encoding: [0x7d,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, null
-// GFX12: encoding: [0x7c,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, null              ; encoding: [0x7c,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, -1
-// GFX12: encoding: [0xc1,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, -1                ; encoding: [0xc1,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, 0.5
-// GFX12: encoding: [0xf0,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, 0.5               ; encoding: [0xf0,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, src_scc
-// GFX12: encoding: [0xfd,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, src_scc           ; encoding: [0xfd,0x84,0x0a,0x7e]
 
 v_movreld_b32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x84,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_movreld_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x84,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_movrels_b32 v5, v1
-// GFX12: encoding: [0x01,0x87,0x0a,0x7e]
+// GFX12: v_movrels_b32_e32 v5, v1                ; encoding: [0x01,0x87,0x0a,0x7e]
 
 v_movrels_b32 v255, v255
-// GFX12: encoding: [0xff,0x87,0xfe,0x7f]
+// GFX12: v_movrels_b32_e32 v255, v255            ; encoding: [0xff,0x87,0xfe,0x7f]
 
 v_movrelsd_2_b32 v5, v1
-// GFX12: encoding: [0x01,0x91,0x0a,0x7e]
+// GFX12: v_movrelsd_2_b32_e32 v5, v1             ; encoding: [0x01,0x91,0x0a,0x7e]
 
 v_movrelsd_2_b32 v255, v255
-// GFX12: encoding: [0xff,0x91,0xfe,0x7f]
+// GFX12: v_movrelsd_2_b32_e32 v255, v255         ; encoding: [0xff,0x91,0xfe,0x7f]
 
 v_movrelsd_b32 v5, v1
-// GFX12: encoding: [0x01,0x89,0x0a,0x7e]
+// GFX12: v_movrelsd_b32_e32 v5, v1               ; encoding: [0x01,0x89,0x0a,0x7e]
 
 v_movrelsd_b32 v255, v255
-// GFX12: encoding: [0xff,0x89,0xfe,0x7f]
+// GFX12: v_movrelsd_b32_e32 v255, v255           ; encoding: [0xff,0x89,0xfe,0x7f]
 
 v_nop
-// GFX12: encoding: [0x00,0x00,0x00,0x7e]
+// GFX12: v_nop                                   ; encoding: [0x00,0x00,0x00,0x7e]
 
 v_not_b16 v5, v1
-// GFX12: encoding: [0x01,0xd3,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, v1                    ; encoding: [0x01,0xd3,0x0a,0x7e]
 
 v_not_b16 v5, v127
-// GFX12: encoding: [0x7f,0xd3,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, v127                  ; encoding: [0x7f,0xd3,0x0a,0x7e]
 
 v_not_b16 v5, s1
-// GFX12: encoding: [0x01,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, s1                    ; encoding: [0x01,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, s105
-// GFX12: encoding: [0x69,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, s105                  ; encoding: [0x69,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, vcc_lo                ; encoding: [0x6a,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, vcc_hi                ; encoding: [0x6b,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, ttmp15                ; encoding: [0x7b,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, m0
-// GFX12: encoding: [0x7d,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, m0                    ; encoding: [0x7d,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, exec_lo               ; encoding: [0x7e,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, exec_hi               ; encoding: [0x7f,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, null
-// GFX12: encoding: [0x7c,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, null                  ; encoding: [0x7c,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, -1
-// GFX12: encoding: [0xc1,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, -1                    ; encoding: [0xc1,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, 0.5
-// GFX12: encoding: [0xf0,0xd2,0x0a,0x7e]
+// GFX12-ASM: v_not_b16_e32 v5, 0.5                   ; encoding: [0xf0,0xd2,0x0a,0x7e]
+// GFX12-DIS: v_not_b16_e32 v5, 0x3800                ; encoding: [0xff,0xd2,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_not_b16 v5, src_scc
-// GFX12: encoding: [0xfd,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, src_scc               ; encoding: [0xfd,0xd2,0x0a,0x7e]
 
 v_not_b16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xd2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_not_b16_e32 v127, 0xfe0b              ; encoding: [0xff,0xd2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_not_b32 v5, v1
-// GFX12: encoding: [0x01,0x6f,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, v1                    ; encoding: [0x01,0x6f,0x0a,0x7e]
 
 v_not_b32 v5, v255
-// GFX12: encoding: [0xff,0x6f,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, v255                  ; encoding: [0xff,0x6f,0x0a,0x7e]
 
 v_not_b32 v5, s1
-// GFX12: encoding: [0x01,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, s1                    ; encoding: [0x01,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, s105
-// GFX12: encoding: [0x69,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, s105                  ; encoding: [0x69,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, ttmp15                ; encoding: [0x7b,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, m0
-// GFX12: encoding: [0x7d,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, m0                    ; encoding: [0x7d,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, exec_lo               ; encoding: [0x7e,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, exec_hi               ; encoding: [0x7f,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, null
-// GFX12: encoding: [0x7c,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, null                  ; encoding: [0x7c,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, -1
-// GFX12: encoding: [0xc1,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, -1                    ; encoding: [0xc1,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, 0.5
-// GFX12: encoding: [0xf0,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, 0.5                   ; encoding: [0xf0,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, src_scc
-// GFX12: encoding: [0xfd,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, src_scc               ; encoding: [0xfd,0x6e,0x0a,0x7e]
 
 v_not_b32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x6e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_not_b32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_permlane64_b32 v5, v1
-// GFX12: encoding: [0x01,0xcf,0x0a,0x7e]
+// GFX12: v_permlane64_b32 v5, v1                 ; encoding: [0x01,0xcf,0x0a,0x7e]
 
 v_permlane64_b32 v255, v255
-// GFX12: encoding: [0xff,0xcf,0xfe,0x7f]
+// GFX12: v_permlane64_b32 v255, v255             ; encoding: [0xff,0xcf,0xfe,0x7f]
 
 v_pipeflush
-// GFX12: encoding: [0x00,0x36,0x00,0x7e]
+// GFX12: v_pipeflush                             ; encoding: [0x00,0x36,0x00,0x7e]
 
 v_rcp_f16 v5, v1
-// GFX12: encoding: [0x01,0xa9,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, v1                    ; encoding: [0x01,0xa9,0x0a,0x7e]
 
 v_rcp_f16 v5, v127
-// GFX12: encoding: [0x7f,0xa9,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, v127                  ; encoding: [0x7f,0xa9,0x0a,0x7e]
 
 v_rcp_f16 v5, s1
-// GFX12: encoding: [0x01,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, s1                    ; encoding: [0x01,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, s105
-// GFX12: encoding: [0x69,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, s105                  ; encoding: [0x69,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, m0
-// GFX12: encoding: [0x7d,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, m0                    ; encoding: [0x7d,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, null
-// GFX12: encoding: [0x7c,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, null                  ; encoding: [0x7c,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, -1
-// GFX12: encoding: [0xc1,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, -1                    ; encoding: [0xc1,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, src_scc               ; encoding: [0xfd,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xa8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_rcp_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xa8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_rcp_f32 v5, v1
-// GFX12: encoding: [0x01,0x55,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, v1                    ; encoding: [0x01,0x55,0x0a,0x7e]
 
 v_rcp_f32 v5, v255
-// GFX12: encoding: [0xff,0x55,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, v255                  ; encoding: [0xff,0x55,0x0a,0x7e]
 
 v_rcp_f32 v5, s1
-// GFX12: encoding: [0x01,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, s1                    ; encoding: [0x01,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, s105
-// GFX12: encoding: [0x69,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, s105                  ; encoding: [0x69,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, m0
-// GFX12: encoding: [0x7d,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, m0                    ; encoding: [0x7d,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, null
-// GFX12: encoding: [0x7c,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, null                  ; encoding: [0x7c,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, -1
-// GFX12: encoding: [0xc1,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, -1                    ; encoding: [0xc1,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, src_scc               ; encoding: [0xfd,0x54,0x0a,0x7e]
 
 v_rcp_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x54,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rcp_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x54,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rcp_f64 v[5:6], v[1:2]
-// GFX12: encoding: [0x01,0x5f,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], v[1:2]            ; encoding: [0x01,0x5f,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], v[254:255]
-// GFX12: encoding: [0xfe,0x5f,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], v[254:255]        ; encoding: [0xfe,0x5f,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], s[2:3]
-// GFX12: encoding: [0x02,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], s[2:3]            ; encoding: [0x02,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], s[104:105]
-// GFX12: encoding: [0x68,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], s[104:105]        ; encoding: [0x68,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], vcc
-// GFX12: encoding: [0x6a,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], vcc               ; encoding: [0x6a,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], ttmp[14:15]       ; encoding: [0x7a,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], exec
-// GFX12: encoding: [0x7e,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], exec              ; encoding: [0x7e,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], null
-// GFX12: encoding: [0x7c,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], null              ; encoding: [0x7c,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], -1
-// GFX12: encoding: [0xc1,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], -1                ; encoding: [0xc1,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], 0.5               ; encoding: [0xf0,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], src_scc           ; encoding: [0xfd,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x5e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rcp_f64_e32 v[254:255], 0xaf123456    ; encoding: [0xff,0x5e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rcp_iflag_f32 v5, v1
-// GFX12: encoding: [0x01,0x57,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, v1              ; encoding: [0x01,0x57,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, v255
-// GFX12: encoding: [0xff,0x57,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, v255            ; encoding: [0xff,0x57,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, s1
-// GFX12: encoding: [0x01,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, s1              ; encoding: [0x01,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, s105
-// GFX12: encoding: [0x69,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, s105            ; encoding: [0x69,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, vcc_lo          ; encoding: [0x6a,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, vcc_hi          ; encoding: [0x6b,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, ttmp15          ; encoding: [0x7b,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, m0
-// GFX12: encoding: [0x7d,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, m0              ; encoding: [0x7d,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, exec_lo         ; encoding: [0x7e,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, exec_hi         ; encoding: [0x7f,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, null
-// GFX12: encoding: [0x7c,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, null            ; encoding: [0x7c,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, -1
-// GFX12: encoding: [0xc1,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, -1              ; encoding: [0xc1,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, 0.5             ; encoding: [0xf0,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, src_scc         ; encoding: [0xfd,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x56,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rcp_iflag_f32_e32 v255, 0xaf123456    ; encoding: [0xff,0x56,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_readfirstlane_b32 s5, v1
-// GFX12: encoding: [0x01,0x05,0x0a,0x7e]
+// GFX12: v_readfirstlane_b32 s5, v1              ; encoding: [0x01,0x05,0x0a,0x7e]
 
 v_readfirstlane_b32 s105, v1
-// GFX12: encoding: [0x01,0x05,0xd2,0x7e]
+// GFX12: v_readfirstlane_b32 s105, v1            ; encoding: [0x01,0x05,0xd2,0x7e]
 
 v_readfirstlane_b32 vcc_lo, v1
-// GFX12: encoding: [0x01,0x05,0xd4,0x7e]
+// GFX12: v_readfirstlane_b32 vcc_lo, v1          ; encoding: [0x01,0x05,0xd4,0x7e]
 
 v_readfirstlane_b32 vcc_hi, v1
-// GFX12: encoding: [0x01,0x05,0xd6,0x7e]
+// GFX12: v_readfirstlane_b32 vcc_hi, v1          ; encoding: [0x01,0x05,0xd6,0x7e]
 
 v_readfirstlane_b32 ttmp15, v1
-// GFX12: encoding: [0x01,0x05,0xf6,0x7e]
+// GFX12: v_readfirstlane_b32 ttmp15, v1          ; encoding: [0x01,0x05,0xf6,0x7e]
 
 v_readfirstlane_b32 null, v255
-// GFX12: encoding: [0xff,0x05,0xf8,0x7e]
+// GFX12: v_readfirstlane_b32 null, v255          ; encoding: [0xff,0x05,0xf8,0x7e]
 
 v_rndne_f16 v5, v1
-// GFX12: encoding: [0x01,0xbd,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, v1                  ; encoding: [0x01,0xbd,0x0a,0x7e]
 
 v_rndne_f16 v5, v127
-// GFX12: encoding: [0x7f,0xbd,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, v127                ; encoding: [0x7f,0xbd,0x0a,0x7e]
 
 v_rndne_f16 v5, s1
-// GFX12: encoding: [0x01,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, s1                  ; encoding: [0x01,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, s105
-// GFX12: encoding: [0x69,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, s105                ; encoding: [0x69,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, m0
-// GFX12: encoding: [0x7d,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, m0                  ; encoding: [0x7d,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, null
-// GFX12: encoding: [0x7c,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, null                ; encoding: [0x7c,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, -1
-// GFX12: encoding: [0xc1,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, -1                  ; encoding: [0xc1,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, src_scc             ; encoding: [0xfd,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xbc,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_rndne_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xbc,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_rndne_f32 v5, v1
-// GFX12: encoding: [0x01,0x47,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, v1                  ; encoding: [0x01,0x47,0x0a,0x7e]
 
 v_rndne_f32 v5, v255
-// GFX12: encoding: [0xff,0x47,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, v255                ; encoding: [0xff,0x47,0x0a,0x7e]
 
 v_rndne_f32 v5, s1
-// GFX12: encoding: [0x01,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, s1                  ; encoding: [0x01,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, s105
-// GFX12: encoding: [0x69,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, s105                ; encoding: [0x69,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, m0
-// GFX12: encoding: [0x7d,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, m0                  ; encoding: [0x7d,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, null
-// GFX12: encoding: [0x7c,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, null                ; encoding: [0x7c,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, -1
-// GFX12: encoding: [0xc1,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, -1                  ; encoding: [0xc1,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, src_scc             ; encoding: [0xfd,0x46,0x0a,0x7e]
 
 v_rndne_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x46,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rndne_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x46,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rndne_f64 v[5:6], v[1:2]
-// GFX12: encoding: [0x01,0x33,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x33,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], v[254:255]
-// GFX12: encoding: [0xfe,0x33,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x33,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], s[2:3]
-// GFX12: encoding: [0x02,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], s[104:105]
-// GFX12: encoding: [0x68,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], vcc
-// GFX12: encoding: [0x6a,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], exec
-// GFX12: encoding: [0x7e,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], null
-// GFX12: encoding: [0x7c,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], null            ; encoding: [0x7c,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], -1
-// GFX12: encoding: [0xc1,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x32,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rndne_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x32,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rsq_f16 v5, v1
-// GFX12: encoding: [0x01,0xad,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, v1                    ; encoding: [0x01,0xad,0x0a,0x7e]
 
 v_rsq_f16 v5, v127
-// GFX12: encoding: [0x7f,0xad,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, v127                  ; encoding: [0x7f,0xad,0x0a,0x7e]
 
 v_rsq_f16 v5, s1
-// GFX12: encoding: [0x01,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, s1                    ; encoding: [0x01,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, s105
-// GFX12: encoding: [0x69,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, s105                  ; encoding: [0x69,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, m0
-// GFX12: encoding: [0x7d,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, m0                    ; encoding: [0x7d,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, null
-// GFX12: encoding: [0x7c,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, null                  ; encoding: [0x7c,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, -1
-// GFX12: encoding: [0xc1,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, -1                    ; encoding: [0xc1,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, src_scc               ; encoding: [0xfd,0xac,0x0a,0x7e]
 
 v_rsq_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xac,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_rsq_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xac,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_rsq_f32 v5, v1
-// GFX12: encoding: [0x01,0x5d,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, v1                    ; encoding: [0x01,0x5d,0x0a,0x7e]
 
 v_rsq_f32 v5, v255
-// GFX12: encoding: [0xff,0x5d,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, v255                  ; encoding: [0xff,0x5d,0x0a,0x7e]
 
 v_rsq_f32 v5, s1
-// GFX12: encoding: [0x01,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, s1                    ; encoding: [0x01,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, s105
-// GFX12: encoding: [0x69,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, s105                  ; encoding: [0x69,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, m0
-// GFX12: encoding: [0x7d,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, m0                    ; encoding: [0x7d,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, null
-// GFX12: encoding: [0x7c,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, null                  ; encoding: [0x7c,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, -1
-// GFX12: encoding: [0xc1,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, -1                    ; encoding: [0xc1,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, src_scc               ; encoding: [0xfd,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x5c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rsq_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x5c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rsq_f64 v[5:6], v[1:2]
-// GFX12: encoding: [0x01,0x63,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], v[1:2]            ; encoding: [0x01,0x63,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], v[254:255]
-// GFX12: encoding: [0xfe,0x63,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], v[254:255]        ; encoding: [0xfe,0x63,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], s[2:3]
-// GFX12: encoding: [0x02,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], s[2:3]            ; encoding: [0x02,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], s[104:105]
-// GFX12: encoding: [0x68,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], s[104:105]        ; encoding: [0x68,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], vcc
-// GFX12: encoding: [0x6a,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], vcc               ; encoding: [0x6a,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], ttmp[14:15]       ; encoding: [0x7a,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], exec
-// GFX12: encoding: [0x7e,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], exec              ; encoding: [0x7e,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], null
-// GFX12: encoding: [0x7c,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], null              ; encoding: [0x7c,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], -1
-// GFX12: encoding: [0xc1,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], -1                ; encoding: [0xc1,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], 0.5               ; encoding: [0xf0,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], src_scc           ; encoding: [0xfd,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x62,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rsq_f64_e32 v[254:255], 0xaf123456    ; encoding: [0xff,0x62,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_sat_pk_u8_i16 v5, v1
-// GFX12: encoding: [0x01,0xc5,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, v1              ; encoding: [0x01,0xc5,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, v255
-// GFX12: encoding: [0xff,0xc5,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, v255            ; encoding: [0xff,0xc5,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, s1
-// GFX12: encoding: [0x01,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, s1              ; encoding: [0x01,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, s105
-// GFX12: encoding: [0x69,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, s105            ; encoding: [0x69,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, vcc_lo          ; encoding: [0x6a,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, vcc_hi          ; encoding: [0x6b,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, ttmp15          ; encoding: [0x7b,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, m0
-// GFX12: encoding: [0x7d,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, m0              ; encoding: [0x7d,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, exec_lo         ; encoding: [0x7e,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, exec_hi         ; encoding: [0x7f,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, null
-// GFX12: encoding: [0x7c,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, null            ; encoding: [0x7c,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, -1
-// GFX12: encoding: [0xc1,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, -1              ; encoding: [0xc1,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, 0.5
-// GFX12: encoding: [0xf0,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, 0.5             ; encoding: [0xf0,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, src_scc
-// GFX12: encoding: [0xfd,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, src_scc         ; encoding: [0xfd,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xc4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_sat_pk_u8_i16_e32 v127, 0xfe0b        ; encoding: [0xff,0xc4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_sin_f16 v5, v1
-// GFX12: encoding: [0x01,0xc1,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, v1                    ; encoding: [0x01,0xc1,0x0a,0x7e]
 
 v_sin_f16 v5, v127
-// GFX12: encoding: [0x7f,0xc1,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, v127                  ; encoding: [0x7f,0xc1,0x0a,0x7e]
 
 v_sin_f16 v5, s1
-// GFX12: encoding: [0x01,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, s1                    ; encoding: [0x01,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, s105
-// GFX12: encoding: [0x69,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, s105                  ; encoding: [0x69,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, m0
-// GFX12: encoding: [0x7d,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, m0                    ; encoding: [0x7d,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, null
-// GFX12: encoding: [0x7c,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, null                  ; encoding: [0x7c,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, -1
-// GFX12: encoding: [0xc1,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, -1                    ; encoding: [0xc1,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, src_scc               ; encoding: [0xfd,0xc0,0x0a,0x7e]
 
 v_sin_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xc0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_sin_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xc0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_sin_f32 v5, v1
-// GFX12: encoding: [0x01,0x6b,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, v1                    ; encoding: [0x01,0x6b,0x0a,0x7e]
 
 v_sin_f32 v5, v255
-// GFX12: encoding: [0xff,0x6b,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, v255                  ; encoding: [0xff,0x6b,0x0a,0x7e]
 
 v_sin_f32 v5, s1
-// GFX12: encoding: [0x01,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, s1                    ; encoding: [0x01,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, s105
-// GFX12: encoding: [0x69,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, s105                  ; encoding: [0x69,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, m0
-// GFX12: encoding: [0x7d,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, m0                    ; encoding: [0x7d,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, null
-// GFX12: encoding: [0x7c,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, null                  ; encoding: [0x7c,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, -1
-// GFX12: encoding: [0xc1,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, -1                    ; encoding: [0xc1,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, src_scc               ; encoding: [0xfd,0x6a,0x0a,0x7e]
 
 v_sin_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x6a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_sin_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_sqrt_f16 v5, v1
-// GFX12: encoding: [0x01,0xab,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, v1                   ; encoding: [0x01,0xab,0x0a,0x7e]
 
 v_sqrt_f16 v5, v127
-// GFX12: encoding: [0x7f,0xab,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, v127                 ; encoding: [0x7f,0xab,0x0a,0x7e]
 
 v_sqrt_f16 v5, s1
-// GFX12: encoding: [0x01,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, s1                   ; encoding: [0x01,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, s105
-// GFX12: encoding: [0x69,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, s105                 ; encoding: [0x69,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, vcc_lo               ; encoding: [0x6a,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, vcc_hi               ; encoding: [0x6b,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, ttmp15               ; encoding: [0x7b,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, m0
-// GFX12: encoding: [0x7d,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, m0                   ; encoding: [0x7d,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, exec_lo              ; encoding: [0x7e,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, exec_hi              ; encoding: [0x7f,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, null
-// GFX12: encoding: [0x7c,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, null                 ; encoding: [0x7c,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, -1
-// GFX12: encoding: [0xc1,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, -1                   ; encoding: [0xc1,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, 0.5                  ; encoding: [0xf0,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, src_scc              ; encoding: [0xfd,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xaa,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_sqrt_f16_e32 v127, 0xfe0b             ; encoding: [0xff,0xaa,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_sqrt_f32 v5, v1
-// GFX12: encoding: [0x01,0x67,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, v1                   ; encoding: [0x01,0x67,0x0a,0x7e]
 
 v_sqrt_f32 v5, v255
-// GFX12: encoding: [0xff,0x67,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, v255                 ; encoding: [0xff,0x67,0x0a,0x7e]
 
 v_sqrt_f32 v5, s1
-// GFX12: encoding: [0x01,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, s1                   ; encoding: [0x01,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, s105
-// GFX12: encoding: [0x69,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, s105                 ; encoding: [0x69,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, vcc_lo               ; encoding: [0x6a,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, vcc_hi               ; encoding: [0x6b,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, ttmp15               ; encoding: [0x7b,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, m0
-// GFX12: encoding: [0x7d,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, m0                   ; encoding: [0x7d,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, exec_lo              ; encoding: [0x7e,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, exec_hi              ; encoding: [0x7f,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, null
-// GFX12: encoding: [0x7c,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, null                 ; encoding: [0x7c,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, -1
-// GFX12: encoding: [0xc1,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, -1                   ; encoding: [0xc1,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, 0.5                  ; encoding: [0xf0,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, src_scc              ; encoding: [0xfd,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x66,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_sqrt_f32_e32 v255, 0xaf123456         ; encoding: [0xff,0x66,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_sqrt_f64 v[5:6], v[1:2]
-// GFX12: encoding: [0x01,0x69,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], v[1:2]           ; encoding: [0x01,0x69,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], v[254:255]
-// GFX12: encoding: [0xfe,0x69,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], v[254:255]       ; encoding: [0xfe,0x69,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], s[2:3]
-// GFX12: encoding: [0x02,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], s[2:3]           ; encoding: [0x02,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], s[104:105]
-// GFX12: encoding: [0x68,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], s[104:105]       ; encoding: [0x68,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], vcc
-// GFX12: encoding: [0x6a,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], vcc              ; encoding: [0x6a,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], ttmp[14:15]      ; encoding: [0x7a,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], exec
-// GFX12: encoding: [0x7e,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], exec             ; encoding: [0x7e,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], null
-// GFX12: encoding: [0x7c,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], null             ; encoding: [0x7c,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], -1
-// GFX12: encoding: [0xc1,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], -1               ; encoding: [0xc1,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], 0.5              ; encoding: [0xf0,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], src_scc          ; encoding: [0xfd,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x68,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_sqrt_f64_e32 v[254:255], 0xaf123456   ; encoding: [0xff,0x68,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_swap_b32 v5, v1
-// GFX12: encoding: [0x01,0xcb,0x0a,0x7e]
+// GFX12: v_swap_b32 v5, v1                       ; encoding: [0x01,0xcb,0x0a,0x7e]
 
 v_swap_b32 v255, v255
-// GFX12: encoding: [0xff,0xcb,0xfe,0x7f]
+// GFX12: v_swap_b32 v255, v255                   ; encoding: [0xff,0xcb,0xfe,0x7f]
 
 v_swaprel_b32 v5, v1
-// GFX12: encoding: [0x01,0xd1,0x0a,0x7e]
+// GFX12: v_swaprel_b32 v5, v1                    ; encoding: [0x01,0xd1,0x0a,0x7e]
 
 v_swaprel_b32 v255, v255
-// GFX12: encoding: [0xff,0xd1,0xfe,0x7f]
+// GFX12: v_swaprel_b32 v255, v255                ; encoding: [0xff,0xd1,0xfe,0x7f]
 
 v_trunc_f16 v5, v1
-// GFX12: encoding: [0x01,0xbb,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, v1                  ; encoding: [0x01,0xbb,0x0a,0x7e]
 
 v_trunc_f16 v5, v127
-// GFX12: encoding: [0x7f,0xbb,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, v127                ; encoding: [0x7f,0xbb,0x0a,0x7e]
 
 v_trunc_f16 v5, s1
-// GFX12: encoding: [0x01,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, s1                  ; encoding: [0x01,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, s105
-// GFX12: encoding: [0x69,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, s105                ; encoding: [0x69,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, vcc_lo
-// GFX12: encoding: [0x6a,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, vcc_hi
-// GFX12: encoding: [0x6b,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, ttmp15
-// GFX12: encoding: [0x7b,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, m0
-// GFX12: encoding: [0x7d,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, m0                  ; encoding: [0x7d,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, exec_lo
-// GFX12: encoding: [0x7e,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, exec_hi
-// GFX12: encoding: [0x7f,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, null
-// GFX12: encoding: [0x7c,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, null                ; encoding: [0x7c,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, -1
-// GFX12: encoding: [0xc1,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, -1                  ; encoding: [0xc1,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, 0.5
-// GFX12: encoding: [0xf0,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, src_scc
-// GFX12: encoding: [0xfd,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, src_scc             ; encoding: [0xfd,0xba,0x0a,0x7e]
 
 v_trunc_f16 v127, 0xfe0b
-// GFX12: encoding: [0xff,0xba,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_trunc_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xba,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_trunc_f32 v5, v1
-// GFX12: encoding: [0x01,0x43,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, v1                  ; encoding: [0x01,0x43,0x0a,0x7e]
 
 v_trunc_f32 v5, v255
-// GFX12: encoding: [0xff,0x43,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, v255                ; encoding: [0xff,0x43,0x0a,0x7e]
 
 v_trunc_f32 v5, s1
-// GFX12: encoding: [0x01,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, s1                  ; encoding: [0x01,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, s105
-// GFX12: encoding: [0x69,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, s105                ; encoding: [0x69,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, vcc_lo
-// GFX12: encoding: [0x6a,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, vcc_hi
-// GFX12: encoding: [0x6b,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, ttmp15
-// GFX12: encoding: [0x7b,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, m0
-// GFX12: encoding: [0x7d,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, m0                  ; encoding: [0x7d,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, exec_lo
-// GFX12: encoding: [0x7e,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, exec_hi
-// GFX12: encoding: [0x7f,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, null
-// GFX12: encoding: [0x7c,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, null                ; encoding: [0x7c,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, -1
-// GFX12: encoding: [0xc1,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, -1                  ; encoding: [0xc1,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, 0.5
-// GFX12: encoding: [0xf0,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, src_scc
-// GFX12: encoding: [0xfd,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, src_scc             ; encoding: [0xfd,0x42,0x0a,0x7e]
 
 v_trunc_f32 v255, 0xaf123456
-// GFX12: encoding: [0xff,0x42,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_trunc_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x42,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_trunc_f64 v[5:6], v[1:2]
-// GFX12: encoding: [0x01,0x2f,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x2f,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], v[254:255]
-// GFX12: encoding: [0xfe,0x2f,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x2f,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], s[2:3]
-// GFX12: encoding: [0x02,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], s[104:105]
-// GFX12: encoding: [0x68,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], vcc
-// GFX12: encoding: [0x6a,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], exec
-// GFX12: encoding: [0x7e,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], null
-// GFX12: encoding: [0x7c,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], null            ; encoding: [0x7c,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], -1
-// GFX12: encoding: [0xc1,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], 0.5
-// GFX12: encoding: [0xf0,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], src_scc
-// GFX12: encoding: [0xfd,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[254:255], 0xaf123456
-// GFX12: encoding: [0xff,0x2e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_trunc_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x2e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/lit.local.cfg b/llvm/test/MC/AMDGPU/lit.local.cfg
index 7c492428aec76..c5853ad178a04 100644
--- a/llvm/test/MC/AMDGPU/lit.local.cfg
+++ b/llvm/test/MC/AMDGPU/lit.local.cfg
@@ -1,2 +1,4 @@
+config.substitutions.append(("%extract-encodings", "sed 's/.*encoding://p'"))
+
 if not "AMDGPU" in config.root.targets:
     config.unsupported = True
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1.txt
deleted file mode 100644
index 6637a88c3eee5..0000000000000
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1.txt
+++ /dev/null
@@ -1,3338 +0,0 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s
-
-# GFX12: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
-0x01,0x71,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, v255                ; encoding: [0xff,0x71,0x0a,0x7e]
-0xff,0x71,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, s1                  ; encoding: [0x01,0x70,0x0a,0x7e]
-0x01,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, s105                ; encoding: [0x69,0x70,0x0a,0x7e]
-0x69,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, vcc_lo              ; encoding: [0x6a,0x70,0x0a,0x7e]
-0x6a,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, vcc_hi              ; encoding: [0x6b,0x70,0x0a,0x7e]
-0x6b,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, ttmp15              ; encoding: [0x7b,0x70,0x0a,0x7e]
-0x7b,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, m0                  ; encoding: [0x7d,0x70,0x0a,0x7e]
-0x7d,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, exec_lo             ; encoding: [0x7e,0x70,0x0a,0x7e]
-0x7e,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, exec_hi             ; encoding: [0x7f,0x70,0x0a,0x7e]
-0x7f,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, null                ; encoding: [0x7c,0x70,0x0a,0x7e]
-0x7c,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, -1                  ; encoding: [0xc1,0x70,0x0a,0x7e]
-0xc1,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, 0.5                 ; encoding: [0xf0,0x70,0x0a,0x7e]
-0xf0,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v5, src_scc             ; encoding: [0xfd,0x70,0x0a,0x7e]
-0xfd,0x70,0x0a,0x7e
-
-# GFX12: v_bfrev_b32_e32 v255, 0xaf123456        ; encoding: [0xff,0x70,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x70,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_ceil_f16_e32 v5, v1                   ; encoding: [0x01,0xb9,0x0a,0x7e]
-0x01,0xb9,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, v127                 ; encoding: [0x7f,0xb9,0x0a,0x7e]
-0x7f,0xb9,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, s1                   ; encoding: [0x01,0xb8,0x0a,0x7e]
-0x01,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, s105                 ; encoding: [0x69,0xb8,0x0a,0x7e]
-0x69,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, vcc_lo               ; encoding: [0x6a,0xb8,0x0a,0x7e]
-0x6a,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, vcc_hi               ; encoding: [0x6b,0xb8,0x0a,0x7e]
-0x6b,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, ttmp15               ; encoding: [0x7b,0xb8,0x0a,0x7e]
-0x7b,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, m0                   ; encoding: [0x7d,0xb8,0x0a,0x7e]
-0x7d,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, exec_lo              ; encoding: [0x7e,0xb8,0x0a,0x7e]
-0x7e,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, exec_hi              ; encoding: [0x7f,0xb8,0x0a,0x7e]
-0x7f,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, null                 ; encoding: [0x7c,0xb8,0x0a,0x7e]
-0x7c,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, -1                   ; encoding: [0xc1,0xb8,0x0a,0x7e]
-0xc1,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, 0.5                  ; encoding: [0xf0,0xb8,0x0a,0x7e]
-0xf0,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v5, src_scc              ; encoding: [0xfd,0xb8,0x0a,0x7e]
-0xfd,0xb8,0x0a,0x7e
-
-# GFX12: v_ceil_f16_e32 v127, 0xfe0b             ; encoding: [0xff,0xb8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xb8,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_ceil_f32_e32 v5, v1                   ; encoding: [0x01,0x45,0x0a,0x7e]
-0x01,0x45,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, v255                 ; encoding: [0xff,0x45,0x0a,0x7e]
-0xff,0x45,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, s1                   ; encoding: [0x01,0x44,0x0a,0x7e]
-0x01,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, s105                 ; encoding: [0x69,0x44,0x0a,0x7e]
-0x69,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, vcc_lo               ; encoding: [0x6a,0x44,0x0a,0x7e]
-0x6a,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, vcc_hi               ; encoding: [0x6b,0x44,0x0a,0x7e]
-0x6b,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, ttmp15               ; encoding: [0x7b,0x44,0x0a,0x7e]
-0x7b,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, m0                   ; encoding: [0x7d,0x44,0x0a,0x7e]
-0x7d,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, exec_lo              ; encoding: [0x7e,0x44,0x0a,0x7e]
-0x7e,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, exec_hi              ; encoding: [0x7f,0x44,0x0a,0x7e]
-0x7f,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, null                 ; encoding: [0x7c,0x44,0x0a,0x7e]
-0x7c,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, -1                   ; encoding: [0xc1,0x44,0x0a,0x7e]
-0xc1,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, 0.5                  ; encoding: [0xf0,0x44,0x0a,0x7e]
-0xf0,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v5, src_scc              ; encoding: [0xfd,0x44,0x0a,0x7e]
-0xfd,0x44,0x0a,0x7e
-
-# GFX12: v_ceil_f32_e32 v255, 0xaf123456         ; encoding: [0xff,0x44,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x44,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_ceil_f64_e32 v[5:6], v[1:2]           ; encoding: [0x01,0x31,0x0a,0x7e]
-0x01,0x31,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[5:6], v[254:255]       ; encoding: [0xfe,0x31,0x0a,0x7e]
-0xfe,0x31,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[5:6], s[2:3]           ; encoding: [0x02,0x30,0x0a,0x7e]
-0x02,0x30,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[5:6], s[104:105]       ; encoding: [0x68,0x30,0x0a,0x7e]
-0x68,0x30,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[5:6], vcc              ; encoding: [0x6a,0x30,0x0a,0x7e]
-0x6a,0x30,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[5:6], ttmp[14:15]      ; encoding: [0x7a,0x30,0x0a,0x7e]
-0x7a,0x30,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[5:6], exec             ; encoding: [0x7e,0x30,0x0a,0x7e]
-0x7e,0x30,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[5:6], null             ; encoding: [0x7c,0x30,0x0a,0x7e]
-0x7c,0x30,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[5:6], -1               ; encoding: [0xc1,0x30,0x0a,0x7e]
-0xc1,0x30,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[5:6], 0.5              ; encoding: [0xf0,0x30,0x0a,0x7e]
-0xf0,0x30,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[5:6], src_scc          ; encoding: [0xfd,0x30,0x0a,0x7e]
-0xfd,0x30,0x0a,0x7e
-
-# GFX12: v_ceil_f64_e32 v[254:255], 0xaf123456   ; encoding: [0xff,0x30,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x30,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cls_i32_e32 v5, v1                    ; encoding: [0x01,0x77,0x0a,0x7e]
-0x01,0x77,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, v255                  ; encoding: [0xff,0x77,0x0a,0x7e]
-0xff,0x77,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, s1                    ; encoding: [0x01,0x76,0x0a,0x7e]
-0x01,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, s105                  ; encoding: [0x69,0x76,0x0a,0x7e]
-0x69,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, vcc_lo                ; encoding: [0x6a,0x76,0x0a,0x7e]
-0x6a,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, vcc_hi                ; encoding: [0x6b,0x76,0x0a,0x7e]
-0x6b,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, ttmp15                ; encoding: [0x7b,0x76,0x0a,0x7e]
-0x7b,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, m0                    ; encoding: [0x7d,0x76,0x0a,0x7e]
-0x7d,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, exec_lo               ; encoding: [0x7e,0x76,0x0a,0x7e]
-0x7e,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, exec_hi               ; encoding: [0x7f,0x76,0x0a,0x7e]
-0x7f,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, null                  ; encoding: [0x7c,0x76,0x0a,0x7e]
-0x7c,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, -1                    ; encoding: [0xc1,0x76,0x0a,0x7e]
-0xc1,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, 0.5                   ; encoding: [0xf0,0x76,0x0a,0x7e]
-0xf0,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v5, src_scc               ; encoding: [0xfd,0x76,0x0a,0x7e]
-0xfd,0x76,0x0a,0x7e
-
-# GFX12: v_cls_i32_e32 v255, 0xaf123456          ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_clz_i32_u32_e32 v5, v1                ; encoding: [0x01,0x73,0x0a,0x7e]
-0x01,0x73,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, v255              ; encoding: [0xff,0x73,0x0a,0x7e]
-0xff,0x73,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, s1                ; encoding: [0x01,0x72,0x0a,0x7e]
-0x01,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, s105              ; encoding: [0x69,0x72,0x0a,0x7e]
-0x69,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x72,0x0a,0x7e]
-0x6a,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x72,0x0a,0x7e]
-0x6b,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x72,0x0a,0x7e]
-0x7b,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, m0                ; encoding: [0x7d,0x72,0x0a,0x7e]
-0x7d,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x72,0x0a,0x7e]
-0x7e,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x72,0x0a,0x7e]
-0x7f,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, null              ; encoding: [0x7c,0x72,0x0a,0x7e]
-0x7c,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, -1                ; encoding: [0xc1,0x72,0x0a,0x7e]
-0xc1,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x72,0x0a,0x7e]
-0xf0,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x72,0x0a,0x7e]
-0xfd,0x72,0x0a,0x7e
-
-# GFX12: v_clz_i32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cos_f16_e32 v5, v1                    ; encoding: [0x01,0xc3,0x0a,0x7e]
-0x01,0xc3,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, v127                  ; encoding: [0x7f,0xc3,0x0a,0x7e]
-0x7f,0xc3,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, s1                    ; encoding: [0x01,0xc2,0x0a,0x7e]
-0x01,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, s105                  ; encoding: [0x69,0xc2,0x0a,0x7e]
-0x69,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xc2,0x0a,0x7e]
-0x6a,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xc2,0x0a,0x7e]
-0x6b,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xc2,0x0a,0x7e]
-0x7b,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, m0                    ; encoding: [0x7d,0xc2,0x0a,0x7e]
-0x7d,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xc2,0x0a,0x7e]
-0x7e,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xc2,0x0a,0x7e]
-0x7f,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, null                  ; encoding: [0x7c,0xc2,0x0a,0x7e]
-0x7c,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, -1                    ; encoding: [0xc1,0xc2,0x0a,0x7e]
-0xc1,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xc2,0x0a,0x7e]
-0xf0,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v5, src_scc               ; encoding: [0xfd,0xc2,0x0a,0x7e]
-0xfd,0xc2,0x0a,0x7e
-
-# GFX12: v_cos_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xc2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xc2,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_cos_f32_e32 v5, v1                    ; encoding: [0x01,0x6d,0x0a,0x7e]
-0x01,0x6d,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, v255                  ; encoding: [0xff,0x6d,0x0a,0x7e]
-0xff,0x6d,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, s1                    ; encoding: [0x01,0x6c,0x0a,0x7e]
-0x01,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, s105                  ; encoding: [0x69,0x6c,0x0a,0x7e]
-0x69,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6c,0x0a,0x7e]
-0x6a,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6c,0x0a,0x7e]
-0x6b,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x6c,0x0a,0x7e]
-0x7b,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, m0                    ; encoding: [0x7d,0x6c,0x0a,0x7e]
-0x7d,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x6c,0x0a,0x7e]
-0x7e,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x6c,0x0a,0x7e]
-0x7f,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, null                  ; encoding: [0x7c,0x6c,0x0a,0x7e]
-0x7c,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, -1                    ; encoding: [0xc1,0x6c,0x0a,0x7e]
-0xc1,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x6c,0x0a,0x7e]
-0xf0,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v5, src_scc               ; encoding: [0xfd,0x6c,0x0a,0x7e]
-0xfd,0x6c,0x0a,0x7e
-
-# GFX12: v_cos_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x6c,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_ctz_i32_b32_e32 v5, v1                ; encoding: [0x01,0x75,0x0a,0x7e]
-0x01,0x75,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, v255              ; encoding: [0xff,0x75,0x0a,0x7e]
-0xff,0x75,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, s1                ; encoding: [0x01,0x74,0x0a,0x7e]
-0x01,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, s105              ; encoding: [0x69,0x74,0x0a,0x7e]
-0x69,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x74,0x0a,0x7e]
-0x6a,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x74,0x0a,0x7e]
-0x6b,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x74,0x0a,0x7e]
-0x7b,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, m0                ; encoding: [0x7d,0x74,0x0a,0x7e]
-0x7d,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x74,0x0a,0x7e]
-0x7e,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x74,0x0a,0x7e]
-0x7f,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, null              ; encoding: [0x7c,0x74,0x0a,0x7e]
-0x7c,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, -1                ; encoding: [0xc1,0x74,0x0a,0x7e]
-0xc1,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, 0.5               ; encoding: [0xf0,0x74,0x0a,0x7e]
-0xf0,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v5, src_scc           ; encoding: [0xfd,0x74,0x0a,0x7e]
-0xfd,0x74,0x0a,0x7e
-
-# GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f32_bf8_e32 v1, s3                ; encoding: [0x03,0xda,0x02,0x7e]
-0x03,0xda,0x02,0x7e
-
-# GFX12: v_cvt_f32_bf8_e32 v1, 3                 ; encoding: [0x83,0xda,0x02,0x7e]
-0x83,0xda,0x02,0x7e
-
-# GFX12: v_cvt_f32_bf8_e32 v1, v3                ; encoding: [0x03,0xdb,0x02,0x7e]
-0x03,0xdb,0x02,0x7e
-
-# GFX12: v_cvt_f32_fp8_e32 v1, s3                ; encoding: [0x03,0xd8,0x02,0x7e]
-0x03,0xd8,0x02,0x7e
-
-# GFX12: v_cvt_f32_fp8_e32 v1, 3                 ; encoding: [0x83,0xd8,0x02,0x7e]
-0x83,0xd8,0x02,0x7e
-
-# GFX12: v_cvt_f32_fp8_e32 v1, v3                ; encoding: [0x03,0xd9,0x02,0x7e]
-0x03,0xd9,0x02,0x7e
-
-# GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], s3         ; encoding: [0x03,0xde,0x04,0x7e]
-0x03,0xde,0x04,0x7e
-
-# GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], 3          ; encoding: [0x83,0xde,0x04,0x7e]
-0x83,0xde,0x04,0x7e
-
-# GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], v3         ; encoding: [0x03,0xdf,0x04,0x7e]
-0x03,0xdf,0x04,0x7e
-
-# GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], s3         ; encoding: [0x03,0xdc,0x04,0x7e]
-0x03,0xdc,0x04,0x7e
-
-# GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], 3          ; encoding: [0x83,0xdc,0x04,0x7e]
-0x83,0xdc,0x04,0x7e
-
-# GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], v3         ; encoding: [0x03,0xdd,0x04,0x7e]
-0x03,0xdd,0x04,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, v1                ; encoding: [0x01,0x15,0x0a,0x7e]
-0x01,0x15,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, v255              ; encoding: [0xff,0x15,0x0a,0x7e]
-0xff,0x15,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, s1                ; encoding: [0x01,0x14,0x0a,0x7e]
-0x01,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, s105              ; encoding: [0x69,0x14,0x0a,0x7e]
-0x69,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x14,0x0a,0x7e]
-0x6a,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x14,0x0a,0x7e]
-0x6b,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x14,0x0a,0x7e]
-0x7b,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, m0                ; encoding: [0x7d,0x14,0x0a,0x7e]
-0x7d,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x14,0x0a,0x7e]
-0x7e,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x14,0x0a,0x7e]
-0x7f,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, null              ; encoding: [0x7c,0x14,0x0a,0x7e]
-0x7c,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, -1                ; encoding: [0xc1,0x14,0x0a,0x7e]
-0xc1,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, 0.5               ; encoding: [0xf0,0x14,0x0a,0x7e]
-0xf0,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v5, src_scc           ; encoding: [0xfd,0x14,0x0a,0x7e]
-0xfd,0x14,0x0a,0x7e
-
-# GFX12: v_cvt_f16_f32_e32 v127, 0xaf123456      ; encoding: [0xff,0x14,0xfe,0x7e,0x56,0x34,0x12,0xaf]
-0xff,0x14,0xfe,0x7e,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f16_i16_e32 v5, v1                ; encoding: [0x01,0xa3,0x0a,0x7e]
-0x01,0xa3,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, v127              ; encoding: [0x7f,0xa3,0x0a,0x7e]
-0x7f,0xa3,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, s1                ; encoding: [0x01,0xa2,0x0a,0x7e]
-0x01,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, s105              ; encoding: [0x69,0xa2,0x0a,0x7e]
-0x69,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa2,0x0a,0x7e]
-0x6a,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa2,0x0a,0x7e]
-0x6b,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, ttmp15            ; encoding: [0x7b,0xa2,0x0a,0x7e]
-0x7b,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, m0                ; encoding: [0x7d,0xa2,0x0a,0x7e]
-0x7d,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, exec_lo           ; encoding: [0x7e,0xa2,0x0a,0x7e]
-0x7e,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, exec_hi           ; encoding: [0x7f,0xa2,0x0a,0x7e]
-0x7f,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, null              ; encoding: [0x7c,0xa2,0x0a,0x7e]
-0x7c,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, -1                ; encoding: [0xc1,0xa2,0x0a,0x7e]
-0xc1,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, 0x3800
-0xf0,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v5, src_scc           ; encoding: [0xfd,0xa2,0x0a,0x7e]
-0xfd,0xa2,0x0a,0x7e
-
-# GFX12: v_cvt_f16_i16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xa2,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_cvt_f16_u16_e32 v5, v1                ; encoding: [0x01,0xa1,0x0a,0x7e]
-0x01,0xa1,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, v127              ; encoding: [0x7f,0xa1,0x0a,0x7e]
-0x7f,0xa1,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, s1                ; encoding: [0x01,0xa0,0x0a,0x7e]
-0x01,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, s105              ; encoding: [0x69,0xa0,0x0a,0x7e]
-0x69,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa0,0x0a,0x7e]
-0x6a,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa0,0x0a,0x7e]
-0x6b,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, ttmp15            ; encoding: [0x7b,0xa0,0x0a,0x7e]
-0x7b,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, m0                ; encoding: [0x7d,0xa0,0x0a,0x7e]
-0x7d,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, exec_lo           ; encoding: [0x7e,0xa0,0x0a,0x7e]
-0x7e,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, exec_hi           ; encoding: [0x7f,0xa0,0x0a,0x7e]
-0x7f,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, null              ; encoding: [0x7c,0xa0,0x0a,0x7e]
-0x7c,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, -1                ; encoding: [0xc1,0xa0,0x0a,0x7e]
-0xc1,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, 0x3800
-0xf0,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v5, src_scc           ; encoding: [0xfd,0xa0,0x0a,0x7e]
-0xfd,0xa0,0x0a,0x7e
-
-# GFX12: v_cvt_f16_u16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xa0,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_cvt_f32_f16_e32 v5, v1                ; encoding: [0x01,0x17,0x0a,0x7e]
-0x01,0x17,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, v127              ; encoding: [0x7f,0x17,0x0a,0x7e]
-0x7f,0x17,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, s1                ; encoding: [0x01,0x16,0x0a,0x7e]
-0x01,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, s105              ; encoding: [0x69,0x16,0x0a,0x7e]
-0x69,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0x16,0x0a,0x7e]
-0x6a,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0x16,0x0a,0x7e]
-0x6b,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, ttmp15            ; encoding: [0x7b,0x16,0x0a,0x7e]
-0x7b,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, m0                ; encoding: [0x7d,0x16,0x0a,0x7e]
-0x7d,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, exec_lo           ; encoding: [0x7e,0x16,0x0a,0x7e]
-0x7e,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, exec_hi           ; encoding: [0x7f,0x16,0x0a,0x7e]
-0x7f,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, null              ; encoding: [0x7c,0x16,0x0a,0x7e]
-0x7c,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, -1                ; encoding: [0xc1,0x16,0x0a,0x7e]
-0xc1,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, 0.5               ; encoding: [0xf0,0x16,0x0a,0x7e]
-0xf0,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v5, src_scc           ; encoding: [0xfd,0x16,0x0a,0x7e]
-0xfd,0x16,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f16_e32 v255, 0xfe0b          ; encoding: [0xff,0x16,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
-0xff,0x16,0xfe,0x7f,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_cvt_f32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x1f,0x0a,0x7e]
-0x01,0x1f,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x1f,0x0a,0x7e]
-0xfe,0x1f,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x1e,0x0a,0x7e]
-0x02,0x1e,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x1e,0x0a,0x7e]
-0x68,0x1e,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v5, vcc               ; encoding: [0x6a,0x1e,0x0a,0x7e]
-0x6a,0x1e,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x1e,0x0a,0x7e]
-0x7a,0x1e,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v5, exec              ; encoding: [0x7e,0x1e,0x0a,0x7e]
-0x7e,0x1e,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v5, null              ; encoding: [0x7c,0x1e,0x0a,0x7e]
-0x7c,0x1e,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v5, -1                ; encoding: [0xc1,0x1e,0x0a,0x7e]
-0xc1,0x1e,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x1e,0x0a,0x7e]
-0xf0,0x1e,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x1e,0x0a,0x7e]
-0xfd,0x1e,0x0a,0x7e
-
-# GFX12: v_cvt_f32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x1e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x1e,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f32_i32_e32 v5, v1                ; encoding: [0x01,0x0b,0x0a,0x7e]
-0x01,0x0b,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, v255              ; encoding: [0xff,0x0b,0x0a,0x7e]
-0xff,0x0b,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, s1                ; encoding: [0x01,0x0a,0x0a,0x7e]
-0x01,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, s105              ; encoding: [0x69,0x0a,0x0a,0x7e]
-0x69,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0a,0x0a,0x7e]
-0x6a,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0a,0x0a,0x7e]
-0x6b,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, ttmp15            ; encoding: [0x7b,0x0a,0x0a,0x7e]
-0x7b,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, m0                ; encoding: [0x7d,0x0a,0x0a,0x7e]
-0x7d,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, exec_lo           ; encoding: [0x7e,0x0a,0x0a,0x7e]
-0x7e,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, exec_hi           ; encoding: [0x7f,0x0a,0x0a,0x7e]
-0x7f,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, null              ; encoding: [0x7c,0x0a,0x0a,0x7e]
-0x7c,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, -1                ; encoding: [0xc1,0x0a,0x0a,0x7e]
-0xc1,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, 0.5               ; encoding: [0xf0,0x0a,0x0a,0x7e]
-0xf0,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v5, src_scc           ; encoding: [0xfd,0x0a,0x0a,0x7e]
-0xfd,0x0a,0x0a,0x7e
-
-# GFX12: v_cvt_f32_i32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x0a,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f32_u32_e32 v5, v1                ; encoding: [0x01,0x0d,0x0a,0x7e]
-0x01,0x0d,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, v255              ; encoding: [0xff,0x0d,0x0a,0x7e]
-0xff,0x0d,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, s1                ; encoding: [0x01,0x0c,0x0a,0x7e]
-0x01,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, s105              ; encoding: [0x69,0x0c,0x0a,0x7e]
-0x69,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0c,0x0a,0x7e]
-0x6a,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0c,0x0a,0x7e]
-0x6b,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x0c,0x0a,0x7e]
-0x7b,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, m0                ; encoding: [0x7d,0x0c,0x0a,0x7e]
-0x7d,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x0c,0x0a,0x7e]
-0x7e,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x0c,0x0a,0x7e]
-0x7f,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, null              ; encoding: [0x7c,0x0c,0x0a,0x7e]
-0x7c,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, -1                ; encoding: [0xc1,0x0c,0x0a,0x7e]
-0xc1,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x0c,0x0a,0x7e]
-0xf0,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x0c,0x0a,0x7e]
-0xfd,0x0c,0x0a,0x7e
-
-# GFX12: v_cvt_f32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x0c,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, v1             ; encoding: [0x01,0x23,0x0a,0x7e]
-0x01,0x23,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, v255           ; encoding: [0xff,0x23,0x0a,0x7e]
-0xff,0x23,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, s1             ; encoding: [0x01,0x22,0x0a,0x7e]
-0x01,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, s105           ; encoding: [0x69,0x22,0x0a,0x7e]
-0x69,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_lo         ; encoding: [0x6a,0x22,0x0a,0x7e]
-0x6a,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_hi         ; encoding: [0x6b,0x22,0x0a,0x7e]
-0x6b,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, ttmp15         ; encoding: [0x7b,0x22,0x0a,0x7e]
-0x7b,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, m0             ; encoding: [0x7d,0x22,0x0a,0x7e]
-0x7d,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, exec_lo        ; encoding: [0x7e,0x22,0x0a,0x7e]
-0x7e,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, exec_hi        ; encoding: [0x7f,0x22,0x0a,0x7e]
-0x7f,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, null           ; encoding: [0x7c,0x22,0x0a,0x7e]
-0x7c,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, -1             ; encoding: [0xc1,0x22,0x0a,0x7e]
-0xc1,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, 0.5            ; encoding: [0xf0,0x22,0x0a,0x7e]
-0xf0,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v5, src_scc        ; encoding: [0xfd,0x22,0x0a,0x7e]
-0xfd,0x22,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte0_e32 v255, 0xaf123456   ; encoding: [0xff,0x22,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x22,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, v1             ; encoding: [0x01,0x25,0x0a,0x7e]
-0x01,0x25,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, v255           ; encoding: [0xff,0x25,0x0a,0x7e]
-0xff,0x25,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, s1             ; encoding: [0x01,0x24,0x0a,0x7e]
-0x01,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, s105           ; encoding: [0x69,0x24,0x0a,0x7e]
-0x69,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_lo         ; encoding: [0x6a,0x24,0x0a,0x7e]
-0x6a,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_hi         ; encoding: [0x6b,0x24,0x0a,0x7e]
-0x6b,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, ttmp15         ; encoding: [0x7b,0x24,0x0a,0x7e]
-0x7b,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, m0             ; encoding: [0x7d,0x24,0x0a,0x7e]
-0x7d,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, exec_lo        ; encoding: [0x7e,0x24,0x0a,0x7e]
-0x7e,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, exec_hi        ; encoding: [0x7f,0x24,0x0a,0x7e]
-0x7f,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, null           ; encoding: [0x7c,0x24,0x0a,0x7e]
-0x7c,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, -1             ; encoding: [0xc1,0x24,0x0a,0x7e]
-0xc1,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, 0.5            ; encoding: [0xf0,0x24,0x0a,0x7e]
-0xf0,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v5, src_scc        ; encoding: [0xfd,0x24,0x0a,0x7e]
-0xfd,0x24,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte1_e32 v255, 0xaf123456   ; encoding: [0xff,0x24,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x24,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, v1             ; encoding: [0x01,0x27,0x0a,0x7e]
-0x01,0x27,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, v255           ; encoding: [0xff,0x27,0x0a,0x7e]
-0xff,0x27,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, s1             ; encoding: [0x01,0x26,0x0a,0x7e]
-0x01,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, s105           ; encoding: [0x69,0x26,0x0a,0x7e]
-0x69,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_lo         ; encoding: [0x6a,0x26,0x0a,0x7e]
-0x6a,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_hi         ; encoding: [0x6b,0x26,0x0a,0x7e]
-0x6b,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, ttmp15         ; encoding: [0x7b,0x26,0x0a,0x7e]
-0x7b,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, m0             ; encoding: [0x7d,0x26,0x0a,0x7e]
-0x7d,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, exec_lo        ; encoding: [0x7e,0x26,0x0a,0x7e]
-0x7e,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, exec_hi        ; encoding: [0x7f,0x26,0x0a,0x7e]
-0x7f,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, null           ; encoding: [0x7c,0x26,0x0a,0x7e]
-0x7c,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, -1             ; encoding: [0xc1,0x26,0x0a,0x7e]
-0xc1,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, 0.5            ; encoding: [0xf0,0x26,0x0a,0x7e]
-0xf0,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v5, src_scc        ; encoding: [0xfd,0x26,0x0a,0x7e]
-0xfd,0x26,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte2_e32 v255, 0xaf123456   ; encoding: [0xff,0x26,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x26,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, v1             ; encoding: [0x01,0x29,0x0a,0x7e]
-0x01,0x29,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, v255           ; encoding: [0xff,0x29,0x0a,0x7e]
-0xff,0x29,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, s1             ; encoding: [0x01,0x28,0x0a,0x7e]
-0x01,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, s105           ; encoding: [0x69,0x28,0x0a,0x7e]
-0x69,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_lo         ; encoding: [0x6a,0x28,0x0a,0x7e]
-0x6a,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_hi         ; encoding: [0x6b,0x28,0x0a,0x7e]
-0x6b,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, ttmp15         ; encoding: [0x7b,0x28,0x0a,0x7e]
-0x7b,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, m0             ; encoding: [0x7d,0x28,0x0a,0x7e]
-0x7d,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, exec_lo        ; encoding: [0x7e,0x28,0x0a,0x7e]
-0x7e,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, exec_hi        ; encoding: [0x7f,0x28,0x0a,0x7e]
-0x7f,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, null           ; encoding: [0x7c,0x28,0x0a,0x7e]
-0x7c,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, -1             ; encoding: [0xc1,0x28,0x0a,0x7e]
-0xc1,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, 0.5            ; encoding: [0xf0,0x28,0x0a,0x7e]
-0xf0,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v5, src_scc        ; encoding: [0xfd,0x28,0x0a,0x7e]
-0xfd,0x28,0x0a,0x7e
-
-# GFX12: v_cvt_f32_ubyte3_e32 v255, 0xaf123456   ; encoding: [0xff,0x28,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x28,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], v1            ; encoding: [0x01,0x21,0x0a,0x7e]
-0x01,0x21,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], v255          ; encoding: [0xff,0x21,0x0a,0x7e]
-0xff,0x21,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], s1            ; encoding: [0x01,0x20,0x0a,0x7e]
-0x01,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], s105          ; encoding: [0x69,0x20,0x0a,0x7e]
-0x69,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x20,0x0a,0x7e]
-0x6a,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x20,0x0a,0x7e]
-0x6b,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x20,0x0a,0x7e]
-0x7b,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], m0            ; encoding: [0x7d,0x20,0x0a,0x7e]
-0x7d,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x20,0x0a,0x7e]
-0x7e,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x20,0x0a,0x7e]
-0x7f,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], null          ; encoding: [0x7c,0x20,0x0a,0x7e]
-0x7c,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], -1            ; encoding: [0xc1,0x20,0x0a,0x7e]
-0xc1,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x20,0x0a,0x7e]
-0xf0,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x20,0x0a,0x7e]
-0xfd,0x20,0x0a,0x7e
-
-# GFX12: v_cvt_f64_f32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x20,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x20,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], v1            ; encoding: [0x01,0x09,0x0a,0x7e]
-0x01,0x09,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], v255          ; encoding: [0xff,0x09,0x0a,0x7e]
-0xff,0x09,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], s1            ; encoding: [0x01,0x08,0x0a,0x7e]
-0x01,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], s105          ; encoding: [0x69,0x08,0x0a,0x7e]
-0x69,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x08,0x0a,0x7e]
-0x6a,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x08,0x0a,0x7e]
-0x6b,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x08,0x0a,0x7e]
-0x7b,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], m0            ; encoding: [0x7d,0x08,0x0a,0x7e]
-0x7d,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x08,0x0a,0x7e]
-0x7e,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x08,0x0a,0x7e]
-0x7f,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], null          ; encoding: [0x7c,0x08,0x0a,0x7e]
-0x7c,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], -1            ; encoding: [0xc1,0x08,0x0a,0x7e]
-0xc1,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x08,0x0a,0x7e]
-0xf0,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x08,0x0a,0x7e]
-0xfd,0x08,0x0a,0x7e
-
-# GFX12: v_cvt_f64_i32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x08,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x08,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], v1            ; encoding: [0x01,0x2d,0x0a,0x7e]
-0x01,0x2d,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], v255          ; encoding: [0xff,0x2d,0x0a,0x7e]
-0xff,0x2d,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], s1            ; encoding: [0x01,0x2c,0x0a,0x7e]
-0x01,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], s105          ; encoding: [0x69,0x2c,0x0a,0x7e]
-0x69,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x2c,0x0a,0x7e]
-0x6a,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x2c,0x0a,0x7e]
-0x6b,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x2c,0x0a,0x7e]
-0x7b,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], m0            ; encoding: [0x7d,0x2c,0x0a,0x7e]
-0x7d,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x2c,0x0a,0x7e]
-0x7e,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x2c,0x0a,0x7e]
-0x7f,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], null          ; encoding: [0x7c,0x2c,0x0a,0x7e]
-0x7c,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], -1            ; encoding: [0xc1,0x2c,0x0a,0x7e]
-0xc1,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x2c,0x0a,0x7e]
-0xf0,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x2c,0x0a,0x7e]
-0xfd,0x2c,0x0a,0x7e
-
-# GFX12: v_cvt_f64_u32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x2c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x2c,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, v1          ; encoding: [0x01,0x1b,0x0a,0x7e]
-0x01,0x1b,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, v255        ; encoding: [0xff,0x1b,0x0a,0x7e]
-0xff,0x1b,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, s1          ; encoding: [0x01,0x1a,0x0a,0x7e]
-0x01,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, s105        ; encoding: [0x69,0x1a,0x0a,0x7e]
-0x69,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x1a,0x0a,0x7e]
-0x6a,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x1a,0x0a,0x7e]
-0x6b,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x1a,0x0a,0x7e]
-0x7b,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x1a,0x0a,0x7e]
-0x7d,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x1a,0x0a,0x7e]
-0x7e,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x1a,0x0a,0x7e]
-0x7f,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, null        ; encoding: [0x7c,0x1a,0x0a,0x7e]
-0x7c,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x1a,0x0a,0x7e]
-0xc1,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x1a,0x0a,0x7e]
-0xf0,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x1a,0x0a,0x7e]
-0xfd,0x1a,0x0a,0x7e
-
-# GFX12: v_cvt_floor_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x1a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x1a,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_i16_f16_e32 v5, v1                ; encoding: [0x01,0xa7,0x0a,0x7e]
-0x01,0xa7,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, v127              ; encoding: [0x7f,0xa7,0x0a,0x7e]
-0x7f,0xa7,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, s1                ; encoding: [0x01,0xa6,0x0a,0x7e]
-0x01,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, s105              ; encoding: [0x69,0xa6,0x0a,0x7e]
-0x69,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa6,0x0a,0x7e]
-0x6a,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa6,0x0a,0x7e]
-0x6b,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, ttmp15            ; encoding: [0x7b,0xa6,0x0a,0x7e]
-0x7b,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, m0                ; encoding: [0x7d,0xa6,0x0a,0x7e]
-0x7d,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, exec_lo           ; encoding: [0x7e,0xa6,0x0a,0x7e]
-0x7e,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, exec_hi           ; encoding: [0x7f,0xa6,0x0a,0x7e]
-0x7f,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, null              ; encoding: [0x7c,0xa6,0x0a,0x7e]
-0x7c,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, -1                ; encoding: [0xc1,0xa6,0x0a,0x7e]
-0xc1,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, 0.5               ; encoding: [0xf0,0xa6,0x0a,0x7e]
-0xf0,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v5, src_scc           ; encoding: [0xfd,0xa6,0x0a,0x7e]
-0xfd,0xa6,0x0a,0x7e
-
-# GFX12: v_cvt_i16_f16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xa6,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_cvt_i32_f32_e32 v5, v1                ; encoding: [0x01,0x11,0x0a,0x7e]
-0x01,0x11,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, v255              ; encoding: [0xff,0x11,0x0a,0x7e]
-0xff,0x11,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, s1                ; encoding: [0x01,0x10,0x0a,0x7e]
-0x01,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, s105              ; encoding: [0x69,0x10,0x0a,0x7e]
-0x69,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x10,0x0a,0x7e]
-0x6a,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x10,0x0a,0x7e]
-0x6b,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x10,0x0a,0x7e]
-0x7b,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, m0                ; encoding: [0x7d,0x10,0x0a,0x7e]
-0x7d,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x10,0x0a,0x7e]
-0x7e,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x10,0x0a,0x7e]
-0x7f,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, null              ; encoding: [0x7c,0x10,0x0a,0x7e]
-0x7c,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, -1                ; encoding: [0xc1,0x10,0x0a,0x7e]
-0xc1,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, 0.5               ; encoding: [0xf0,0x10,0x0a,0x7e]
-0xf0,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v5, src_scc           ; encoding: [0xfd,0x10,0x0a,0x7e]
-0xfd,0x10,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f32_e32 v255, 0xaf123456      ; encoding: [0xff,0x10,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x10,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_i32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x07,0x0a,0x7e]
-0x01,0x07,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x07,0x0a,0x7e]
-0xfe,0x07,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x06,0x0a,0x7e]
-0x02,0x06,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x06,0x0a,0x7e]
-0x68,0x06,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v5, vcc               ; encoding: [0x6a,0x06,0x0a,0x7e]
-0x6a,0x06,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x06,0x0a,0x7e]
-0x7a,0x06,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v5, exec              ; encoding: [0x7e,0x06,0x0a,0x7e]
-0x7e,0x06,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v5, null              ; encoding: [0x7c,0x06,0x0a,0x7e]
-0x7c,0x06,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v5, -1                ; encoding: [0xc1,0x06,0x0a,0x7e]
-0xc1,0x06,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x06,0x0a,0x7e]
-0xf0,0x06,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x06,0x0a,0x7e]
-0xfd,0x06,0x0a,0x7e
-
-# GFX12: v_cvt_i32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x06,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x06,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_i32_i16_e32 v5, v1                ; encoding: [0x01,0xd5,0x0a,0x7e]
-0x01,0xd5,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, v127              ; encoding: [0x7f,0xd5,0x0a,0x7e]
-0x7f,0xd5,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, s1                ; encoding: [0x01,0xd4,0x0a,0x7e]
-0x01,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, s105              ; encoding: [0x69,0xd4,0x0a,0x7e]
-0x69,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, vcc_lo            ; encoding: [0x6a,0xd4,0x0a,0x7e]
-0x6a,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, vcc_hi            ; encoding: [0x6b,0xd4,0x0a,0x7e]
-0x6b,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, ttmp15            ; encoding: [0x7b,0xd4,0x0a,0x7e]
-0x7b,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, m0                ; encoding: [0x7d,0xd4,0x0a,0x7e]
-0x7d,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, exec_lo           ; encoding: [0x7e,0xd4,0x0a,0x7e]
-0x7e,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, exec_hi           ; encoding: [0x7f,0xd4,0x0a,0x7e]
-0x7f,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, null              ; encoding: [0x7c,0xd4,0x0a,0x7e]
-0x7c,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, -1                ; encoding: [0xc1,0xd4,0x0a,0x7e]
-0xc1,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, 0x3800
-0xf0,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v5, src_scc           ; encoding: [0xfd,0xd4,0x0a,0x7e]
-0xfd,0xd4,0x0a,0x7e
-
-# GFX12: v_cvt_i32_i16_e32 v255, 0xfe0b          ; encoding: [0xff,0xd4,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
-0xff,0xd4,0xfe,0x7f,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, v1        ; encoding: [0x01,0x19,0x0a,0x7e]
-0x01,0x19,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, v255      ; encoding: [0xff,0x19,0x0a,0x7e]
-0xff,0x19,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, s1        ; encoding: [0x01,0x18,0x0a,0x7e]
-0x01,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, s105      ; encoding: [0x69,0x18,0x0a,0x7e]
-0x69,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo    ; encoding: [0x6a,0x18,0x0a,0x7e]
-0x6a,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi    ; encoding: [0x6b,0x18,0x0a,0x7e]
-0x6b,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15    ; encoding: [0x7b,0x18,0x0a,0x7e]
-0x7b,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, m0        ; encoding: [0x7d,0x18,0x0a,0x7e]
-0x7d,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo   ; encoding: [0x7e,0x18,0x0a,0x7e]
-0x7e,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi   ; encoding: [0x7f,0x18,0x0a,0x7e]
-0x7f,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, null      ; encoding: [0x7c,0x18,0x0a,0x7e]
-0x7c,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, -1        ; encoding: [0xc1,0x18,0x0a,0x7e]
-0xc1,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5       ; encoding: [0xf0,0x18,0x0a,0x7e]
-0xf0,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc   ; encoding: [0xfd,0x18,0x0a,0x7e]
-0xfd,0x18,0x0a,0x7e
-
-# GFX12: v_cvt_nearest_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x18,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x18,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, v1           ; encoding: [0x01,0xc7,0x0a,0x7e]
-0x01,0xc7,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, v127         ; encoding: [0x7f,0xc7,0x0a,0x7e]
-0x7f,0xc7,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, s1           ; encoding: [0x01,0xc6,0x0a,0x7e]
-0x01,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, s105         ; encoding: [0x69,0xc6,0x0a,0x7e]
-0x69,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_lo       ; encoding: [0x6a,0xc6,0x0a,0x7e]
-0x6a,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_hi       ; encoding: [0x6b,0xc6,0x0a,0x7e]
-0x6b,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, ttmp15       ; encoding: [0x7b,0xc6,0x0a,0x7e]
-0x7b,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, m0           ; encoding: [0x7d,0xc6,0x0a,0x7e]
-0x7d,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, exec_lo      ; encoding: [0x7e,0xc6,0x0a,0x7e]
-0x7e,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, exec_hi      ; encoding: [0x7f,0xc6,0x0a,0x7e]
-0x7f,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, null         ; encoding: [0x7c,0xc6,0x0a,0x7e]
-0x7c,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, -1           ; encoding: [0xc1,0xc6,0x0a,0x7e]
-0xc1,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, 0.5          ; encoding: [0xf0,0xc6,0x0a,0x7e]
-0xf0,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v5, src_scc      ; encoding: [0xfd,0xc6,0x0a,0x7e]
-0xfd,0xc6,0x0a,0x7e
-
-# GFX12: v_cvt_norm_i16_f16_e32 v127, 0xfe0b     ; encoding: [0xff,0xc6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xc6,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, v1           ; encoding: [0x01,0xc9,0x0a,0x7e]
-0x01,0xc9,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, v127         ; encoding: [0x7f,0xc9,0x0a,0x7e]
-0x7f,0xc9,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, s1           ; encoding: [0x01,0xc8,0x0a,0x7e]
-0x01,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, s105         ; encoding: [0x69,0xc8,0x0a,0x7e]
-0x69,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_lo       ; encoding: [0x6a,0xc8,0x0a,0x7e]
-0x6a,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_hi       ; encoding: [0x6b,0xc8,0x0a,0x7e]
-0x6b,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, ttmp15       ; encoding: [0x7b,0xc8,0x0a,0x7e]
-0x7b,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, m0           ; encoding: [0x7d,0xc8,0x0a,0x7e]
-0x7d,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, exec_lo      ; encoding: [0x7e,0xc8,0x0a,0x7e]
-0x7e,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, exec_hi      ; encoding: [0x7f,0xc8,0x0a,0x7e]
-0x7f,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, null         ; encoding: [0x7c,0xc8,0x0a,0x7e]
-0x7c,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, -1           ; encoding: [0xc1,0xc8,0x0a,0x7e]
-0xc1,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, 0.5          ; encoding: [0xf0,0xc8,0x0a,0x7e]
-0xf0,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v5, src_scc      ; encoding: [0xfd,0xc8,0x0a,0x7e]
-0xfd,0xc8,0x0a,0x7e
-
-# GFX12: v_cvt_norm_u16_f16_e32 v127, 0xfe0b     ; encoding: [0xff,0xc8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xc8,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, v1             ; encoding: [0x01,0x1d,0x0a,0x7e]
-0x01,0x1d,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, v255           ; encoding: [0xff,0x1d,0x0a,0x7e]
-0xff,0x1d,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, s1             ; encoding: [0x01,0x1c,0x0a,0x7e]
-0x01,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, s105           ; encoding: [0x69,0x1c,0x0a,0x7e]
-0x69,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, vcc_lo         ; encoding: [0x6a,0x1c,0x0a,0x7e]
-0x6a,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, vcc_hi         ; encoding: [0x6b,0x1c,0x0a,0x7e]
-0x6b,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, ttmp15         ; encoding: [0x7b,0x1c,0x0a,0x7e]
-0x7b,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, m0             ; encoding: [0x7d,0x1c,0x0a,0x7e]
-0x7d,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, exec_lo        ; encoding: [0x7e,0x1c,0x0a,0x7e]
-0x7e,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, exec_hi        ; encoding: [0x7f,0x1c,0x0a,0x7e]
-0x7f,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, null           ; encoding: [0x7c,0x1c,0x0a,0x7e]
-0x7c,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, -1             ; encoding: [0xc1,0x1c,0x0a,0x7e]
-0xc1,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, 0.5            ; encoding: [0xf0,0x1c,0x0a,0x7e]
-0xf0,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v5, src_scc        ; encoding: [0xfd,0x1c,0x0a,0x7e]
-0xfd,0x1c,0x0a,0x7e
-
-# GFX12: v_cvt_off_f32_i4_e32 v255, 0x4f         ; encoding: [0xff,0x1c,0xfe,0x7f,0x4f,0x00,0x00,0x00]
-0xff,0x1c,0xfe,0x7f,0x4f,0x00,0x00,0x00
-
-# GFX12: v_cvt_u16_f16_e32 v5, v1                ; encoding: [0x01,0xa5,0x0a,0x7e]
-0x01,0xa5,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, v127              ; encoding: [0x7f,0xa5,0x0a,0x7e]
-0x7f,0xa5,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, s1                ; encoding: [0x01,0xa4,0x0a,0x7e]
-0x01,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, s105              ; encoding: [0x69,0xa4,0x0a,0x7e]
-0x69,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa4,0x0a,0x7e]
-0x6a,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa4,0x0a,0x7e]
-0x6b,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, ttmp15            ; encoding: [0x7b,0xa4,0x0a,0x7e]
-0x7b,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, m0                ; encoding: [0x7d,0xa4,0x0a,0x7e]
-0x7d,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, exec_lo           ; encoding: [0x7e,0xa4,0x0a,0x7e]
-0x7e,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, exec_hi           ; encoding: [0x7f,0xa4,0x0a,0x7e]
-0x7f,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, null              ; encoding: [0x7c,0xa4,0x0a,0x7e]
-0x7c,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, -1                ; encoding: [0xc1,0xa4,0x0a,0x7e]
-0xc1,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, 0.5               ; encoding: [0xf0,0xa4,0x0a,0x7e]
-0xf0,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v5, src_scc           ; encoding: [0xfd,0xa4,0x0a,0x7e]
-0xfd,0xa4,0x0a,0x7e
-
-# GFX12: v_cvt_u16_f16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xa4,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_cvt_u32_f32_e32 v5, v1                ; encoding: [0x01,0x0f,0x0a,0x7e]
-0x01,0x0f,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, v255              ; encoding: [0xff,0x0f,0x0a,0x7e]
-0xff,0x0f,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, s1                ; encoding: [0x01,0x0e,0x0a,0x7e]
-0x01,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, s105              ; encoding: [0x69,0x0e,0x0a,0x7e]
-0x69,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0e,0x0a,0x7e]
-0x6a,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0e,0x0a,0x7e]
-0x6b,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x0e,0x0a,0x7e]
-0x7b,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, m0                ; encoding: [0x7d,0x0e,0x0a,0x7e]
-0x7d,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x0e,0x0a,0x7e]
-0x7e,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x0e,0x0a,0x7e]
-0x7f,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, null              ; encoding: [0x7c,0x0e,0x0a,0x7e]
-0x7c,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, -1                ; encoding: [0xc1,0x0e,0x0a,0x7e]
-0xc1,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, 0.5               ; encoding: [0xf0,0x0e,0x0a,0x7e]
-0xf0,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v5, src_scc           ; encoding: [0xfd,0x0e,0x0a,0x7e]
-0xfd,0x0e,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x0e,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_u32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x2b,0x0a,0x7e]
-0x01,0x2b,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x2b,0x0a,0x7e]
-0xfe,0x2b,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x2a,0x0a,0x7e]
-0x02,0x2a,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x2a,0x0a,0x7e]
-0x68,0x2a,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v5, vcc               ; encoding: [0x6a,0x2a,0x0a,0x7e]
-0x6a,0x2a,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x2a,0x0a,0x7e]
-0x7a,0x2a,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v5, exec              ; encoding: [0x7e,0x2a,0x0a,0x7e]
-0x7e,0x2a,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v5, null              ; encoding: [0x7c,0x2a,0x0a,0x7e]
-0x7c,0x2a,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v5, -1                ; encoding: [0xc1,0x2a,0x0a,0x7e]
-0xc1,0x2a,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x2a,0x0a,0x7e]
-0xf0,0x2a,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x2a,0x0a,0x7e]
-0xfd,0x2a,0x0a,0x7e
-
-# GFX12: v_cvt_u32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x2a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x2a,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_cvt_u32_u16_e32 v5, v1                ; encoding: [0x01,0xd7,0x0a,0x7e]
-0x01,0xd7,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, v127              ; encoding: [0x7f,0xd7,0x0a,0x7e]
-0x7f,0xd7,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, s1                ; encoding: [0x01,0xd6,0x0a,0x7e]
-0x01,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, s105              ; encoding: [0x69,0xd6,0x0a,0x7e]
-0x69,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, vcc_lo            ; encoding: [0x6a,0xd6,0x0a,0x7e]
-0x6a,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, vcc_hi            ; encoding: [0x6b,0xd6,0x0a,0x7e]
-0x6b,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, ttmp15            ; encoding: [0x7b,0xd6,0x0a,0x7e]
-0x7b,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, m0                ; encoding: [0x7d,0xd6,0x0a,0x7e]
-0x7d,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, exec_lo           ; encoding: [0x7e,0xd6,0x0a,0x7e]
-0x7e,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, exec_hi           ; encoding: [0x7f,0xd6,0x0a,0x7e]
-0x7f,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, null              ; encoding: [0x7c,0xd6,0x0a,0x7e]
-0x7c,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, -1                ; encoding: [0xc1,0xd6,0x0a,0x7e]
-0xc1,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, 0x3800
-0xf0,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v5, src_scc           ; encoding: [0xfd,0xd6,0x0a,0x7e]
-0xfd,0xd6,0x0a,0x7e
-
-# GFX12: v_cvt_u32_u16_e32 v255, 0xfe0b          ; encoding: [0xff,0xd6,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
-0xff,0xd6,0xfe,0x7f,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_exp_f16_e32 v5, v1                    ; encoding: [0x01,0xb1,0x0a,0x7e]
-0x01,0xb1,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, v127                  ; encoding: [0x7f,0xb1,0x0a,0x7e]
-0x7f,0xb1,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, s1                    ; encoding: [0x01,0xb0,0x0a,0x7e]
-0x01,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, s105                  ; encoding: [0x69,0xb0,0x0a,0x7e]
-0x69,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xb0,0x0a,0x7e]
-0x6a,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xb0,0x0a,0x7e]
-0x6b,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xb0,0x0a,0x7e]
-0x7b,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, m0                    ; encoding: [0x7d,0xb0,0x0a,0x7e]
-0x7d,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xb0,0x0a,0x7e]
-0x7e,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xb0,0x0a,0x7e]
-0x7f,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, null                  ; encoding: [0x7c,0xb0,0x0a,0x7e]
-0x7c,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, -1                    ; encoding: [0xc1,0xb0,0x0a,0x7e]
-0xc1,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xb0,0x0a,0x7e]
-0xf0,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v5, src_scc               ; encoding: [0xfd,0xb0,0x0a,0x7e]
-0xfd,0xb0,0x0a,0x7e
-
-# GFX12: v_exp_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xb0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xb0,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_exp_f32_e32 v5, v1                    ; encoding: [0x01,0x4b,0x0a,0x7e]
-0x01,0x4b,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, v255                  ; encoding: [0xff,0x4b,0x0a,0x7e]
-0xff,0x4b,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, s1                    ; encoding: [0x01,0x4a,0x0a,0x7e]
-0x01,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, s105                  ; encoding: [0x69,0x4a,0x0a,0x7e]
-0x69,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x4a,0x0a,0x7e]
-0x6a,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x4a,0x0a,0x7e]
-0x6b,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x4a,0x0a,0x7e]
-0x7b,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, m0                    ; encoding: [0x7d,0x4a,0x0a,0x7e]
-0x7d,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x4a,0x0a,0x7e]
-0x7e,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x4a,0x0a,0x7e]
-0x7f,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, null                  ; encoding: [0x7c,0x4a,0x0a,0x7e]
-0x7c,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, -1                    ; encoding: [0xc1,0x4a,0x0a,0x7e]
-0xc1,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x4a,0x0a,0x7e]
-0xf0,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v5, src_scc               ; encoding: [0xfd,0x4a,0x0a,0x7e]
-0xfd,0x4a,0x0a,0x7e
-
-# GFX12: v_exp_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x4a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x4a,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_floor_f16_e32 v5, v1                  ; encoding: [0x01,0xb7,0x0a,0x7e]
-0x01,0xb7,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, v127                ; encoding: [0x7f,0xb7,0x0a,0x7e]
-0x7f,0xb7,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, s1                  ; encoding: [0x01,0xb6,0x0a,0x7e]
-0x01,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, s105                ; encoding: [0x69,0xb6,0x0a,0x7e]
-0x69,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xb6,0x0a,0x7e]
-0x6a,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xb6,0x0a,0x7e]
-0x6b,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xb6,0x0a,0x7e]
-0x7b,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, m0                  ; encoding: [0x7d,0xb6,0x0a,0x7e]
-0x7d,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xb6,0x0a,0x7e]
-0x7e,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xb6,0x0a,0x7e]
-0x7f,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, null                ; encoding: [0x7c,0xb6,0x0a,0x7e]
-0x7c,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, -1                  ; encoding: [0xc1,0xb6,0x0a,0x7e]
-0xc1,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xb6,0x0a,0x7e]
-0xf0,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v5, src_scc             ; encoding: [0xfd,0xb6,0x0a,0x7e]
-0xfd,0xb6,0x0a,0x7e
-
-# GFX12: v_floor_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xb6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xb6,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_floor_f32_e32 v5, v1                  ; encoding: [0x01,0x49,0x0a,0x7e]
-0x01,0x49,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, v255                ; encoding: [0xff,0x49,0x0a,0x7e]
-0xff,0x49,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, s1                  ; encoding: [0x01,0x48,0x0a,0x7e]
-0x01,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, s105                ; encoding: [0x69,0x48,0x0a,0x7e]
-0x69,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x48,0x0a,0x7e]
-0x6a,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x48,0x0a,0x7e]
-0x6b,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x48,0x0a,0x7e]
-0x7b,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, m0                  ; encoding: [0x7d,0x48,0x0a,0x7e]
-0x7d,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x48,0x0a,0x7e]
-0x7e,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x48,0x0a,0x7e]
-0x7f,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, null                ; encoding: [0x7c,0x48,0x0a,0x7e]
-0x7c,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, -1                  ; encoding: [0xc1,0x48,0x0a,0x7e]
-0xc1,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x48,0x0a,0x7e]
-0xf0,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v5, src_scc             ; encoding: [0xfd,0x48,0x0a,0x7e]
-0xfd,0x48,0x0a,0x7e
-
-# GFX12: v_floor_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x48,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x48,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_floor_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x35,0x0a,0x7e]
-0x01,0x35,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x35,0x0a,0x7e]
-0xfe,0x35,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x34,0x0a,0x7e]
-0x02,0x34,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x34,0x0a,0x7e]
-0x68,0x34,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x34,0x0a,0x7e]
-0x6a,0x34,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x34,0x0a,0x7e]
-0x7a,0x34,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x34,0x0a,0x7e]
-0x7e,0x34,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[5:6], null            ; encoding: [0x7c,0x34,0x0a,0x7e]
-0x7c,0x34,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x34,0x0a,0x7e]
-0xc1,0x34,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x34,0x0a,0x7e]
-0xf0,0x34,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x34,0x0a,0x7e]
-0xfd,0x34,0x0a,0x7e
-
-# GFX12: v_floor_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x34,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x34,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_fract_f16_e32 v5, v1                  ; encoding: [0x01,0xbf,0x0a,0x7e]
-0x01,0xbf,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, v127                ; encoding: [0x7f,0xbf,0x0a,0x7e]
-0x7f,0xbf,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, s1                  ; encoding: [0x01,0xbe,0x0a,0x7e]
-0x01,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, s105                ; encoding: [0x69,0xbe,0x0a,0x7e]
-0x69,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xbe,0x0a,0x7e]
-0x6a,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xbe,0x0a,0x7e]
-0x6b,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xbe,0x0a,0x7e]
-0x7b,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, m0                  ; encoding: [0x7d,0xbe,0x0a,0x7e]
-0x7d,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xbe,0x0a,0x7e]
-0x7e,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xbe,0x0a,0x7e]
-0x7f,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, null                ; encoding: [0x7c,0xbe,0x0a,0x7e]
-0x7c,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, -1                  ; encoding: [0xc1,0xbe,0x0a,0x7e]
-0xc1,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xbe,0x0a,0x7e]
-0xf0,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v5, src_scc             ; encoding: [0xfd,0xbe,0x0a,0x7e]
-0xfd,0xbe,0x0a,0x7e
-
-# GFX12: v_fract_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xbe,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xbe,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_fract_f32_e32 v5, v1                  ; encoding: [0x01,0x41,0x0a,0x7e]
-0x01,0x41,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, v255                ; encoding: [0xff,0x41,0x0a,0x7e]
-0xff,0x41,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, s1                  ; encoding: [0x01,0x40,0x0a,0x7e]
-0x01,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, s105                ; encoding: [0x69,0x40,0x0a,0x7e]
-0x69,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x40,0x0a,0x7e]
-0x6a,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x40,0x0a,0x7e]
-0x6b,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x40,0x0a,0x7e]
-0x7b,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, m0                  ; encoding: [0x7d,0x40,0x0a,0x7e]
-0x7d,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x40,0x0a,0x7e]
-0x7e,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x40,0x0a,0x7e]
-0x7f,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, null                ; encoding: [0x7c,0x40,0x0a,0x7e]
-0x7c,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, -1                  ; encoding: [0xc1,0x40,0x0a,0x7e]
-0xc1,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x40,0x0a,0x7e]
-0xf0,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v5, src_scc             ; encoding: [0xfd,0x40,0x0a,0x7e]
-0xfd,0x40,0x0a,0x7e
-
-# GFX12: v_fract_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x40,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x40,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_fract_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x7d,0x0a,0x7e]
-0x01,0x7d,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x7d,0x0a,0x7e]
-0xfe,0x7d,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x7c,0x0a,0x7e]
-0x02,0x7c,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x7c,0x0a,0x7e]
-0x68,0x7c,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x7c,0x0a,0x7e]
-0x6a,0x7c,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x7c,0x0a,0x7e]
-0x7a,0x7c,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x7c,0x0a,0x7e]
-0x7e,0x7c,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[5:6], null            ; encoding: [0x7c,0x7c,0x0a,0x7e]
-0x7c,0x7c,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x7c,0x0a,0x7e]
-0xc1,0x7c,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x7c,0x0a,0x7e]
-0xf0,0x7c,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x7c,0x0a,0x7e]
-0xfd,0x7c,0x0a,0x7e
-
-# GFX12: v_fract_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x7c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x7c,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, v1          ; encoding: [0x01,0xb5,0x0a,0x7e]
-0x01,0xb5,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, v127        ; encoding: [0x7f,0xb5,0x0a,0x7e]
-0x7f,0xb5,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, s1          ; encoding: [0x01,0xb4,0x0a,0x7e]
-0x01,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, s105        ; encoding: [0x69,0xb4,0x0a,0x7e]
-0x69,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_lo      ; encoding: [0x6a,0xb4,0x0a,0x7e]
-0x6a,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_hi      ; encoding: [0x6b,0xb4,0x0a,0x7e]
-0x6b,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, ttmp15      ; encoding: [0x7b,0xb4,0x0a,0x7e]
-0x7b,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, m0          ; encoding: [0x7d,0xb4,0x0a,0x7e]
-0x7d,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, exec_lo     ; encoding: [0x7e,0xb4,0x0a,0x7e]
-0x7e,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, exec_hi     ; encoding: [0x7f,0xb4,0x0a,0x7e]
-0x7f,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, null        ; encoding: [0x7c,0xb4,0x0a,0x7e]
-0x7c,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, -1          ; encoding: [0xc1,0xb4,0x0a,0x7e]
-0xc1,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, 0.5         ; encoding: [0xf0,0xb4,0x0a,0x7e]
-0xf0,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v5, src_scc     ; encoding: [0xfd,0xb4,0x0a,0x7e]
-0xfd,0xb4,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i16_f16_e32 v127, 0xfe0b    ; encoding: [0xff,0xb4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xb4,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, v1          ; encoding: [0x01,0x7f,0x0a,0x7e]
-0x01,0x7f,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, v255        ; encoding: [0xff,0x7f,0x0a,0x7e]
-0xff,0x7f,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, s1          ; encoding: [0x01,0x7e,0x0a,0x7e]
-0x01,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, s105        ; encoding: [0x69,0x7e,0x0a,0x7e]
-0x69,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x7e,0x0a,0x7e]
-0x6a,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x7e,0x0a,0x7e]
-0x6b,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x7e,0x0a,0x7e]
-0x7b,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x7e,0x0a,0x7e]
-0x7d,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x7e,0x0a,0x7e]
-0x7e,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x7e,0x0a,0x7e]
-0x7f,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, null        ; encoding: [0x7c,0x7e,0x0a,0x7e]
-0x7c,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x7e,0x0a,0x7e]
-0xc1,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x7e,0x0a,0x7e]
-0xf0,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x7e,0x0a,0x7e]
-0xfd,0x7e,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x7e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x7e,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, v[1:2]      ; encoding: [0x01,0x79,0x0a,0x7e]
-0x01,0x79,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, v[254:255]  ; encoding: [0xfe,0x79,0x0a,0x7e]
-0xfe,0x79,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, s[2:3]      ; encoding: [0x02,0x78,0x0a,0x7e]
-0x02,0x78,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, s[104:105]  ; encoding: [0x68,0x78,0x0a,0x7e]
-0x68,0x78,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, vcc         ; encoding: [0x6a,0x78,0x0a,0x7e]
-0x6a,0x78,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, ttmp[14:15] ; encoding: [0x7a,0x78,0x0a,0x7e]
-0x7a,0x78,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, exec        ; encoding: [0x7e,0x78,0x0a,0x7e]
-0x7e,0x78,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, null        ; encoding: [0x7c,0x78,0x0a,0x7e]
-0x7c,0x78,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, -1          ; encoding: [0xc1,0x78,0x0a,0x7e]
-0xc1,0x78,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, 0.5         ; encoding: [0xf0,0x78,0x0a,0x7e]
-0xf0,0x78,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v5, src_scc     ; encoding: [0xfd,0x78,0x0a,0x7e]
-0xfd,0x78,0x0a,0x7e
-
-# GFX12: v_frexp_exp_i32_f64_e32 v255, 0xaf123456 ; encoding: [0xff,0x78,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x78,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_frexp_mant_f16_e32 v5, v1             ; encoding: [0x01,0xb3,0x0a,0x7e]
-0x01,0xb3,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, v127           ; encoding: [0x7f,0xb3,0x0a,0x7e]
-0x7f,0xb3,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, s1             ; encoding: [0x01,0xb2,0x0a,0x7e]
-0x01,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, s105           ; encoding: [0x69,0xb2,0x0a,0x7e]
-0x69,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, vcc_lo         ; encoding: [0x6a,0xb2,0x0a,0x7e]
-0x6a,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, vcc_hi         ; encoding: [0x6b,0xb2,0x0a,0x7e]
-0x6b,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, ttmp15         ; encoding: [0x7b,0xb2,0x0a,0x7e]
-0x7b,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, m0             ; encoding: [0x7d,0xb2,0x0a,0x7e]
-0x7d,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, exec_lo        ; encoding: [0x7e,0xb2,0x0a,0x7e]
-0x7e,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, exec_hi        ; encoding: [0x7f,0xb2,0x0a,0x7e]
-0x7f,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, null           ; encoding: [0x7c,0xb2,0x0a,0x7e]
-0x7c,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, -1             ; encoding: [0xc1,0xb2,0x0a,0x7e]
-0xc1,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, 0.5            ; encoding: [0xf0,0xb2,0x0a,0x7e]
-0xf0,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v5, src_scc        ; encoding: [0xfd,0xb2,0x0a,0x7e]
-0xfd,0xb2,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f16_e32 v127, 0xfe0b       ; encoding: [0xff,0xb2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xb2,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_frexp_mant_f32_e32 v5, v1             ; encoding: [0x01,0x81,0x0a,0x7e]
-0x01,0x81,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, v255           ; encoding: [0xff,0x81,0x0a,0x7e]
-0xff,0x81,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, s1             ; encoding: [0x01,0x80,0x0a,0x7e]
-0x01,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, s105           ; encoding: [0x69,0x80,0x0a,0x7e]
-0x69,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, vcc_lo         ; encoding: [0x6a,0x80,0x0a,0x7e]
-0x6a,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, vcc_hi         ; encoding: [0x6b,0x80,0x0a,0x7e]
-0x6b,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, ttmp15         ; encoding: [0x7b,0x80,0x0a,0x7e]
-0x7b,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, m0             ; encoding: [0x7d,0x80,0x0a,0x7e]
-0x7d,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, exec_lo        ; encoding: [0x7e,0x80,0x0a,0x7e]
-0x7e,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, exec_hi        ; encoding: [0x7f,0x80,0x0a,0x7e]
-0x7f,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, null           ; encoding: [0x7c,0x80,0x0a,0x7e]
-0x7c,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, -1             ; encoding: [0xc1,0x80,0x0a,0x7e]
-0xc1,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, 0.5            ; encoding: [0xf0,0x80,0x0a,0x7e]
-0xf0,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v5, src_scc        ; encoding: [0xfd,0x80,0x0a,0x7e]
-0xfd,0x80,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f32_e32 v255, 0xaf123456   ; encoding: [0xff,0x80,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x80,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], v[1:2]     ; encoding: [0x01,0x7b,0x0a,0x7e]
-0x01,0x7b,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x7b,0x0a,0x7e]
-0xfe,0x7b,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], s[2:3]     ; encoding: [0x02,0x7a,0x0a,0x7e]
-0x02,0x7a,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x7a,0x0a,0x7e]
-0x68,0x7a,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], vcc        ; encoding: [0x6a,0x7a,0x0a,0x7e]
-0x6a,0x7a,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x7a,0x0a,0x7e]
-0x7a,0x7a,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], exec       ; encoding: [0x7e,0x7a,0x0a,0x7e]
-0x7e,0x7a,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], null       ; encoding: [0x7c,0x7a,0x0a,0x7e]
-0x7c,0x7a,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], -1         ; encoding: [0xc1,0x7a,0x0a,0x7e]
-0xc1,0x7a,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], 0.5        ; encoding: [0xf0,0x7a,0x0a,0x7e]
-0xf0,0x7a,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[5:6], src_scc    ; encoding: [0xfd,0x7a,0x0a,0x7e]
-0xfd,0x7a,0x0a,0x7e
-
-# GFX12: v_frexp_mant_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x7a,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x7a,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_log_f16_e32 v5, v1                    ; encoding: [0x01,0xaf,0x0a,0x7e]
-0x01,0xaf,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, v127                  ; encoding: [0x7f,0xaf,0x0a,0x7e]
-0x7f,0xaf,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, s1                    ; encoding: [0x01,0xae,0x0a,0x7e]
-0x01,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, s105                  ; encoding: [0x69,0xae,0x0a,0x7e]
-0x69,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xae,0x0a,0x7e]
-0x6a,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xae,0x0a,0x7e]
-0x6b,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xae,0x0a,0x7e]
-0x7b,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, m0                    ; encoding: [0x7d,0xae,0x0a,0x7e]
-0x7d,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xae,0x0a,0x7e]
-0x7e,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xae,0x0a,0x7e]
-0x7f,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, null                  ; encoding: [0x7c,0xae,0x0a,0x7e]
-0x7c,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, -1                    ; encoding: [0xc1,0xae,0x0a,0x7e]
-0xc1,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xae,0x0a,0x7e]
-0xf0,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v5, src_scc               ; encoding: [0xfd,0xae,0x0a,0x7e]
-0xfd,0xae,0x0a,0x7e
-
-# GFX12: v_log_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xae,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xae,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_log_f32_e32 v5, v1                    ; encoding: [0x01,0x4f,0x0a,0x7e]
-0x01,0x4f,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, v255                  ; encoding: [0xff,0x4f,0x0a,0x7e]
-0xff,0x4f,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, s1                    ; encoding: [0x01,0x4e,0x0a,0x7e]
-0x01,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, s105                  ; encoding: [0x69,0x4e,0x0a,0x7e]
-0x69,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x4e,0x0a,0x7e]
-0x6a,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x4e,0x0a,0x7e]
-0x6b,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x4e,0x0a,0x7e]
-0x7b,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, m0                    ; encoding: [0x7d,0x4e,0x0a,0x7e]
-0x7d,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x4e,0x0a,0x7e]
-0x7e,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x4e,0x0a,0x7e]
-0x7f,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, null                  ; encoding: [0x7c,0x4e,0x0a,0x7e]
-0x7c,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, -1                    ; encoding: [0xc1,0x4e,0x0a,0x7e]
-0xc1,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x4e,0x0a,0x7e]
-0xf0,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v5, src_scc               ; encoding: [0xfd,0x4e,0x0a,0x7e]
-0xfd,0x4e,0x0a,0x7e
-
-# GFX12: v_log_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x4e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x4e,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_mov_b32_e32 v5, v1                    ; encoding: [0x01,0x03,0x0a,0x7e]
-0x01,0x03,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, v255                  ; encoding: [0xff,0x03,0x0a,0x7e]
-0xff,0x03,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, s1                    ; encoding: [0x01,0x02,0x0a,0x7e]
-0x01,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, s105                  ; encoding: [0x69,0x02,0x0a,0x7e]
-0x69,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, vcc_lo                ; encoding: [0x6a,0x02,0x0a,0x7e]
-0x6a,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, vcc_hi                ; encoding: [0x6b,0x02,0x0a,0x7e]
-0x6b,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, ttmp15                ; encoding: [0x7b,0x02,0x0a,0x7e]
-0x7b,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, m0                    ; encoding: [0x7d,0x02,0x0a,0x7e]
-0x7d,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, exec_lo               ; encoding: [0x7e,0x02,0x0a,0x7e]
-0x7e,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, exec_hi               ; encoding: [0x7f,0x02,0x0a,0x7e]
-0x7f,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, null                  ; encoding: [0x7c,0x02,0x0a,0x7e]
-0x7c,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, -1                    ; encoding: [0xc1,0x02,0x0a,0x7e]
-0xc1,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, 0.5                   ; encoding: [0xf0,0x02,0x0a,0x7e]
-0xf0,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v5, src_scc               ; encoding: [0xfd,0x02,0x0a,0x7e]
-0xfd,0x02,0x0a,0x7e
-
-# GFX12: v_mov_b32_e32 v255, 0xaf123456          ; encoding: [0xff,0x02,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x02,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_movreld_b32_e32 v5, v1                ; encoding: [0x01,0x85,0x0a,0x7e]
-0x01,0x85,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, v255              ; encoding: [0xff,0x85,0x0a,0x7e]
-0xff,0x85,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, s1                ; encoding: [0x01,0x84,0x0a,0x7e]
-0x01,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, s105              ; encoding: [0x69,0x84,0x0a,0x7e]
-0x69,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x84,0x0a,0x7e]
-0x6a,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x84,0x0a,0x7e]
-0x6b,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x84,0x0a,0x7e]
-0x7b,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, m0                ; encoding: [0x7d,0x84,0x0a,0x7e]
-0x7d,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x84,0x0a,0x7e]
-0x7e,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x84,0x0a,0x7e]
-0x7f,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, null              ; encoding: [0x7c,0x84,0x0a,0x7e]
-0x7c,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, -1                ; encoding: [0xc1,0x84,0x0a,0x7e]
-0xc1,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, 0.5               ; encoding: [0xf0,0x84,0x0a,0x7e]
-0xf0,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v5, src_scc           ; encoding: [0xfd,0x84,0x0a,0x7e]
-0xfd,0x84,0x0a,0x7e
-
-# GFX12: v_movreld_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x84,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x84,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_movrels_b32_e32 v5, v1                ; encoding: [0x01,0x87,0x0a,0x7e]
-0x01,0x87,0x0a,0x7e
-
-# GFX12: v_movrels_b32_e32 v255, v255            ; encoding: [0xff,0x87,0xfe,0x7f]
-0xff,0x87,0xfe,0x7f
-
-# GFX12: v_movrelsd_2_b32_e32 v5, v1             ; encoding: [0x01,0x91,0x0a,0x7e]
-0x01,0x91,0x0a,0x7e
-
-# GFX12: v_movrelsd_2_b32_e32 v255, v255         ; encoding: [0xff,0x91,0xfe,0x7f]
-0xff,0x91,0xfe,0x7f
-
-# GFX12: v_movrelsd_b32_e32 v5, v1               ; encoding: [0x01,0x89,0x0a,0x7e]
-0x01,0x89,0x0a,0x7e
-
-# GFX12: v_movrelsd_b32_e32 v255, v255           ; encoding: [0xff,0x89,0xfe,0x7f]
-0xff,0x89,0xfe,0x7f
-
-# GFX12: v_nop                                   ; encoding: [0x00,0x00,0x00,0x7e]
-0x00,0x00,0x00,0x7e
-
-# GFX12: v_not_b16_e32 v5, v1                    ; encoding: [0x01,0xd3,0x0a,0x7e]
-0x01,0xd3,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, v127                  ; encoding: [0x7f,0xd3,0x0a,0x7e]
-0x7f,0xd3,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, s1                    ; encoding: [0x01,0xd2,0x0a,0x7e]
-0x01,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, s105                  ; encoding: [0x69,0xd2,0x0a,0x7e]
-0x69,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, vcc_lo                ; encoding: [0x6a,0xd2,0x0a,0x7e]
-0x6a,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, vcc_hi                ; encoding: [0x6b,0xd2,0x0a,0x7e]
-0x6b,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, ttmp15                ; encoding: [0x7b,0xd2,0x0a,0x7e]
-0x7b,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, m0                    ; encoding: [0x7d,0xd2,0x0a,0x7e]
-0x7d,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, exec_lo               ; encoding: [0x7e,0xd2,0x0a,0x7e]
-0x7e,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, exec_hi               ; encoding: [0x7f,0xd2,0x0a,0x7e]
-0x7f,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, null                  ; encoding: [0x7c,0xd2,0x0a,0x7e]
-0x7c,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, -1                    ; encoding: [0xc1,0xd2,0x0a,0x7e]
-0xc1,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, 0x3800
-0xf0,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v5, src_scc               ; encoding: [0xfd,0xd2,0x0a,0x7e]
-0xfd,0xd2,0x0a,0x7e
-
-# GFX12: v_not_b16_e32 v127, 0xfe0b              ; encoding: [0xff,0xd2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xd2,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_not_b32_e32 v5, v1                    ; encoding: [0x01,0x6f,0x0a,0x7e]
-0x01,0x6f,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, v255                  ; encoding: [0xff,0x6f,0x0a,0x7e]
-0xff,0x6f,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, s1                    ; encoding: [0x01,0x6e,0x0a,0x7e]
-0x01,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, s105                  ; encoding: [0x69,0x6e,0x0a,0x7e]
-0x69,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6e,0x0a,0x7e]
-0x6a,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6e,0x0a,0x7e]
-0x6b,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, ttmp15                ; encoding: [0x7b,0x6e,0x0a,0x7e]
-0x7b,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, m0                    ; encoding: [0x7d,0x6e,0x0a,0x7e]
-0x7d,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, exec_lo               ; encoding: [0x7e,0x6e,0x0a,0x7e]
-0x7e,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, exec_hi               ; encoding: [0x7f,0x6e,0x0a,0x7e]
-0x7f,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, null                  ; encoding: [0x7c,0x6e,0x0a,0x7e]
-0x7c,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, -1                    ; encoding: [0xc1,0x6e,0x0a,0x7e]
-0xc1,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, 0.5                   ; encoding: [0xf0,0x6e,0x0a,0x7e]
-0xf0,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v5, src_scc               ; encoding: [0xfd,0x6e,0x0a,0x7e]
-0xfd,0x6e,0x0a,0x7e
-
-# GFX12: v_not_b32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x6e,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_permlane64_b32 v5, v1                 ; encoding: [0x01,0xcf,0x0a,0x7e]
-0x01,0xcf,0x0a,0x7e
-
-# GFX12: v_permlane64_b32 v255, v255             ; encoding: [0xff,0xcf,0xfe,0x7f]
-0xff,0xcf,0xfe,0x7f
-
-# GFX12: v_pipeflush                             ; encoding: [0x00,0x36,0x00,0x7e]
-0x00,0x36,0x00,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, v1                    ; encoding: [0x01,0xa9,0x0a,0x7e]
-0x01,0xa9,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, v127                  ; encoding: [0x7f,0xa9,0x0a,0x7e]
-0x7f,0xa9,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, s1                    ; encoding: [0x01,0xa8,0x0a,0x7e]
-0x01,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, s105                  ; encoding: [0x69,0xa8,0x0a,0x7e]
-0x69,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xa8,0x0a,0x7e]
-0x6a,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xa8,0x0a,0x7e]
-0x6b,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xa8,0x0a,0x7e]
-0x7b,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, m0                    ; encoding: [0x7d,0xa8,0x0a,0x7e]
-0x7d,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xa8,0x0a,0x7e]
-0x7e,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xa8,0x0a,0x7e]
-0x7f,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, null                  ; encoding: [0x7c,0xa8,0x0a,0x7e]
-0x7c,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, -1                    ; encoding: [0xc1,0xa8,0x0a,0x7e]
-0xc1,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xa8,0x0a,0x7e]
-0xf0,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v5, src_scc               ; encoding: [0xfd,0xa8,0x0a,0x7e]
-0xfd,0xa8,0x0a,0x7e
-
-# GFX12: v_rcp_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xa8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xa8,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_rcp_f32_e32 v5, v1                    ; encoding: [0x01,0x55,0x0a,0x7e]
-0x01,0x55,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, v255                  ; encoding: [0xff,0x55,0x0a,0x7e]
-0xff,0x55,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, s1                    ; encoding: [0x01,0x54,0x0a,0x7e]
-0x01,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, s105                  ; encoding: [0x69,0x54,0x0a,0x7e]
-0x69,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x54,0x0a,0x7e]
-0x6a,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x54,0x0a,0x7e]
-0x6b,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x54,0x0a,0x7e]
-0x7b,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, m0                    ; encoding: [0x7d,0x54,0x0a,0x7e]
-0x7d,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x54,0x0a,0x7e]
-0x7e,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x54,0x0a,0x7e]
-0x7f,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, null                  ; encoding: [0x7c,0x54,0x0a,0x7e]
-0x7c,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, -1                    ; encoding: [0xc1,0x54,0x0a,0x7e]
-0xc1,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x54,0x0a,0x7e]
-0xf0,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v5, src_scc               ; encoding: [0xfd,0x54,0x0a,0x7e]
-0xfd,0x54,0x0a,0x7e
-
-# GFX12: v_rcp_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x54,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x54,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_rcp_f64_e32 v[5:6], v[1:2]            ; encoding: [0x01,0x5f,0x0a,0x7e]
-0x01,0x5f,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[5:6], v[254:255]        ; encoding: [0xfe,0x5f,0x0a,0x7e]
-0xfe,0x5f,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[5:6], s[2:3]            ; encoding: [0x02,0x5e,0x0a,0x7e]
-0x02,0x5e,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[5:6], s[104:105]        ; encoding: [0x68,0x5e,0x0a,0x7e]
-0x68,0x5e,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[5:6], vcc               ; encoding: [0x6a,0x5e,0x0a,0x7e]
-0x6a,0x5e,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[5:6], ttmp[14:15]       ; encoding: [0x7a,0x5e,0x0a,0x7e]
-0x7a,0x5e,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[5:6], exec              ; encoding: [0x7e,0x5e,0x0a,0x7e]
-0x7e,0x5e,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[5:6], null              ; encoding: [0x7c,0x5e,0x0a,0x7e]
-0x7c,0x5e,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[5:6], -1                ; encoding: [0xc1,0x5e,0x0a,0x7e]
-0xc1,0x5e,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[5:6], 0.5               ; encoding: [0xf0,0x5e,0x0a,0x7e]
-0xf0,0x5e,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[5:6], src_scc           ; encoding: [0xfd,0x5e,0x0a,0x7e]
-0xfd,0x5e,0x0a,0x7e
-
-# GFX12: v_rcp_f64_e32 v[254:255], 0xaf123456    ; encoding: [0xff,0x5e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x5e,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_rcp_iflag_f32_e32 v5, v1              ; encoding: [0x01,0x57,0x0a,0x7e]
-0x01,0x57,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, v255            ; encoding: [0xff,0x57,0x0a,0x7e]
-0xff,0x57,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, s1              ; encoding: [0x01,0x56,0x0a,0x7e]
-0x01,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, s105            ; encoding: [0x69,0x56,0x0a,0x7e]
-0x69,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, vcc_lo          ; encoding: [0x6a,0x56,0x0a,0x7e]
-0x6a,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, vcc_hi          ; encoding: [0x6b,0x56,0x0a,0x7e]
-0x6b,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, ttmp15          ; encoding: [0x7b,0x56,0x0a,0x7e]
-0x7b,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, m0              ; encoding: [0x7d,0x56,0x0a,0x7e]
-0x7d,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, exec_lo         ; encoding: [0x7e,0x56,0x0a,0x7e]
-0x7e,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, exec_hi         ; encoding: [0x7f,0x56,0x0a,0x7e]
-0x7f,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, null            ; encoding: [0x7c,0x56,0x0a,0x7e]
-0x7c,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, -1              ; encoding: [0xc1,0x56,0x0a,0x7e]
-0xc1,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, 0.5             ; encoding: [0xf0,0x56,0x0a,0x7e]
-0xf0,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v5, src_scc         ; encoding: [0xfd,0x56,0x0a,0x7e]
-0xfd,0x56,0x0a,0x7e
-
-# GFX12: v_rcp_iflag_f32_e32 v255, 0xaf123456    ; encoding: [0xff,0x56,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x56,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_rndne_f16_e32 v5, v1                  ; encoding: [0x01,0xbd,0x0a,0x7e]
-0x01,0xbd,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, v127                ; encoding: [0x7f,0xbd,0x0a,0x7e]
-0x7f,0xbd,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, s1                  ; encoding: [0x01,0xbc,0x0a,0x7e]
-0x01,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, s105                ; encoding: [0x69,0xbc,0x0a,0x7e]
-0x69,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xbc,0x0a,0x7e]
-0x6a,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xbc,0x0a,0x7e]
-0x6b,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xbc,0x0a,0x7e]
-0x7b,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, m0                  ; encoding: [0x7d,0xbc,0x0a,0x7e]
-0x7d,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xbc,0x0a,0x7e]
-0x7e,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xbc,0x0a,0x7e]
-0x7f,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, null                ; encoding: [0x7c,0xbc,0x0a,0x7e]
-0x7c,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, -1                  ; encoding: [0xc1,0xbc,0x0a,0x7e]
-0xc1,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xbc,0x0a,0x7e]
-0xf0,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v5, src_scc             ; encoding: [0xfd,0xbc,0x0a,0x7e]
-0xfd,0xbc,0x0a,0x7e
-
-# GFX12: v_rndne_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xbc,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xbc,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_rndne_f32_e32 v5, v1                  ; encoding: [0x01,0x47,0x0a,0x7e]
-0x01,0x47,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, v255                ; encoding: [0xff,0x47,0x0a,0x7e]
-0xff,0x47,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, s1                  ; encoding: [0x01,0x46,0x0a,0x7e]
-0x01,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, s105                ; encoding: [0x69,0x46,0x0a,0x7e]
-0x69,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x46,0x0a,0x7e]
-0x6a,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x46,0x0a,0x7e]
-0x6b,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x46,0x0a,0x7e]
-0x7b,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, m0                  ; encoding: [0x7d,0x46,0x0a,0x7e]
-0x7d,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x46,0x0a,0x7e]
-0x7e,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x46,0x0a,0x7e]
-0x7f,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, null                ; encoding: [0x7c,0x46,0x0a,0x7e]
-0x7c,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, -1                  ; encoding: [0xc1,0x46,0x0a,0x7e]
-0xc1,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x46,0x0a,0x7e]
-0xf0,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v5, src_scc             ; encoding: [0xfd,0x46,0x0a,0x7e]
-0xfd,0x46,0x0a,0x7e
-
-# GFX12: v_rndne_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x46,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x46,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_rndne_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x33,0x0a,0x7e]
-0x01,0x33,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x33,0x0a,0x7e]
-0xfe,0x33,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x32,0x0a,0x7e]
-0x02,0x32,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x32,0x0a,0x7e]
-0x68,0x32,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x32,0x0a,0x7e]
-0x6a,0x32,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x32,0x0a,0x7e]
-0x7a,0x32,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x32,0x0a,0x7e]
-0x7e,0x32,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[5:6], null            ; encoding: [0x7c,0x32,0x0a,0x7e]
-0x7c,0x32,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x32,0x0a,0x7e]
-0xc1,0x32,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x32,0x0a,0x7e]
-0xf0,0x32,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x32,0x0a,0x7e]
-0xfd,0x32,0x0a,0x7e
-
-# GFX12: v_rndne_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x32,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x32,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_rsq_f16_e32 v5, v1                    ; encoding: [0x01,0xad,0x0a,0x7e]
-0x01,0xad,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, v127                  ; encoding: [0x7f,0xad,0x0a,0x7e]
-0x7f,0xad,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, s1                    ; encoding: [0x01,0xac,0x0a,0x7e]
-0x01,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, s105                  ; encoding: [0x69,0xac,0x0a,0x7e]
-0x69,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xac,0x0a,0x7e]
-0x6a,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xac,0x0a,0x7e]
-0x6b,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xac,0x0a,0x7e]
-0x7b,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, m0                    ; encoding: [0x7d,0xac,0x0a,0x7e]
-0x7d,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xac,0x0a,0x7e]
-0x7e,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xac,0x0a,0x7e]
-0x7f,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, null                  ; encoding: [0x7c,0xac,0x0a,0x7e]
-0x7c,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, -1                    ; encoding: [0xc1,0xac,0x0a,0x7e]
-0xc1,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xac,0x0a,0x7e]
-0xf0,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v5, src_scc               ; encoding: [0xfd,0xac,0x0a,0x7e]
-0xfd,0xac,0x0a,0x7e
-
-# GFX12: v_rsq_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xac,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xac,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_rsq_f32_e32 v5, v1                    ; encoding: [0x01,0x5d,0x0a,0x7e]
-0x01,0x5d,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, v255                  ; encoding: [0xff,0x5d,0x0a,0x7e]
-0xff,0x5d,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, s1                    ; encoding: [0x01,0x5c,0x0a,0x7e]
-0x01,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, s105                  ; encoding: [0x69,0x5c,0x0a,0x7e]
-0x69,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x5c,0x0a,0x7e]
-0x6a,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x5c,0x0a,0x7e]
-0x6b,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x5c,0x0a,0x7e]
-0x7b,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, m0                    ; encoding: [0x7d,0x5c,0x0a,0x7e]
-0x7d,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x5c,0x0a,0x7e]
-0x7e,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x5c,0x0a,0x7e]
-0x7f,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, null                  ; encoding: [0x7c,0x5c,0x0a,0x7e]
-0x7c,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, -1                    ; encoding: [0xc1,0x5c,0x0a,0x7e]
-0xc1,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x5c,0x0a,0x7e]
-0xf0,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v5, src_scc               ; encoding: [0xfd,0x5c,0x0a,0x7e]
-0xfd,0x5c,0x0a,0x7e
-
-# GFX12: v_rsq_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x5c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x5c,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_rsq_f64_e32 v[5:6], v[1:2]            ; encoding: [0x01,0x63,0x0a,0x7e]
-0x01,0x63,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[5:6], v[254:255]        ; encoding: [0xfe,0x63,0x0a,0x7e]
-0xfe,0x63,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[5:6], s[2:3]            ; encoding: [0x02,0x62,0x0a,0x7e]
-0x02,0x62,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[5:6], s[104:105]        ; encoding: [0x68,0x62,0x0a,0x7e]
-0x68,0x62,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[5:6], vcc               ; encoding: [0x6a,0x62,0x0a,0x7e]
-0x6a,0x62,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[5:6], ttmp[14:15]       ; encoding: [0x7a,0x62,0x0a,0x7e]
-0x7a,0x62,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[5:6], exec              ; encoding: [0x7e,0x62,0x0a,0x7e]
-0x7e,0x62,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[5:6], null              ; encoding: [0x7c,0x62,0x0a,0x7e]
-0x7c,0x62,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[5:6], -1                ; encoding: [0xc1,0x62,0x0a,0x7e]
-0xc1,0x62,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[5:6], 0.5               ; encoding: [0xf0,0x62,0x0a,0x7e]
-0xf0,0x62,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[5:6], src_scc           ; encoding: [0xfd,0x62,0x0a,0x7e]
-0xfd,0x62,0x0a,0x7e
-
-# GFX12: v_rsq_f64_e32 v[254:255], 0xaf123456    ; encoding: [0xff,0x62,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x62,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, v1              ; encoding: [0x01,0xc5,0x0a,0x7e]
-0x01,0xc5,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, v255            ; encoding: [0xff,0xc5,0x0a,0x7e]
-0xff,0xc5,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, s1              ; encoding: [0x01,0xc4,0x0a,0x7e]
-0x01,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, s105            ; encoding: [0x69,0xc4,0x0a,0x7e]
-0x69,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, vcc_lo          ; encoding: [0x6a,0xc4,0x0a,0x7e]
-0x6a,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, vcc_hi          ; encoding: [0x6b,0xc4,0x0a,0x7e]
-0x6b,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, ttmp15          ; encoding: [0x7b,0xc4,0x0a,0x7e]
-0x7b,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, m0              ; encoding: [0x7d,0xc4,0x0a,0x7e]
-0x7d,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, exec_lo         ; encoding: [0x7e,0xc4,0x0a,0x7e]
-0x7e,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, exec_hi         ; encoding: [0x7f,0xc4,0x0a,0x7e]
-0x7f,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, null            ; encoding: [0x7c,0xc4,0x0a,0x7e]
-0x7c,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, -1              ; encoding: [0xc1,0xc4,0x0a,0x7e]
-0xc1,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, 0.5             ; encoding: [0xf0,0xc4,0x0a,0x7e]
-0xf0,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v5, src_scc         ; encoding: [0xfd,0xc4,0x0a,0x7e]
-0xfd,0xc4,0x0a,0x7e
-
-# GFX12: v_sat_pk_u8_i16_e32 v127, 0xfe0b        ; encoding: [0xff,0xc4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xc4,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_sin_f16_e32 v5, v1                    ; encoding: [0x01,0xc1,0x0a,0x7e]
-0x01,0xc1,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, v127                  ; encoding: [0x7f,0xc1,0x0a,0x7e]
-0x7f,0xc1,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, s1                    ; encoding: [0x01,0xc0,0x0a,0x7e]
-0x01,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, s105                  ; encoding: [0x69,0xc0,0x0a,0x7e]
-0x69,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xc0,0x0a,0x7e]
-0x6a,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xc0,0x0a,0x7e]
-0x6b,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xc0,0x0a,0x7e]
-0x7b,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, m0                    ; encoding: [0x7d,0xc0,0x0a,0x7e]
-0x7d,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xc0,0x0a,0x7e]
-0x7e,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xc0,0x0a,0x7e]
-0x7f,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, null                  ; encoding: [0x7c,0xc0,0x0a,0x7e]
-0x7c,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, -1                    ; encoding: [0xc1,0xc0,0x0a,0x7e]
-0xc1,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xc0,0x0a,0x7e]
-0xf0,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v5, src_scc               ; encoding: [0xfd,0xc0,0x0a,0x7e]
-0xfd,0xc0,0x0a,0x7e
-
-# GFX12: v_sin_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xc0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_sin_f32_e32 v5, v1                    ; encoding: [0x01,0x6b,0x0a,0x7e]
-0x01,0x6b,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, v255                  ; encoding: [0xff,0x6b,0x0a,0x7e]
-0xff,0x6b,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, s1                    ; encoding: [0x01,0x6a,0x0a,0x7e]
-0x01,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, s105                  ; encoding: [0x69,0x6a,0x0a,0x7e]
-0x69,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6a,0x0a,0x7e]
-0x6a,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6a,0x0a,0x7e]
-0x6b,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x6a,0x0a,0x7e]
-0x7b,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, m0                    ; encoding: [0x7d,0x6a,0x0a,0x7e]
-0x7d,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x6a,0x0a,0x7e]
-0x7e,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x6a,0x0a,0x7e]
-0x7f,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, null                  ; encoding: [0x7c,0x6a,0x0a,0x7e]
-0x7c,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, -1                    ; encoding: [0xc1,0x6a,0x0a,0x7e]
-0xc1,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x6a,0x0a,0x7e]
-0xf0,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v5, src_scc               ; encoding: [0xfd,0x6a,0x0a,0x7e]
-0xfd,0x6a,0x0a,0x7e
-
-# GFX12: v_sin_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x6a,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_sqrt_f16_e32 v5, v1                   ; encoding: [0x01,0xab,0x0a,0x7e]
-0x01,0xab,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, v127                 ; encoding: [0x7f,0xab,0x0a,0x7e]
-0x7f,0xab,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, s1                   ; encoding: [0x01,0xaa,0x0a,0x7e]
-0x01,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, s105                 ; encoding: [0x69,0xaa,0x0a,0x7e]
-0x69,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, vcc_lo               ; encoding: [0x6a,0xaa,0x0a,0x7e]
-0x6a,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, vcc_hi               ; encoding: [0x6b,0xaa,0x0a,0x7e]
-0x6b,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, ttmp15               ; encoding: [0x7b,0xaa,0x0a,0x7e]
-0x7b,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, m0                   ; encoding: [0x7d,0xaa,0x0a,0x7e]
-0x7d,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, exec_lo              ; encoding: [0x7e,0xaa,0x0a,0x7e]
-0x7e,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, exec_hi              ; encoding: [0x7f,0xaa,0x0a,0x7e]
-0x7f,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, null                 ; encoding: [0x7c,0xaa,0x0a,0x7e]
-0x7c,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, -1                   ; encoding: [0xc1,0xaa,0x0a,0x7e]
-0xc1,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, 0.5                  ; encoding: [0xf0,0xaa,0x0a,0x7e]
-0xf0,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v5, src_scc              ; encoding: [0xfd,0xaa,0x0a,0x7e]
-0xfd,0xaa,0x0a,0x7e
-
-# GFX12: v_sqrt_f16_e32 v127, 0xfe0b             ; encoding: [0xff,0xaa,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xaa,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_sqrt_f32_e32 v5, v1                   ; encoding: [0x01,0x67,0x0a,0x7e]
-0x01,0x67,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, v255                 ; encoding: [0xff,0x67,0x0a,0x7e]
-0xff,0x67,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, s1                   ; encoding: [0x01,0x66,0x0a,0x7e]
-0x01,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, s105                 ; encoding: [0x69,0x66,0x0a,0x7e]
-0x69,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, vcc_lo               ; encoding: [0x6a,0x66,0x0a,0x7e]
-0x6a,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, vcc_hi               ; encoding: [0x6b,0x66,0x0a,0x7e]
-0x6b,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, ttmp15               ; encoding: [0x7b,0x66,0x0a,0x7e]
-0x7b,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, m0                   ; encoding: [0x7d,0x66,0x0a,0x7e]
-0x7d,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, exec_lo              ; encoding: [0x7e,0x66,0x0a,0x7e]
-0x7e,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, exec_hi              ; encoding: [0x7f,0x66,0x0a,0x7e]
-0x7f,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, null                 ; encoding: [0x7c,0x66,0x0a,0x7e]
-0x7c,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, -1                   ; encoding: [0xc1,0x66,0x0a,0x7e]
-0xc1,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, 0.5                  ; encoding: [0xf0,0x66,0x0a,0x7e]
-0xf0,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v5, src_scc              ; encoding: [0xfd,0x66,0x0a,0x7e]
-0xfd,0x66,0x0a,0x7e
-
-# GFX12: v_sqrt_f32_e32 v255, 0xaf123456         ; encoding: [0xff,0x66,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x66,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_sqrt_f64_e32 v[5:6], v[1:2]           ; encoding: [0x01,0x69,0x0a,0x7e]
-0x01,0x69,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[5:6], v[254:255]       ; encoding: [0xfe,0x69,0x0a,0x7e]
-0xfe,0x69,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[5:6], s[2:3]           ; encoding: [0x02,0x68,0x0a,0x7e]
-0x02,0x68,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[5:6], s[104:105]       ; encoding: [0x68,0x68,0x0a,0x7e]
-0x68,0x68,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[5:6], vcc              ; encoding: [0x6a,0x68,0x0a,0x7e]
-0x6a,0x68,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[5:6], ttmp[14:15]      ; encoding: [0x7a,0x68,0x0a,0x7e]
-0x7a,0x68,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[5:6], exec             ; encoding: [0x7e,0x68,0x0a,0x7e]
-0x7e,0x68,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[5:6], null             ; encoding: [0x7c,0x68,0x0a,0x7e]
-0x7c,0x68,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[5:6], -1               ; encoding: [0xc1,0x68,0x0a,0x7e]
-0xc1,0x68,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[5:6], 0.5              ; encoding: [0xf0,0x68,0x0a,0x7e]
-0xf0,0x68,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[5:6], src_scc          ; encoding: [0xfd,0x68,0x0a,0x7e]
-0xfd,0x68,0x0a,0x7e
-
-# GFX12: v_sqrt_f64_e32 v[254:255], 0xaf123456   ; encoding: [0xff,0x68,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x68,0xfc,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_swap_b32 v5, v1                       ; encoding: [0x01,0xcb,0x0a,0x7e]
-0x01,0xcb,0x0a,0x7e
-
-# GFX12: v_swap_b32 v255, v255                   ; encoding: [0xff,0xcb,0xfe,0x7f]
-0xff,0xcb,0xfe,0x7f
-
-# GFX12: v_swaprel_b32 v5, v1                    ; encoding: [0x01,0xd1,0x0a,0x7e]
-0x01,0xd1,0x0a,0x7e
-
-# GFX12: v_swaprel_b32 v255, v255                ; encoding: [0xff,0xd1,0xfe,0x7f]
-0xff,0xd1,0xfe,0x7f
-
-# GFX12: v_trunc_f16_e32 v5, v1                  ; encoding: [0x01,0xbb,0x0a,0x7e]
-0x01,0xbb,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, v127                ; encoding: [0x7f,0xbb,0x0a,0x7e]
-0x7f,0xbb,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, s1                  ; encoding: [0x01,0xba,0x0a,0x7e]
-0x01,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, s105                ; encoding: [0x69,0xba,0x0a,0x7e]
-0x69,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xba,0x0a,0x7e]
-0x6a,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xba,0x0a,0x7e]
-0x6b,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xba,0x0a,0x7e]
-0x7b,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, m0                  ; encoding: [0x7d,0xba,0x0a,0x7e]
-0x7d,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xba,0x0a,0x7e]
-0x7e,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xba,0x0a,0x7e]
-0x7f,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, null                ; encoding: [0x7c,0xba,0x0a,0x7e]
-0x7c,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, -1                  ; encoding: [0xc1,0xba,0x0a,0x7e]
-0xc1,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xba,0x0a,0x7e]
-0xf0,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v5, src_scc             ; encoding: [0xfd,0xba,0x0a,0x7e]
-0xfd,0xba,0x0a,0x7e
-
-# GFX12: v_trunc_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xba,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
-0xff,0xba,0xfe,0x7e,0x0b,0xfe,0x00,0x00
-
-# GFX12: v_trunc_f32_e32 v5, v1                  ; encoding: [0x01,0x43,0x0a,0x7e]
-0x01,0x43,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, v255                ; encoding: [0xff,0x43,0x0a,0x7e]
-0xff,0x43,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, s1                  ; encoding: [0x01,0x42,0x0a,0x7e]
-0x01,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, s105                ; encoding: [0x69,0x42,0x0a,0x7e]
-0x69,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x42,0x0a,0x7e]
-0x6a,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x42,0x0a,0x7e]
-0x6b,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x42,0x0a,0x7e]
-0x7b,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, m0                  ; encoding: [0x7d,0x42,0x0a,0x7e]
-0x7d,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x42,0x0a,0x7e]
-0x7e,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x42,0x0a,0x7e]
-0x7f,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, null                ; encoding: [0x7c,0x42,0x0a,0x7e]
-0x7c,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, -1                  ; encoding: [0xc1,0x42,0x0a,0x7e]
-0xc1,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x42,0x0a,0x7e]
-0xf0,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v5, src_scc             ; encoding: [0xfd,0x42,0x0a,0x7e]
-0xfd,0x42,0x0a,0x7e
-
-# GFX12: v_trunc_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x42,0xfe,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x42,0xfe,0x7f,0x56,0x34,0x12,0xaf
-
-# GFX12: v_trunc_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x2f,0x0a,0x7e]
-0x01,0x2f,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x2f,0x0a,0x7e]
-0xfe,0x2f,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x2e,0x0a,0x7e]
-0x02,0x2e,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x2e,0x0a,0x7e]
-0x68,0x2e,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x2e,0x0a,0x7e]
-0x6a,0x2e,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x2e,0x0a,0x7e]
-0x7a,0x2e,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x2e,0x0a,0x7e]
-0x7e,0x2e,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[5:6], null            ; encoding: [0x7c,0x2e,0x0a,0x7e]
-0x7c,0x2e,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x2e,0x0a,0x7e]
-0xc1,0x2e,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x2e,0x0a,0x7e]
-0xf0,0x2e,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x2e,0x0a,0x7e]
-0xfd,0x2e,0x0a,0x7e
-
-# GFX12: v_trunc_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x2e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
-0xff,0x2e,0xfc,0x7f,0x56,0x34,0x12,0xaf

From 04760bfadb399cc4ded9b32bd523ec7703aa7462 Mon Sep 17 00:00:00 2001
From: nicole mazzuca <nicole@strega-nil.co>
Date: Fri, 19 Jul 2024 09:42:16 -0700
Subject: [PATCH 220/486] [libc++][ranges] P1223R5: `find_last` (#99312)

Implements [P1223R5][] completely.

Includes an implementation of `find_last`, `find_last_if`, and
`find_last_if_not`.

[P1223R5]: https://wg21.link/p1223r5
---
 libcxx/docs/FeatureTestMacroTable.rst         |   2 +
 libcxx/docs/ReleaseNotes/19.rst               |   1 +
 libcxx/docs/Status/Cxx23Papers.csv            |   2 +-
 libcxx/docs/Status/RangesAlgorithms.csv       |   6 +-
 libcxx/include/CMakeLists.txt                 |   1 +
 libcxx/include/__algorithm/ranges_find_last.h | 175 ++++++++++++
 libcxx/include/algorithm                      |  26 ++
 libcxx/include/module.modulemap               |   1 +
 libcxx/include/version                        |   2 +
 libcxx/modules/std/algorithm.inc              |   5 +-
 ...obust_against_copying_comparators.pass.cpp |  10 +
 ...obust_against_copying_projections.pass.cpp |  14 +
 .../algorithm.nodiscard.verify.cpp            |  12 +
 .../alg.find.last/ranges.find_last.pass.cpp   | 204 ++++++++++++++
 .../ranges.find_last_if.pass.cpp              | 252 +++++++++++++++++
 .../ranges.find_last_if_not.pass.cpp          | 254 ++++++++++++++++++
 .../ranges_robust_against_dangling.pass.cpp   |   5 +
 ...es_robust_against_nonbool.compile.pass.cpp |   4 +
 ...es_robust_against_proxy_iterators.pass.cpp |   5 +
 .../algorithm.version.compile.pass.cpp        |  31 +++
 .../version.version.compile.pass.cpp          |  31 +++
 .../niebloid.compile.pass.cpp                 |   3 +
 .../generate_feature_test_macro_components.py |   5 +
 23 files changed, 1045 insertions(+), 6 deletions(-)
 create mode 100644 libcxx/include/__algorithm/ranges_find_last.h
 create mode 100644 libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last.pass.cpp
 create mode 100644 libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last_if.pass.cpp
 create mode 100644 libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last_if_not.pass.cpp

diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index b548322b1b2b8..262da3f8937d2 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -360,6 +360,8 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_ranges_contains``                              ``202207L``
     ---------------------------------------------------------- -----------------
+    ``__cpp_lib_ranges_find_last``                             ``202207L``
+    ---------------------------------------------------------- -----------------
     ``__cpp_lib_ranges_iota``                                  *unimplemented*
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_ranges_join_with``                             *unimplemented*
diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 784934eafe9f7..43767552960e4 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -58,6 +58,7 @@ Implemented Papers
 - P2231R1 - Missing ``constexpr`` in ``std::optional`` and ``std::variant``
 - P0019R8 - ``std::atomic_ref``
 - P2389R2 - Alias template ``dims`` for the ``extents`` of ``mdspan``
+- P1223R5 - ``ranges::find_last()``, ``ranges::find_last_if()``, and ``ranges::find_last_if_not()``
 
 Improvements and New Features
 -----------------------------
diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index 6dd6bdad07658..f8970320ce1fa 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -55,7 +55,7 @@
 "`P0429R9 <https://wg21.link/P0429R9>`__","LWG","A Standard ``flat_map``","July 2022","",""
 "`P1169R4 <https://wg21.link/P1169R4>`__","LWG","``static operator()``","July 2022","|Complete|","16.0"
 "`P1222R4 <https://wg21.link/P1222R4>`__","LWG","A Standard ``flat_set``","July 2022","",""
-"`P1223R5 <https://wg21.link/P1223R5>`__","LWG","``ranges::find_last()``, ``ranges::find_last_if()``, and ``ranges::find_last_if_not()``","July 2022","","","|ranges|"
+"`P1223R5 <https://wg21.link/P1223R5>`__","LWG","``ranges::find_last()``, ``ranges::find_last_if()``, and ``ranges::find_last_if_not()``","July 2022","|Complete|","19.0","|ranges|"
 "`P1467R9 <https://wg21.link/P1467R9>`__","LWG","Extended ``floating-point`` types and standard names","July 2022","",""
 "`P1642R11 <https://wg21.link/P1642R11>`__","LWG","Freestanding ``[utilities]``, ``[ranges]``, and ``[iterators]``","July 2022","",""
 "`P1899R3 <https://wg21.link/P1899R3>`__","LWG","``stride_view``","July 2022","","","|ranges|"
diff --git a/libcxx/docs/Status/RangesAlgorithms.csv b/libcxx/docs/Status/RangesAlgorithms.csv
index f7a51f732c4b1..469ea21a76aab 100644
--- a/libcxx/docs/Status/RangesAlgorithms.csv
+++ b/libcxx/docs/Status/RangesAlgorithms.csv
@@ -1,8 +1,8 @@
 Standard,Algorithm,Assignee,CL,Status
 C++20,all C++20 algorithms,N/A,N/A,✅
-C++23,`find_last <https://wg21.link/P1223R5>`_,Unassigned,No patch yet,Not started
-C++23,`find_last_if <https://wg21.link/P1223R5>`_,Unassigned,No patch yet,Not started
-C++23,`find_last_if_not <https://wg21.link/P1223R5>`_,Unassigned,No patch yet,Not started
+C++23,`find_last <https://wg21.link/P1223R5>`_,Nicole Mazzuca,`#99312 <https://github.com/llvm/llvm-project/pull/99312>`_,Complete
+C++23,`find_last_if <https://wg21.link/P1223R5>`_,Nicole Mazzuca,`#99312 <https://github.com/llvm/llvm-project/pull/99312>`_,Complete
+C++23,`find_last_if_not <https://wg21.link/P1223R5>`_,Nicole Mazzuca,`#99312 <https://github.com/llvm/llvm-project/pull/99312>`_,Complete
 C++23,`starts_with <https://wg21.link/P1659R3>`_,Zijun Zhao,`D150735 <https://llvm.org/D150735>`_,Complete
 C++23,`ends_with <https://wg21.link/P1659R3>`_,Zijun Zhao, `D150831 <https://llvm.org/D150831>`_,Complete
 C++23,`shift_left <https://wg21.link/p2440r1>`_,Unassigned,No patch yet,Not started
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index fa6736dc11e66..1a4d9c7070f14 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -97,6 +97,7 @@ set(files
   __algorithm/ranges_find_first_of.h
   __algorithm/ranges_find_if.h
   __algorithm/ranges_find_if_not.h
+  __algorithm/ranges_find_last.h
   __algorithm/ranges_for_each.h
   __algorithm/ranges_for_each_n.h
   __algorithm/ranges_generate.h
diff --git a/libcxx/include/__algorithm/ranges_find_last.h b/libcxx/include/__algorithm/ranges_find_last.h
new file mode 100644
index 0000000000000..95f7e77b8ccbe
--- /dev/null
+++ b/libcxx/include/__algorithm/ranges_find_last.h
@@ -0,0 +1,175 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_LAST_H
+#define _LIBCPP___ALGORITHM_RANGES_FIND_LAST_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__iterator/next.h>
+#include <__iterator/prev.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/subrange.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Iter, class _Sent, class _Pred, class _Proj>
+_LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+__find_last_impl(_Iter __first, _Sent __last, _Pred __pred, _Proj& __proj) {
+  if (__first == __last) {
+    return subrange<_Iter>(__first, __first);
+  }
+
+  if constexpr (bidirectional_iterator<_Iter>) {
+    auto __last_it = ranges::next(__first, __last);
+    for (auto __it = ranges::prev(__last_it); __it != __first; --__it) {
+      if (__pred(std::invoke(__proj, *__it))) {
+        return subrange<_Iter>(std::move(__it), std::move(__last_it));
+      }
+    }
+    if (__pred(std::invoke(__proj, *__first))) {
+      return subrange<_Iter>(std::move(__first), std::move(__last_it));
+    }
+    return subrange<_Iter>(__last_it, __last_it);
+  } else {
+    bool __found = false;
+    _Iter __found_it;
+    for (; __first != __last; ++__first) {
+      if (__pred(std::invoke(__proj, *__first))) {
+        __found    = true;
+        __found_it = __first;
+      }
+    }
+
+    if (__found) {
+      return subrange<_Iter>(std::move(__found_it), std::move(__first));
+    } else {
+      return subrange<_Iter>(__first, __first);
+    }
+  }
+}
+
+namespace __find_last {
+struct __fn {
+  template <class _Type>
+  struct __op {
+    const _Type& __value;
+    template <class _Elem>
+    _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator()(_Elem&& __elem) const {
+      return std::forward<_Elem>(__elem) == __value;
+    }
+  };
+
+  template <forward_iterator _Iter, sentinel_for<_Iter> _Sent, class _Type, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<_Iter, _Proj>, const _Type*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Proj __proj = {}) {
+    return ranges::__find_last_impl(std::move(__first), std::move(__last), __op<_Type>{__value}, __proj);
+  }
+
+  template <forward_range _Range, class _Type, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, const _Type& __value, _Proj __proj = {}) {
+    return ranges::__find_last_impl(ranges::begin(__range), ranges::end(__range), __op<_Type>{__value}, __proj);
+  }
+};
+} // namespace __find_last
+
+namespace __find_last_if {
+struct __fn {
+  template <class _Pred>
+  struct __op {
+    _Pred& __pred;
+    template <class _Elem>
+    _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator()(_Elem&& __elem) const {
+      return std::invoke(__pred, std::forward<_Elem>(__elem));
+    }
+  };
+
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) {
+    return ranges::__find_last_impl(std::move(__first), std::move(__last), __op<_Pred>{__pred}, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) {
+    return ranges::__find_last_impl(ranges::begin(__range), ranges::end(__range), __op<_Pred>{__pred}, __proj);
+  }
+};
+} // namespace __find_last_if
+
+namespace __find_last_if_not {
+struct __fn {
+  template <class _Pred>
+  struct __op {
+    _Pred& __pred;
+    template <class _Elem>
+    _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator()(_Elem&& __elem) const {
+      return !std::invoke(__pred, std::forward<_Elem>(__elem));
+    }
+  };
+
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) {
+    return ranges::__find_last_impl(std::move(__first), std::move(__last), __op<_Pred>{__pred}, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) {
+    return ranges::__find_last_impl(ranges::begin(__range), ranges::end(__range), __op<_Pred>{__pred}, __proj);
+  }
+};
+} // namespace __find_last_if_not
+
+inline namespace __cpo {
+inline constexpr auto find_last        = __find_last::__fn{};
+inline constexpr auto find_last_if     = __find_last_if::__fn{};
+inline constexpr auto find_last_if_not = __find_last_if_not::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FIND_LAST_H
diff --git a/libcxx/include/algorithm b/libcxx/include/algorithm
index a522a60f1b551..698e6f5cb7ad1 100644
--- a/libcxx/include/algorithm
+++ b/libcxx/include/algorithm
@@ -102,6 +102,31 @@ namespace ranges {
     constexpr borrowed_iterator_t<R>
       find_if_not(R&& r, Pred pred, Proj proj = {});                                                      // since C++20
 
+  template<forward_iterator I, sentinel_for<I> S, class T, class Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<I, Proj>, const T*>
+    constexpr subrange<I> find_last(I first, S last, const T& value, Proj proj = {});                     // since C++23
+
+  template<forward_range R, class T, class Proj = identity>
+    requires
+      indirect_binary_predicate<ranges::equal_to, projected<iterator_t<R>, Proj>, const T*>
+    constexpr borrowed_subrange_t<R> find_last(R&& r, const T& value, Proj proj = {});                   // since C++23
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr subrange<I> find_last_if(I first, S last, Pred pred, Proj proj = {});                      // since C++23
+
+  template<forward_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr borrowed_subrange_t<R> find_last_if(R&& r, Pred pred, Proj proj = {});                     // since C++23
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr subrange<I> find_last_if_not(I first, S last, Pred pred, Proj proj = {});                  // since C++23
+
+  template<forward_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr borrowed_subrange_t<R> find_last_if_not(R&& r, Pred pred, Proj proj = {});                 // since C++23
+
   template<class T, class Proj = identity,
            indirect_strict_weak_order<projected<const T*, Proj>> Comp = ranges::less>
     constexpr const T& min(const T& a, const T& b, Comp comp = {}, Proj proj = {});                       // since C++20
@@ -1989,6 +2014,7 @@ template <class BidirectionalIterator, class Compare>
 #  include <__algorithm/fold.h>
 #  include <__algorithm/ranges_contains_subrange.h>
 #  include <__algorithm/ranges_ends_with.h>
+#  include <__algorithm/ranges_find_last.h>
 #  include <__algorithm/ranges_starts_with.h>
 #endif // _LIBCPP_STD_VER >= 23
 
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 43ab9c6987c84..3443fbc3347a3 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -764,6 +764,7 @@ module std_private_algorithm_ranges_find_end                             [system
 module std_private_algorithm_ranges_find_first_of                        [system] { header "__algorithm/ranges_find_first_of.h" }
 module std_private_algorithm_ranges_find_if                              [system] { header "__algorithm/ranges_find_if.h" }
 module std_private_algorithm_ranges_find_if_not                          [system] { header "__algorithm/ranges_find_if_not.h" }
+module std_private_algorithm_ranges_find_last                            [system] { header "__algorithm/ranges_find_last.h" }
 module std_private_algorithm_ranges_for_each                             [system] {
   header "__algorithm/ranges_for_each.h"
   export std_private_algorithm_in_fun_result
diff --git a/libcxx/include/version b/libcxx/include/version
index 68aa88a48fa8f..40548098a92d6 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -190,6 +190,7 @@ __cpp_lib_ranges_chunk                                  202202L <ranges>
 __cpp_lib_ranges_chunk_by                               202202L <ranges>
 __cpp_lib_ranges_concat                                 202403L <ranges>
 __cpp_lib_ranges_contains                               202207L <algorithm>
+__cpp_lib_ranges_find_last                              202207L <algorithm>
 __cpp_lib_ranges_iota                                   202202L <numeric>
 __cpp_lib_ranges_join_with                              202202L <ranges>
 __cpp_lib_ranges_repeat                                 202207L <ranges>
@@ -484,6 +485,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 // # define __cpp_lib_ranges_chunk                         202202L
 # define __cpp_lib_ranges_chunk_by                      202202L
 # define __cpp_lib_ranges_contains                      202207L
+# define __cpp_lib_ranges_find_last                     202207L
 // # define __cpp_lib_ranges_iota                          202202L
 // # define __cpp_lib_ranges_join_with                     202202L
 # define __cpp_lib_ranges_repeat                        202207L
diff --git a/libcxx/modules/std/algorithm.inc b/libcxx/modules/std/algorithm.inc
index e7796bfa26af8..3c2139cd64ee4 100644
--- a/libcxx/modules/std/algorithm.inc
+++ b/libcxx/modules/std/algorithm.inc
@@ -77,13 +77,14 @@ export namespace std {
     using std::ranges::find_if_not;
   } // namespace ranges
 
+#if _LIBCPP_STD_VER >= 23
+  // [alg.find.last], find last
   namespace ranges {
-#if 0
     using std::ranges::find_last;
     using std::ranges::find_last_if;
     using std::ranges::find_last_if_not;
-#endif
   } // namespace ranges
+#endif
 
   // [alg.find.end], find end
   using std::find_end;
diff --git a/libcxx/test/libcxx/algorithms/ranges_robust_against_copying_comparators.pass.cpp b/libcxx/test/libcxx/algorithms/ranges_robust_against_copying_comparators.pass.cpp
index 1a6c0d11460c5..6bac7deadfa6e 100644
--- a/libcxx/test/libcxx/algorithms/ranges_robust_against_copying_comparators.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/ranges_robust_against_copying_comparators.pass.cpp
@@ -119,6 +119,16 @@ constexpr bool all_the_algorithms()
     (void)std::ranges::find_if(a, UnaryTrue(&copies)); assert(copies == 0);
     (void)std::ranges::find_if_not(first, last, UnaryTrue(&copies)); assert(copies == 0);
     (void)std::ranges::find_if_not(a, UnaryTrue(&copies)); assert(copies == 0);
+#if TEST_STD_VER >= 23
+    (void)std::ranges::find_last_if(first, last, UnaryTrue(&copies));
+    assert(copies == 0);
+    (void)std::ranges::find_last_if(a, UnaryTrue(&copies));
+    assert(copies == 0);
+    (void)std::ranges::find_last_if_not(first, last, UnaryTrue(&copies));
+    assert(copies == 0);
+    (void)std::ranges::find_last_if_not(a, UnaryTrue(&copies));
+    assert(copies == 0);
+#endif
     (void)std::ranges::for_each(first, last, UnaryVoid(&copies)); assert(copies == 1); copies = 0;
     (void)std::ranges::for_each(a, UnaryVoid(&copies)); assert(copies == 1); copies = 0;
     (void)std::ranges::for_each_n(first, count, UnaryVoid(&copies)); assert(copies == 1); copies = 0;
diff --git a/libcxx/test/libcxx/algorithms/ranges_robust_against_copying_projections.pass.cpp b/libcxx/test/libcxx/algorithms/ranges_robust_against_copying_projections.pass.cpp
index 71823d9afc1a4..bc71150c9e2d8 100644
--- a/libcxx/test/libcxx/algorithms/ranges_robust_against_copying_projections.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/ranges_robust_against_copying_projections.pass.cpp
@@ -115,6 +115,20 @@ constexpr bool all_the_algorithms()
     (void)std::ranges::find_if(a, UnaryTrue(), Proj(&copies)); assert(copies == 0);
     (void)std::ranges::find_if_not(first, last, UnaryTrue(), Proj(&copies)); assert(copies == 0);
     (void)std::ranges::find_if_not(a, UnaryTrue(), Proj(&copies)); assert(copies == 0);
+#if TEST_STD_VER >= 23
+    (void)std::ranges::find_last(first, last, value, Proj(&copies));
+    assert(copies == 0);
+    (void)std::ranges::find_last(a, value, Proj(&copies));
+    assert(copies == 0);
+    (void)std::ranges::find_last_if(first, last, UnaryTrue(), Proj(&copies));
+    assert(copies == 0);
+    (void)std::ranges::find_last_if(a, UnaryTrue(), Proj(&copies));
+    assert(copies == 0);
+    (void)std::ranges::find_last_if_not(first, last, UnaryTrue(), Proj(&copies));
+    assert(copies == 0);
+    (void)std::ranges::find_last_if_not(a, UnaryTrue(), Proj(&copies));
+    assert(copies == 0);
+#endif
     (void)std::ranges::for_each(first, last, UnaryVoid(), Proj(&copies)); assert(copies == 0);
     (void)std::ranges::for_each(a, UnaryVoid(), Proj(&copies)); assert(copies == 0);
     (void)std::ranges::for_each_n(first, count, UnaryVoid(), Proj(&copies)); assert(copies == 0);
diff --git a/libcxx/test/libcxx/diagnostics/algorithm.nodiscard.verify.cpp b/libcxx/test/libcxx/diagnostics/algorithm.nodiscard.verify.cpp
index b36a426082ccb..14febc12a8a2d 100644
--- a/libcxx/test/libcxx/diagnostics/algorithm.nodiscard.verify.cpp
+++ b/libcxx/test/libcxx/diagnostics/algorithm.nodiscard.verify.cpp
@@ -372,6 +372,18 @@ void test() {
   // expected-warning@-1 {{ignoring return value of function declared with 'nodiscard' attribute}}
   std::ranges::contains_subrange(iter, iter, iter, iter);
   // expected-warning@-1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::ranges::find_last_if_not(iter, iter, pred);
+  // expected-warning@-1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::ranges::find_last_if_not(range, pred);
+  // expected-warning@-1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::ranges::find_last_if(iter, iter, pred);
+  // expected-warning@-1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::ranges::find_last_if(range, pred);
+  // expected-warning@-1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::ranges::find_last(iter, iter, 1);
+  // expected-warning@-1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::ranges::find_last(range, 1);
+  // expected-warning@-1 {{ignoring return value of function declared with 'nodiscard' attribute}}
   std::ranges::fold_left(range, 0, std::plus());
   // expected-warning@-1{{ignoring return value of function declared with 'nodiscard' attribute}}
   std::ranges::fold_left(iter, iter, 0, std::plus());
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last.pass.cpp
new file mode 100644
index 0000000000000..2a2b12fb2c288
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last.pass.cpp
@@ -0,0 +1,204 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <algorithm>
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// ADDITIONAL_COMPILE_FLAGS(gcc-style-warnings): -Wno-sign-compare
+// MSVC warning C4242: 'argument': conversion from 'const _Ty' to 'ElementT', possible loss of data
+// MSVC warning C4244: 'argument': conversion from 'const _Ty' to 'ElementT', possible loss of data
+// ADDITIONAL_COMPILE_FLAGS(cl-style-warnings): /wd4242 /wd4244
+
+// template<forward_iterator I, sentinel_for<I> S, class T, class Proj = identity>
+//   requires indirect_binary_predicate<ranges::equal_to, projected<I, Proj>, const T*>
+//   constexpr subrange<I> ranges::find_last(I first, S last, const T& value, Proj proj = {});
+// template<forward_range R, class T, class Proj = identity>
+//   requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<R>, Proj>, const T*>
+//   constexpr borrowed_subrange_t<R> ranges::find_last(R&& r, const T& value, Proj proj = {});
+
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <ranges>
+#include <vector>
+
+#include "almost_satisfies_types.h"
+#include "test_iterators.h"
+
+struct NotEqualityComparable {};
+
+template <class It, class Sent = It>
+concept HasFindLastIt = requires(It it, Sent sent) { std::ranges::find_last(it, sent, *it); };
+static_assert(HasFindLastIt<int*>);
+static_assert(HasFindLastIt<forward_iterator<int*>>);
+static_assert(!HasFindLastIt<cpp20_input_iterator<int*>>);
+static_assert(!HasFindLastIt<NotEqualityComparable*>);
+static_assert(!HasFindLastIt<ForwardIteratorNotDerivedFrom>);
+static_assert(!HasFindLastIt<ForwardIteratorNotIncrementable>);
+static_assert(!HasFindLastIt<forward_iterator<int*>, SentinelForNotSemiregular>);
+static_assert(!HasFindLastIt<forward_iterator<int*>, InputRangeNotSentinelEqualityComparableWith>);
+
+static_assert(!HasFindLastIt<int*, int>);
+static_assert(!HasFindLastIt<int, int*>);
+
+template <class Range, class ValT>
+concept HasFindLastR = requires(Range r) { std::ranges::find_last(r, ValT{}); };
+static_assert(HasFindLastR<std::array<int, 1>, int>);
+static_assert(!HasFindLastR<int, int>);
+static_assert(!HasFindLastR<std::array<NotEqualityComparable, 1>, NotEqualityComparable>);
+static_assert(!HasFindLastR<ForwardRangeNotDerivedFrom, int>);
+static_assert(!HasFindLastR<ForwardRangeNotIncrementable, int>);
+static_assert(!HasFindLastR<ForwardRangeNotSentinelSemiregular, int>);
+static_assert(!HasFindLastR<ForwardRangeNotSentinelEqualityComparableWith, int>);
+
+template <class It, class Sent = It>
+constexpr void test_iterators() {
+  using ValueT    = std::iter_value_t<It>;
+  auto make_range = [](auto& a) {
+    return std::ranges::subrange(It(std::ranges::begin(a)), Sent(It(std::ranges::end(a))));
+  };
+  { // simple test
+    {
+      ValueT a[] = {1, 2, 3, 4};
+
+      std::same_as<std::ranges::subrange<It>> auto ret = std::ranges::find_last(It(a), Sent(It(a + 4)), 2);
+      assert(base(ret.begin()) == a + 1);
+      assert(*ret.begin() == 2);
+    }
+    {
+      ValueT a[] = {1, 2, 3, 4};
+
+      std::same_as<std::ranges::subrange<It>> auto ret = std::ranges::find_last(make_range(a), 2);
+      assert(base(ret.begin()) == a + 1);
+      assert(*ret.begin() == 2);
+    }
+  }
+
+  { // check that an empty range works
+    {
+      std::array<ValueT, 0> a = {};
+
+      auto ret = std::ranges::find_last(It(a.data()), Sent(It(a.data())), 1).begin();
+      assert(ret == It(a.data()));
+    }
+    {
+      std::array<ValueT, 0> a = {};
+
+      auto ret = std::ranges::find_last(make_range(a), 1).begin();
+      assert(ret == It(a.begin()));
+    }
+  }
+
+  { // check that last is returned with no match
+    {
+      ValueT a[] = {1, 1, 1};
+
+      auto ret = std::ranges::find_last(It(a), Sent(It(a + 3)), 0).begin();
+      assert(ret == It(a + 3));
+    }
+    {
+      ValueT a[] = {1, 1, 1};
+
+      auto ret = std::ranges::find_last(make_range(a), 0).begin();
+      assert(ret == It(a + 3));
+    }
+  }
+}
+
+template <template <class> class IteratorT>
+constexpr void test_iterator_classes() {
+  { // check that the last element is returned
+    struct S {
+      int comp;
+      int other;
+    };
+    using it = IteratorT<S*>;
+    S a[]    = {{0, 0}, {0, 2}, {0, 1}};
+
+    auto ret = std::ranges::find_last(it(std::begin(a)), it(std::end(a)), 0, &S::comp).begin();
+    assert(ret == it(a + 2));
+    assert((*ret).comp == 0);
+    assert((*ret).other == 1);
+  }
+
+  {
+    // count invocations of the projection
+    using it = IteratorT<int*>;
+
+    int a[]              = {1, 2, 3, 4};
+    int projection_count = 0;
+
+    auto ret = std::ranges::find_last(it(std::begin(a)), it(std::end(a)), 2, [&](int i) {
+                 ++projection_count;
+                 return i;
+               }).begin();
+    assert(ret == it(a + 1));
+    assert(*ret == 2);
+    if (std::bidirectional_iterator<it>) {
+      assert(projection_count == 3);
+    } else {
+      assert(projection_count == 4); // must go through entire list
+    }
+  }
+}
+
+template <class ElementT>
+class TriviallyComparable {
+  ElementT el_;
+
+public:
+  constexpr TriviallyComparable(ElementT el) : el_(el) {}
+  bool operator==(const TriviallyComparable&) const = default;
+};
+
+constexpr bool test() {
+  types::for_each(types::type_list<char, int, TriviallyComparable<char>>{}, []<class T> {
+    types::for_each(types::forward_iterator_list<T*>{}, []<class Iter> {
+      test_iterators<Iter>();
+      test_iterators<Iter, sentinel_wrapper<Iter>>();
+      test_iterators<Iter, sized_sentinel<Iter>>();
+    });
+  });
+
+  test_iterator_classes<forward_iterator>();
+  test_iterator_classes<bidirectional_iterator>();
+  test_iterator_classes<random_access_iterator>();
+  test_iterator_classes<std::type_identity_t>();
+
+  {
+    std::vector<std::vector<int>> vec = {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}};
+
+    auto view = vec | std::views::join;
+    assert(std::ranges::find_last(view.begin(), view.end(), 4).begin() == std::next(view.begin(), 3));
+    assert(std::ranges::find_last(view, 4).begin() == std::next(view.begin(), 3));
+  }
+
+  {
+    // check that an iterator is returned with a borrowing range
+    int a[] = {1, 2, 3, 4};
+
+    std::same_as<std::ranges::subrange<int*>> auto ret = std::ranges::find_last(std::views::all(a), 1);
+    assert(ret.begin() == a);
+    assert(*ret.begin() == 1);
+  }
+
+  {
+    // check that dangling ranges are dangling
+    std::same_as<std::ranges::dangling> auto ret = std::ranges::find_last(std::vector<int>(), 0);
+    (void)ret;
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last_if.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last_if.pass.cpp
new file mode 100644
index 0000000000000..a15f81bd4e481
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last_if.pass.cpp
@@ -0,0 +1,252 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <algorithm>
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+//          indirect_unary_predicate<projected<I, Proj>> Pred>
+//   constexpr subrange<I> ranges::find_last_if(I first, S last, Pred pred, Proj proj = {});
+// template<forward_range R, class Proj = identity,
+//          indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+//   constexpr borrowed_subrange_t<R> ranges::find_last_if(R&& r, Pred pred, Proj proj = {});
+
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <ranges>
+
+#include "almost_satisfies_types.h"
+#include "test_iterators.h"
+
+struct Predicate {
+  bool operator()(int);
+};
+
+template <class It, class Sent = It>
+concept HasFindLastIfIt = requires(It it, Sent sent) { std::ranges::find_last_if(it, sent, Predicate{}); };
+static_assert(HasFindLastIfIt<int*>);
+static_assert(HasFindLastIfIt<forward_iterator<int*>>);
+static_assert(!HasFindLastIfIt<cpp20_input_iterator<int*>>);
+static_assert(!HasFindLastIfIt<ForwardIteratorNotDerivedFrom>);
+static_assert(!HasFindLastIfIt<ForwardIteratorNotIncrementable>);
+static_assert(!HasFindLastIfIt<forward_iterator<int*>, SentinelForNotSemiregular>);
+static_assert(!HasFindLastIfIt<forward_iterator<int*>, InputRangeNotSentinelEqualityComparableWith>);
+
+static_assert(!HasFindLastIfIt<int*, int>);
+static_assert(!HasFindLastIfIt<int, int*>);
+
+template <class Pred>
+concept HasFindLastIfPred = requires(int* it, Pred pred) { std::ranges::find_last_if(it, it, pred); };
+
+static_assert(!HasFindLastIfPred<IndirectUnaryPredicateNotCopyConstructible>);
+static_assert(!HasFindLastIfPred<IndirectUnaryPredicateNotPredicate>);
+
+template <class R>
+concept HasFindLastIfR = requires(R r) { std::ranges::find_last_if(r, Predicate{}); };
+static_assert(HasFindLastIfR<std::array<int, 0>>);
+static_assert(!HasFindLastIfR<int>);
+static_assert(!HasFindLastIfR<ForwardRangeNotDerivedFrom>);
+static_assert(!HasFindLastIfR<ForwardRangeNotIncrementable>);
+static_assert(!HasFindLastIfR<ForwardRangeNotSentinelSemiregular>);
+static_assert(!HasFindLastIfR<ForwardRangeNotSentinelEqualityComparableWith>);
+
+template <class It, class Sent>
+constexpr auto make_range(auto& a) {
+  return std::ranges::subrange(It(std::ranges::begin(a)), Sent(It(std::ranges::end(a))));
+}
+
+template <template <class> class IteratorT, template <class> class SentinelT>
+constexpr void test_iterator_classes() {
+  {
+    using it   = IteratorT<int*>;
+    using sent = SentinelT<it>;
+
+    {
+      int a[] = {1, 2, 3, 4};
+      std::same_as<std::ranges::subrange<it>> auto ret =
+          std::ranges::find_last_if(it(a), sent(it(a + 4)), [](int x) { return x == 4; });
+      assert(base(ret.begin()) == a + 3);
+      assert(*ret.begin() == 4);
+    }
+    {
+      int a[] = {1, 2, 3, 4};
+
+      std::same_as<std::ranges::subrange<it>> auto ret =
+          std::ranges::find_last_if(make_range<it, sent>(a), [](int x) { return x == 4; });
+      assert(ret.begin() == it(a + 3));
+      assert(*ret.begin() == 4);
+    }
+  }
+
+  { // check that an empty range works
+    using it   = IteratorT<std::ranges::iterator_t<std::array<int, 0>&>>;
+    using sent = SentinelT<it>;
+
+    {
+      std::array<int, 0> a = {};
+
+      auto ret = std::ranges::find_last_if(it(a.data()), sent(it(a.data())), [](auto&&) { return true; }).begin();
+      assert(ret == it(a.data()));
+    }
+    {
+      std::array<int, 0> a = {};
+
+      auto ret = std::ranges::find_last_if(make_range<it, sent>(a), [](auto&&) { return true; }).begin();
+      assert(ret == it(a.begin()));
+    }
+  }
+
+  { // check that last is returned with no match
+    using it   = IteratorT<int*>;
+    using sent = SentinelT<it>;
+
+    {
+      int a[] = {1, 1, 1};
+
+      auto ret = std::ranges::find_last_if(it(a), sent(it(a + 3)), [](auto&&) { return false; }).begin();
+      assert(ret == it(a + 3));
+    }
+    {
+      int a[] = {1, 1, 1};
+
+      auto ret = std::ranges::find_last_if(make_range<it, sent>(a), [](auto&&) { return false; }).begin();
+      assert(ret == it(a + 3));
+    }
+  }
+
+  { // check that the last element is returned
+    struct S {
+      int comp;
+      int other;
+    };
+    using it   = IteratorT<S*>;
+    using sent = SentinelT<it>;
+
+    S a[] = {{0, 0}, {0, 2}, {0, 1}};
+
+    auto ret = std::ranges::find_last_if(
+                   it(std::begin(a)), sent(it(std::end(a))), [](int c) { return c == 0; }, &S::comp)
+                   .begin();
+    assert(ret == it(a + 2));
+    assert((*ret).comp == 0);
+    assert((*ret).other == 1);
+  }
+
+  {
+    // count projection and predicate invocation count
+    {
+      int a[]              = {1, 2, 3, 4};
+      int predicate_count  = 0;
+      int projection_count = 0;
+
+      using it   = IteratorT<int*>;
+      using sent = SentinelT<it>;
+
+      auto ret =
+          std::ranges::find_last_if(
+              it(a),
+              sent(it(a + 4)),
+              [&](int i) {
+                ++predicate_count;
+                return i == 2;
+              },
+              [&](int i) {
+                ++projection_count;
+                return i;
+              })
+              .begin();
+      assert(ret == it(a + 1));
+      assert(*ret == 2);
+
+      if constexpr (std::bidirectional_iterator<it>) {
+        assert(predicate_count == 3);
+        assert(projection_count == 3);
+      } else {
+        assert(predicate_count == 4);
+        assert(projection_count == 4);
+      }
+    }
+  }
+}
+
+struct NonConstComparable {
+  friend constexpr bool operator==(const NonConstComparable&, const NonConstComparable&) { return false; }
+  friend constexpr bool operator==(NonConstComparable&, NonConstComparable&) { return false; }
+  friend constexpr bool operator==(const NonConstComparable&, NonConstComparable&) { return false; }
+  friend constexpr bool operator==(NonConstComparable&, const NonConstComparable&) { return true; }
+};
+
+template <class T>
+using add_const_to_ptr_t = std::add_pointer_t<std::add_const_t<std::remove_pointer_t<T>>>;
+
+constexpr bool test() {
+  test_iterator_classes<std::type_identity_t, std::type_identity_t>();
+  test_iterator_classes<add_const_to_ptr_t, std::type_identity_t>();
+  test_iterator_classes<contiguous_iterator, std::type_identity_t>();
+  test_iterator_classes<random_access_iterator, std::type_identity_t>();
+  test_iterator_classes<bidirectional_iterator, std::type_identity_t>();
+  test_iterator_classes<forward_iterator, std::type_identity_t>();
+  test_iterator_classes<forward_iterator, sentinel_wrapper>();
+
+  {
+    // check that projections are used properly and that they are called with the iterator directly
+    {
+      int a[] = {1, 2, 3, 4};
+      auto ret =
+          std::ranges::find_last_if(a, a + 4, [&](int* i) { return i == a + 3; }, [](int& i) { return &i; }).begin();
+      assert(ret == a + 3);
+    }
+    {
+      int a[]  = {1, 2, 3, 4};
+      auto ret = std::ranges::find_last_if(a, [&](int* i) { return i == a + 3; }, [](int& i) { return &i; }).begin();
+      assert(ret == a + 3);
+    }
+  }
+
+  {
+    // check that ranges::dangling is returned
+    [[maybe_unused]] std::same_as<std::ranges::dangling> auto ret =
+        std::ranges::find_last_if(std::array{1, 2}, [](int) { return false; });
+  }
+
+  {
+    // check that a subrange is returned with a borrowing range
+    int a[] = {1, 2, 3, 4};
+    std::same_as<std::ranges::subrange<int*>> auto ret =
+        std::ranges::find_last_if(std::views::all(a), [](int) { return true; });
+    assert(ret.begin() == a + 3);
+    assert(*ret.begin() == 4);
+  }
+
+  {
+    // check that the return type of `iter::operator*` doesn't change
+    {
+      NonConstComparable a[] = {NonConstComparable{}};
+
+      auto ret = std::ranges::find_last_if(a, a + 1, [](auto&& e) { return e == NonConstComparable{}; }).begin();
+      assert(ret == a);
+    }
+    {
+      NonConstComparable a[] = {NonConstComparable{}};
+
+      auto ret = std::ranges::find_last_if(a, [](auto&& e) { return e == NonConstComparable{}; }).begin();
+      assert(ret == a);
+    }
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last_if_not.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last_if_not.pass.cpp
new file mode 100644
index 0000000000000..bb0e411acf0fa
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find.last/ranges.find_last_if_not.pass.cpp
@@ -0,0 +1,254 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <algorithm>
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+//          indirect_unary_predicate<projected<I, Proj>> Pred>
+//   constexpr subrange<I> ranges::find_last_if_not(I first, S last, Pred pred, Proj proj = {});
+// template<forward_range R, class Proj = identity,
+//          indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+//   constexpr borrowed_subrange_t<R> ranges::find_last_if_not(R&& r, Pred pred, Proj proj = {});
+
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <ranges>
+
+#include "almost_satisfies_types.h"
+#include "test_iterators.h"
+
+struct Predicate {
+  bool operator()(int);
+};
+
+template <class It, class Sent = It>
+concept HasFindLastIfIt = requires(It it, Sent sent) { std::ranges::find_last_if_not(it, sent, Predicate{}); };
+static_assert(HasFindLastIfIt<int*>);
+static_assert(HasFindLastIfIt<forward_iterator<int*>>);
+static_assert(!HasFindLastIfIt<cpp20_input_iterator<int*>>);
+static_assert(!HasFindLastIfIt<ForwardIteratorNotDerivedFrom>);
+static_assert(!HasFindLastIfIt<ForwardIteratorNotIncrementable>);
+static_assert(!HasFindLastIfIt<forward_iterator<int*>, SentinelForNotSemiregular>);
+static_assert(!HasFindLastIfIt<forward_iterator<int*>, InputRangeNotSentinelEqualityComparableWith>);
+
+static_assert(!HasFindLastIfIt<int*, int>);
+static_assert(!HasFindLastIfIt<int, int*>);
+
+template <class Pred>
+concept HasFindLastIfPred = requires(int* it, Pred pred) { std::ranges::find_last_if_not(it, it, pred); };
+
+static_assert(!HasFindLastIfPred<IndirectUnaryPredicateNotCopyConstructible>);
+static_assert(!HasFindLastIfPred<IndirectUnaryPredicateNotPredicate>);
+
+template <class R>
+concept HasFindLastIfR = requires(R r) { std::ranges::find_last_if_not(r, Predicate{}); };
+static_assert(HasFindLastIfR<std::array<int, 0>>);
+static_assert(!HasFindLastIfR<int>);
+static_assert(!HasFindLastIfR<ForwardRangeNotDerivedFrom>);
+static_assert(!HasFindLastIfR<ForwardRangeNotIncrementable>);
+static_assert(!HasFindLastIfR<ForwardRangeNotSentinelSemiregular>);
+static_assert(!HasFindLastIfR<ForwardRangeNotSentinelEqualityComparableWith>);
+
+template <class It, class Sent>
+constexpr auto make_range(auto& a) {
+  return std::ranges::subrange(It(std::ranges::begin(a)), Sent(It(std::ranges::end(a))));
+}
+
+template <template <class> class IteratorT, template <class> class SentinelT>
+constexpr void test_iterator_classes() {
+  {
+    using it   = IteratorT<int*>;
+    using sent = SentinelT<it>;
+
+    {
+      int a[] = {1, 2, 3, 4};
+      std::same_as<std::ranges::subrange<it>> auto ret =
+          std::ranges::find_last_if_not(it(a), sent(it(a + 4)), [](int x) { return x != 4; });
+      assert(base(ret.begin()) == a + 3);
+      assert(*ret.begin() == 4);
+    }
+    {
+      int a[] = {1, 2, 3, 4};
+
+      std::same_as<std::ranges::subrange<it>> auto ret =
+          std::ranges::find_last_if_not(make_range<it, sent>(a), [](int x) { return x != 4; });
+      assert(ret.begin() == it(a + 3));
+      assert(*ret.begin() == 4);
+    }
+  }
+
+  { // check that an empty range works
+    using it   = IteratorT<std::ranges::iterator_t<std::array<int, 0>&>>;
+    using sent = SentinelT<it>;
+
+    {
+      std::array<int, 0> a = {};
+
+      auto ret = std::ranges::find_last_if_not(it(a.data()), sent(it(a.data())), [](auto&&) { return false; }).begin();
+      assert(ret == it(a.data()));
+    }
+    {
+      std::array<int, 0> a = {};
+
+      auto ret = std::ranges::find_last_if_not(make_range<it, sent>(a), [](auto&&) { return false; }).begin();
+      assert(ret == it(a.begin()));
+    }
+  }
+
+  { // check that last is returned with no match
+    using it   = IteratorT<int*>;
+    using sent = SentinelT<it>;
+
+    {
+      int a[] = {1, 1, 1};
+
+      auto ret = std::ranges::find_last_if_not(it(a), sent(it(a + 3)), [](auto&&) { return true; }).begin();
+      assert(ret == it(a + 3));
+    }
+    {
+      int a[] = {1, 1, 1};
+
+      auto ret = std::ranges::find_last_if_not(make_range<it, sent>(a), [](auto&&) { return true; }).begin();
+      assert(ret == it(a + 3));
+    }
+  }
+
+  { // check that the last element is returned
+    struct S {
+      int comp;
+      int other;
+    };
+    using it   = IteratorT<S*>;
+    using sent = SentinelT<it>;
+
+    S a[] = {{0, 0}, {0, 2}, {0, 1}};
+
+    auto ret = std::ranges::find_last_if_not(
+                   it(std::begin(a)), sent(it(std::end(a))), [](int c) { return c != 0; }, &S::comp)
+                   .begin();
+    assert(ret == it(a + 2));
+    assert((*ret).comp == 0);
+    assert((*ret).other == 1);
+  }
+
+  {
+    // count projection and predicate invocation count
+    {
+      int a[]              = {1, 2, 3, 4};
+      int predicate_count  = 0;
+      int projection_count = 0;
+
+      using it   = IteratorT<int*>;
+      using sent = SentinelT<it>;
+
+      auto ret =
+          std::ranges::find_last_if_not(
+              it(a),
+              sent(it(a + 4)),
+              [&](int i) {
+                ++predicate_count;
+                return i != 2;
+              },
+              [&](int i) {
+                ++projection_count;
+                return i;
+              })
+              .begin();
+      assert(ret == it(a + 1));
+      assert(*ret == 2);
+
+      if constexpr (std::bidirectional_iterator<it>) {
+        assert(predicate_count == 3);
+        assert(projection_count == 3);
+      } else {
+        assert(predicate_count == 4);
+        assert(projection_count == 4);
+      }
+    }
+  }
+}
+
+struct NonConstComparable {
+  friend constexpr bool operator!=(const NonConstComparable&, const NonConstComparable&) { return true; }
+  friend constexpr bool operator!=(NonConstComparable&, NonConstComparable&) { return true; }
+  friend constexpr bool operator!=(const NonConstComparable&, NonConstComparable&) { return true; }
+  friend constexpr bool operator!=(NonConstComparable&, const NonConstComparable&) { return false; }
+};
+
+template <class T>
+using add_const_to_ptr_t = std::add_pointer_t<std::add_const_t<std::remove_pointer_t<T>>>;
+
+constexpr bool test() {
+  test_iterator_classes<std::type_identity_t, std::type_identity_t>();
+  test_iterator_classes<add_const_to_ptr_t, std::type_identity_t>();
+  test_iterator_classes<contiguous_iterator, std::type_identity_t>();
+  test_iterator_classes<random_access_iterator, std::type_identity_t>();
+  test_iterator_classes<bidirectional_iterator, std::type_identity_t>();
+  test_iterator_classes<forward_iterator, std::type_identity_t>();
+  test_iterator_classes<forward_iterator, sentinel_wrapper>();
+
+  {
+    // check that projections are used properly and that they are called with the iterator directly
+    {
+      int a[]  = {1, 2, 3, 4};
+      auto ret = std::ranges::find_last_if_not(
+                     a, a + 4, [&](int* i) { return i != a + 3; }, [](int& i) { return &i; })
+                     .begin();
+      assert(ret == a + 3);
+    }
+    {
+      int a[] = {1, 2, 3, 4};
+      auto ret =
+          std::ranges::find_last_if_not(a, [&](int* i) { return i != a + 3; }, [](int& i) { return &i; }).begin();
+      assert(ret == a + 3);
+    }
+  }
+
+  {
+    // check that ranges::dangling is returned
+    [[maybe_unused]] std::same_as<std::ranges::dangling> auto ret =
+        std::ranges::find_last_if_not(std::array{1, 2}, [](int) { return false; });
+  }
+
+  {
+    // check that a subrange is returned with a borrowing range
+    int a[] = {1, 2, 3, 4};
+    std::same_as<std::ranges::subrange<int*>> auto ret =
+        std::ranges::find_last_if_not(std::views::all(a), [](int) { return false; });
+    assert(ret.begin() == a + 3);
+    assert(*ret.begin() == 4);
+  }
+
+  {
+    // check that the return type of `iter::operator*` doesn't change
+    {
+      NonConstComparable a[] = {NonConstComparable{}};
+
+      auto ret = std::ranges::find_last_if_not(a, a + 1, [](auto&& e) { return e != NonConstComparable{}; }).begin();
+      assert(ret == a);
+    }
+    {
+      NonConstComparable a[] = {NonConstComparable{}};
+
+      auto ret = std::ranges::find_last_if_not(a, [](auto&& e) { return e != NonConstComparable{}; }).begin();
+      assert(ret == a);
+    }
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/ranges_robust_against_dangling.pass.cpp b/libcxx/test/std/algorithms/ranges_robust_against_dangling.pass.cpp
index e01a86d9f746f..0624a6c2d49c7 100644
--- a/libcxx/test/std/algorithms/ranges_robust_against_dangling.pass.cpp
+++ b/libcxx/test/std/algorithms/ranges_robust_against_dangling.pass.cpp
@@ -116,6 +116,11 @@ constexpr bool test_all() {
   dangling_1st(std::ranges::find, in, x);
   dangling_1st(std::ranges::find_if, in, unary_pred);
   dangling_1st(std::ranges::find_if_not, in, unary_pred);
+#if TEST_STD_VER >= 23
+  dangling_1st(std::ranges::find_last, in, x);
+  dangling_1st(std::ranges::find_last_if, in, unary_pred);
+  dangling_1st(std::ranges::find_last_if_not, in, unary_pred);
+#endif
   dangling_1st(std::ranges::find_first_of, in, in2);
   dangling_1st(std::ranges::adjacent_find, in);
   dangling_1st<mismatch_result<dangling, InIter>>(std::ranges::mismatch, in, in2);
diff --git a/libcxx/test/std/algorithms/ranges_robust_against_nonbool.compile.pass.cpp b/libcxx/test/std/algorithms/ranges_robust_against_nonbool.compile.pass.cpp
index a5668f20ccfd2..5203ee18d7bb0 100644
--- a/libcxx/test/std/algorithms/ranges_robust_against_nonbool.compile.pass.cpp
+++ b/libcxx/test/std/algorithms/ranges_robust_against_nonbool.compile.pass.cpp
@@ -53,6 +53,10 @@ void f(Iterator it, Range in, Iterator out, std::size_t n, Value const& val, std
   in_pred(std::ranges::none_of, pred1);
   in_pred(std::ranges::find_if, pred1);
   in_pred(std::ranges::find_if_not, pred1);
+#if TEST_STD_VER >= 23
+  in_pred(std::ranges::find_last_if, pred1);
+  in_pred(std::ranges::find_last_if_not, pred1);
+#endif
   in_in_pred(std::ranges::find_first_of, pred2);
   in_pred(std::ranges::adjacent_find, pred2);
   in_in_pred(std::ranges::mismatch, pred2);
diff --git a/libcxx/test/std/algorithms/ranges_robust_against_proxy_iterators.pass.cpp b/libcxx/test/std/algorithms/ranges_robust_against_proxy_iterators.pass.cpp
index 139f1999bc9dc..ca1433b778751 100644
--- a/libcxx/test/std/algorithms/ranges_robust_against_proxy_iterators.pass.cpp
+++ b/libcxx/test/std/algorithms/ranges_robust_against_proxy_iterators.pass.cpp
@@ -82,6 +82,11 @@ constexpr void run_tests() {
   test(std::ranges::find, in, x);
   test(std::ranges::find_if, in, unary_pred);
   test(std::ranges::find_if_not, in, unary_pred);
+#if TEST_STD_VER >= 23
+  test(std::ranges::find_last, in, x);
+  test(std::ranges::find_last_if, in, unary_pred);
+  test(std::ranges::find_last_if_not, in, unary_pred);
+#endif
   test(std::ranges::find_first_of, in, in2);
   test(std::ranges::adjacent_find, in);
   test(std::ranges::mismatch, in, in2);
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/algorithm.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/algorithm.version.compile.pass.cpp
index 8ccd252115ac8..ded8006063241 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/algorithm.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/algorithm.version.compile.pass.cpp
@@ -23,6 +23,7 @@
     __cpp_lib_parallel_algorithm                            201603L [C++17]
     __cpp_lib_ranges                                        202207L [C++20]
     __cpp_lib_ranges_contains                               202207L [C++23]
+    __cpp_lib_ranges_find_last                              202207L [C++23]
     __cpp_lib_ranges_starts_ends_with                       202106L [C++23]
     __cpp_lib_robust_nonmodifying_seq_ops                   201304L [C++14]
     __cpp_lib_sample                                        201603L [C++17]
@@ -62,6 +63,10 @@
 #   error "__cpp_lib_ranges_contains should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_ranges_starts_ends_with
 #   error "__cpp_lib_ranges_starts_ends_with should not be defined before c++23"
 # endif
@@ -108,6 +113,10 @@
 #   error "__cpp_lib_ranges_contains should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_ranges_starts_ends_with
 #   error "__cpp_lib_ranges_starts_ends_with should not be defined before c++23"
 # endif
@@ -169,6 +178,10 @@
 #   error "__cpp_lib_ranges_contains should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_ranges_starts_ends_with
 #   error "__cpp_lib_ranges_starts_ends_with should not be defined before c++23"
 # endif
@@ -239,6 +252,10 @@
 #   error "__cpp_lib_ranges_contains should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_ranges_starts_ends_with
 #   error "__cpp_lib_ranges_starts_ends_with should not be defined before c++23"
 # endif
@@ -315,6 +332,13 @@
 #   error "__cpp_lib_ranges_contains should have the value 202207L in c++23"
 # endif
 
+# ifndef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should be defined in c++23"
+# endif
+# if __cpp_lib_ranges_find_last != 202207L
+#   error "__cpp_lib_ranges_find_last should have the value 202207L in c++23"
+# endif
+
 # ifndef __cpp_lib_ranges_starts_ends_with
 #   error "__cpp_lib_ranges_starts_ends_with should be defined in c++23"
 # endif
@@ -412,6 +436,13 @@
 #   error "__cpp_lib_ranges_contains should have the value 202207L in c++26"
 # endif
 
+# ifndef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should be defined in c++26"
+# endif
+# if __cpp_lib_ranges_find_last != 202207L
+#   error "__cpp_lib_ranges_find_last should have the value 202207L in c++26"
+# endif
+
 # ifndef __cpp_lib_ranges_starts_ends_with
 #   error "__cpp_lib_ranges_starts_ends_with should be defined in c++26"
 # endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index 945de95df67e1..f26e7dc4b4c63 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -173,6 +173,7 @@
     __cpp_lib_ranges_chunk_by                               202202L [C++23]
     __cpp_lib_ranges_concat                                 202403L [C++26]
     __cpp_lib_ranges_contains                               202207L [C++23]
+    __cpp_lib_ranges_find_last                              202207L [C++23]
     __cpp_lib_ranges_iota                                   202202L [C++23]
     __cpp_lib_ranges_join_with                              202202L [C++23]
     __cpp_lib_ranges_repeat                                 202207L [C++23]
@@ -849,6 +850,10 @@
 #   error "__cpp_lib_ranges_contains should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_ranges_iota
 #   error "__cpp_lib_ranges_iota should not be defined before c++23"
 # endif
@@ -1745,6 +1750,10 @@
 #   error "__cpp_lib_ranges_contains should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_ranges_iota
 #   error "__cpp_lib_ranges_iota should not be defined before c++23"
 # endif
@@ -2812,6 +2821,10 @@
 #   error "__cpp_lib_ranges_contains should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_ranges_iota
 #   error "__cpp_lib_ranges_iota should not be defined before c++23"
 # endif
@@ -4146,6 +4159,10 @@
 #   error "__cpp_lib_ranges_contains should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_ranges_iota
 #   error "__cpp_lib_ranges_iota should not be defined before c++23"
 # endif
@@ -5657,6 +5674,13 @@
 #   error "__cpp_lib_ranges_contains should have the value 202207L in c++23"
 # endif
 
+# ifndef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should be defined in c++23"
+# endif
+# if __cpp_lib_ranges_find_last != 202207L
+#   error "__cpp_lib_ranges_find_last should have the value 202207L in c++23"
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_ranges_iota
 #     error "__cpp_lib_ranges_iota should be defined in c++23"
@@ -7510,6 +7534,13 @@
 #   error "__cpp_lib_ranges_contains should have the value 202207L in c++26"
 # endif
 
+# ifndef __cpp_lib_ranges_find_last
+#   error "__cpp_lib_ranges_find_last should be defined in c++26"
+# endif
+# if __cpp_lib_ranges_find_last != 202207L
+#   error "__cpp_lib_ranges_find_last should have the value 202207L in c++26"
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_ranges_iota
 #     error "__cpp_lib_ranges_iota should be defined in c++26"
diff --git a/libcxx/test/std/library/description/conventions/customization.point.object/niebloid.compile.pass.cpp b/libcxx/test/std/library/description/conventions/customization.point.object/niebloid.compile.pass.cpp
index 9506ca1c768bd..b26b397056343 100644
--- a/libcxx/test/std/library/description/conventions/customization.point.object/niebloid.compile.pass.cpp
+++ b/libcxx/test/std/library/description/conventions/customization.point.object/niebloid.compile.pass.cpp
@@ -88,6 +88,9 @@ static_assert(test(std::ranges::find_first_of, a, a));
 static_assert(test(std::ranges::find_if, a, odd));
 static_assert(test(std::ranges::find_if_not, a, odd));
 #if TEST_STD_VER >= 23
+static_assert(test(std::ranges::find_last, a, 42));
+static_assert(test(std::ranges::find_last_if, a, odd));
+static_assert(test(std::ranges::find_last_if_not, a, odd));
 static_assert(test(std::ranges::fold_left, a, 0, std::plus()));
 static_assert(test(std::ranges::fold_left_with_iter, a, 0, std::plus()));
 #endif
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index a09beb29565ad..a351112471295 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -1033,6 +1033,11 @@ def add_version_header(tc):
             "values": {"c++23": 202207},
             "headers": ["algorithm"],
         },
+        {
+            "name": "__cpp_lib_ranges_find_last",
+            "values": {"c++23": 202207},
+            "headers": ["algorithm"],
+        },
         {
             "name": "__cpp_lib_ranges_iota",
             "values": {"c++23": 202202},

From 22eb290a9696e2a3fd042096c61e35eca2fcce0c Mon Sep 17 00:00:00 2001
From: Edd Dawson <edd.dawson@sony.com>
Date: Fri, 19 Jul 2024 17:52:00 +0100
Subject: [PATCH 221/486] [PS4/PS5][Driver][DWARF] Always emit .debug_aranges
 for SCE tuning (#99629)

Some of SIE's post-mortem analysis infrastructure currently makes use of
.debug_aranges, so we'd like to ensure the section's presence in
PlayStation binaries. The simplest way to do this is force emission when
the debugger tuning is set to SCE (which is in turn typically
initialized from the target triple). This also simplifies the driver.

SIE tracker: TOOLCHAIN-16951
---
 clang/lib/Driver/ToolChains/Clang.cpp         |  7 ++--
 clang/lib/Driver/ToolChains/PS4CPU.cpp        |  8 -----
 clang/test/Driver/debug-options.c             | 21 ++++++------
 clang/test/Driver/lto-jobs.c                  |  2 +-
 clang/test/Driver/ps4-linker.c                |  8 ++---
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  5 ++-
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h      |  3 ++
 .../DebugInfo/X86/debug-aranges-sce-tuning.ll | 33 +++++++++++++++++++
 8 files changed, 57 insertions(+), 30 deletions(-)
 create mode 100644 llvm/test/DebugInfo/X86/debug-aranges-sce-tuning.ll

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index f7b987bf810c1..abe517f0f5dea 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -4659,11 +4659,8 @@ renderDebugOptions(const ToolChain &TC, const Driver &D, const llvm::Triple &T,
 
   // -gdwarf-aranges turns on the emission of the aranges section in the
   // backend.
-  // Always enabled for SCE tuning.
-  bool NeedAranges = DebuggerTuning == llvm::DebuggerKind::SCE;
-  if (const Arg *A = Args.getLastArg(options::OPT_gdwarf_aranges))
-    NeedAranges = checkDebugInfoOption(A, Args, D, TC) || NeedAranges;
-  if (NeedAranges) {
+  if (const Arg *A = Args.getLastArg(options::OPT_gdwarf_aranges);
+      A && checkDebugInfoOption(A, Args, D, TC)) {
     CmdArgs.push_back("-mllvm");
     CmdArgs.push_back("-generate-arange-section");
   }
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.cpp b/clang/lib/Driver/ToolChains/PS4CPU.cpp
index 974e486a0082b..d6af9388e54a6 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -162,10 +162,6 @@ void tools::PS4cpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   };
 
   if (UseLTO) {
-    // We default to creating the arange section, but LTO does not. Enable it
-    // here.
-    AddCodeGenFlag("-generate-arange-section");
-
     // This tells LTO to perform JustMyCode instrumentation.
     if (UseJMC)
       AddCodeGenFlag("-enable-jmc-instrument");
@@ -272,10 +268,6 @@ void tools::PS5cpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   };
 
   if (UseLTO) {
-    // We default to creating the arange section, but LTO does not. Enable it
-    // here.
-    AddCodeGenFlag("-generate-arange-section");
-
     // This tells LTO to perform JustMyCode instrumentation.
     if (UseJMC)
       AddCodeGenFlag("-enable-jmc-instrument");
diff --git a/clang/test/Driver/debug-options.c b/clang/test/Driver/debug-options.c
index 0a665f7017d63..21785ba01cb41 100644
--- a/clang/test/Driver/debug-options.c
+++ b/clang/test/Driver/debug-options.c
@@ -118,27 +118,28 @@
 // RUN: %clang_cl -### -c -Z7 -target x86_64-windows-msvc -- %s 2>&1 \
 // RUN:             | FileCheck -check-prefix=G_NOTUNING %s
 
-// On the PS4/PS5, -g defaults to -gno-column-info, and we always generate the
-// arange section.
+// On the PS4/PS5, -g defaults to -gno-column-info. We default to always
+// generating the arange section, but keyed off SCE DebuggerTuning being in
+// play during codegen, instead of -generate-arange-section.
 // RUN: %clang -### -c %s -target x86_64-scei-ps4 2>&1 \
 // RUN:             | FileCheck -check-prefix=NOG_PS %s
 // RUN: %clang -### -c %s -target x86_64-sie-ps5 2>&1 \
 // RUN:             | FileCheck -check-prefix=NOG_PS %s
 /// PS4 will stay on v4 even if the generic default version changes.
 // RUN: %clang -### -c %s -g -target x86_64-scei-ps4 2>&1 \
-// RUN:             | FileCheck -check-prefixes=G_DWARF4,GARANGE,G_SCE,NOCI,FWD_TMPL_PARAMS %s
+// RUN:             | FileCheck -check-prefixes=G_DWARF4,G_SCE,NOCI,FWD_TMPL_PARAMS %s
 // RUN: %clang -### -c %s -g -target x86_64-sie-ps5 2>&1 \
-// RUN:             | FileCheck -check-prefixes=G_DWARF5,GARANGE,G_SCE,NOCI,FWD_TMPL_PARAMS %s
+// RUN:             | FileCheck -check-prefixes=G_DWARF5,G_SCE,NOCI,FWD_TMPL_PARAMS %s
 // RUN: %clang -### -c %s -g -gcolumn-info -target x86_64-scei-ps4 2>&1 \
 // RUN:             | FileCheck -check-prefix=CI %s
 // RUN: %clang -### -c %s -gsce -target x86_64-unknown-linux 2>&1 \
 // RUN:             | FileCheck -check-prefix=NOCI %s
 // RUN: %clang -### %s -g -flto=thin -target x86_64-scei-ps4 2>&1 \
-// RUN:             | FileCheck -check-prefix=SNLDTLTOGARANGE %s
+// RUN:             | FileCheck -check-prefix=LDGARANGE %s
 // RUN: %clang -### %s -g -flto=full -target x86_64-scei-ps4 2>&1 \
-// RUN:             | FileCheck -check-prefix=SNLDFLTOGARANGE %s
+// RUN:             | FileCheck -check-prefix=LDGARANGE %s
 // RUN: %clang -### %s -g -flto -target x86_64-scei-ps5 2>&1 \
-// RUN:             | FileCheck -check-prefix=LLDGARANGE %s
+// RUN:             | FileCheck -check-prefix=LDGARANGE %s
 // RUN: %clang -### %s -g -target x86_64-scei-ps5 2>&1 \
 // RUN:             | FileCheck -check-prefix=LDGARANGE %s
 
@@ -321,8 +322,7 @@
 //
 // NOG_PS: "-cc1"
 // NOG_PS-NOT: "-dwarf-version=
-// NOG_PS: "-generate-arange-section"
-// NOG_PS-NOT: "-dwarf-version=
+// NOG_PS-NOT: "-generate-arange-section"
 //
 // G_ERR: error: unknown argument:
 //
@@ -402,8 +402,7 @@
 //
 
 // LDGARANGE: {{".*ld.*"}} {{.*}}
-// LDGARANGE-NOT: "-plugin-opt=-generate-arange-section"
-// LLDGARANGE: {{".*lld.*"}} {{.*}} "-plugin-opt=-generate-arange-section"
+// LDGARANGE-NOT: -generate-arange-section"
 // SNLDTLTOGARANGE: {{".*orbis-ld.*"}} {{.*}} "-lto-thin-debug-options= -generate-arange-section"
 // SNLDFLTOGARANGE: {{".*orbis-ld.*"}} {{.*}} "-lto-debug-options= -generate-arange-section"
 
diff --git a/clang/test/Driver/lto-jobs.c b/clang/test/Driver/lto-jobs.c
index 43a478b0664d8..b4f109e4c502c 100644
--- a/clang/test/Driver/lto-jobs.c
+++ b/clang/test/Driver/lto-jobs.c
@@ -11,7 +11,7 @@
 // RUN: %clang --target=x86_64-scei-ps4 -### %s -flto=thin -flto-jobs=5 2> %t
 // RUN: FileCheck -check-prefix=CHECK-PS4-LINK-THIN-JOBS-ACTION < %t %s
 //
-// CHECK-PS4-LINK-THIN-JOBS-ACTION: "-lto-thin-debug-options= -generate-arange-section -threads=5"
+// CHECK-PS4-LINK-THIN-JOBS-ACTION: "-lto-thin-debug-options= -threads=5"
 
 // RUN: %clang --target=x86_64-apple-darwin13.3.0 -### %s -flto=thin -flto-jobs=5 2> %t
 // RUN: FileCheck -check-prefix=CHECK-LINK-THIN-JOBS2-ACTION < %t %s
diff --git a/clang/test/Driver/ps4-linker.c b/clang/test/Driver/ps4-linker.c
index be0103bffe813..be989cdd7d5b1 100644
--- a/clang/test/Driver/ps4-linker.c
+++ b/clang/test/Driver/ps4-linker.c
@@ -5,8 +5,8 @@
 // RUN: %clang --target=x86_64-scei-ps4 -flto=full -fjmc %s -### 2>&1 | FileCheck --check-prefixes=CHECK-FULL-LTO,CHECK-LIB %s
 
 // CHECK-NOT: -enable-jmc-instrument
-// CHECK-THIN-LTO: "-lto-thin-debug-options= -generate-arange-section -enable-jmc-instrument"
-// CHECK-FULL-LTO: "-lto-debug-options= -generate-arange-section -enable-jmc-instrument"
+// CHECK-THIN-LTO: "-lto-thin-debug-options= -enable-jmc-instrument"
+// CHECK-FULL-LTO: "-lto-debug-options= -enable-jmc-instrument"
 
 // Check the default library name.
 // CHECK-LIB: "--whole-archive" "-lSceDbgJmc" "--no-whole-archive"
@@ -16,5 +16,5 @@
 // RUN: %clang --target=x86_64-scei-ps4 -flto=thin -fcrash-diagnostics-dir=mydumps %s -### 2>&1 | FileCheck --check-prefixes=CHECK-DIAG-THIN-LTO %s
 // RUN: %clang --target=x86_64-scei-ps4 -flto=full -fcrash-diagnostics-dir=mydumps %s -### 2>&1 | FileCheck --check-prefixes=CHECK-DIAG-FULL-LTO %s
 
-// CHECK-DIAG-THIN-LTO: "-lto-thin-debug-options= -generate-arange-section -crash-diagnostics-dir=mydumps"
-// CHECK-DIAG-FULL-LTO: "-lto-debug-options= -generate-arange-section -crash-diagnostics-dir=mydumps"
+// CHECK-DIAG-THIN-LTO: "-lto-thin-debug-options= -crash-diagnostics-dir=mydumps"
+// CHECK-DIAG-FULL-LTO: "-lto-debug-options= -crash-diagnostics-dir=mydumps"
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 80cd5ec501f25..578a90d402c54 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -354,6 +354,9 @@ DwarfDebug::DwarfDebug(AsmPrinter *A)
 
   UseLocSection = !TT.isNVPTX();
 
+  // Always emit .debug_aranges for SCE tuning.
+  UseARangesSection = GenerateARangeSection || tuneForSCE();
+
   HasAppleExtensionAttributes = tuneForLLDB();
 
   // Handle split DWARF.
@@ -1450,7 +1453,7 @@ void DwarfDebug::endModule() {
   emitDebugInfo();
 
   // Emit info into a debug aranges section.
-  if (GenerateARangeSection)
+  if (UseARangesSection)
     emitDebugARanges();
 
   // Emit info into a debug ranges section.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 452485b632c45..13f4c379e0027 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -435,6 +435,9 @@ class DwarfDebug : public DebugHandlerBase {
   ///Allow emission of the .debug_loc section.
   bool UseLocSection = true;
 
+  /// Allow emission of .debug_aranges section
+  bool UseARangesSection = false;
+
   /// Generate DWARF v4 type units.
   bool GenerateTypeUnits;
 
diff --git a/llvm/test/DebugInfo/X86/debug-aranges-sce-tuning.ll b/llvm/test/DebugInfo/X86/debug-aranges-sce-tuning.ll
new file mode 100644
index 0000000000000..ac0d61aef2b1c
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/debug-aranges-sce-tuning.ll
@@ -0,0 +1,33 @@
+; This checks that .debug_aranges is always generated for the SCE debugger
+; tuning.
+
+; RUN: llc -mtriple=x86_64 -debugger-tune=sce -filetype=obj %s -o %t
+; RUN: llvm-dwarfdump -debug-aranges %t | FileCheck %s
+
+; CHECK:      .debug_aranges contents:
+; CHECK-NEXT: Address Range Header:
+; CHECK-SAME:   length = 0x0000002c,
+
+; IR generated and reduced from:
+; $ cat foo.c
+; int foo;
+; $ clang -g -S -emit-llvm foo.c -o foo.ll
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = dso_local global i32 0, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!6, !7, !8}
+!llvm.ident = !{!9}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "foo", scope: !2, file: !3, line: 1, type: !5, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C11, file: !3, producer: "clang version 19.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None)
+!3 = !DIFile(filename: "foo.c", directory: "/tmp")
+!4 = !{!0}
+!5 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!6 = !{i32 7, !"Dwarf Version", i32 5}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{i32 1, !"wchar_size", i32 4}
+!9 = !{!"clang version 19.0.0"}

From 2f8b64d327a27a73b57a54eb5d1f28c41e650aee Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm@amd.com>
Date: Fri, 19 Jul 2024 19:37:24 +0200
Subject: [PATCH 222/486] [flang][runtime] Build ISO_FORTRAN_ENV to export kind
 arrays as linkable symbols (#95388)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Moves definitions of the kind arrays into a Fortran MODULE to not only
emit the MOD file, but also compile that MODULE file into an object
file. This file is then linked into libFortranRuntime.so.

Eventually this workaround PR shoud be redone and a proper runtime build
should be setup that will then also compile Fortran MODULE files.

Fixes #89403

---------

Co-authored-by: Valentin Clement (�レンタイン クレメン) <clementval@gmail.com>
Co-authored-by: Michael Kruse <github@meinersbur.de>
---
 flang/CMakeLists.txt                    |   1 +
 flang/module/iso_fortran_env.f90        | 118 +++++++-----------------
 flang/module/iso_fortran_env_impl.f90   | 114 +++++++++++++++++++++++
 flang/runtime/CMakeLists.txt            |   1 +
 flang/tools/f18/CMakeLists.txt          |  47 +++++++++-
 flang/tools/flang-driver/CMakeLists.txt |   7 --
 6 files changed, 191 insertions(+), 97 deletions(-)
 create mode 100644 flang/module/iso_fortran_env_impl.f90

diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index cbe8f1186236a..070c39eb6e9ab 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -538,3 +538,4 @@ get_clang_resource_dir(HEADER_INSTALL_DIR SUBDIR include)
 install(
   FILES include/flang/ISO_Fortran_binding.h
   DESTINATION ${HEADER_INSTALL_DIR} )
+
diff --git a/flang/module/iso_fortran_env.f90 b/flang/module/iso_fortran_env.f90
index 6ca98e518aeac..cc1f58e7feccb 100644
--- a/flang/module/iso_fortran_env.f90
+++ b/flang/module/iso_fortran_env.f90
@@ -22,6 +22,23 @@ module iso_fortran_env
     compiler_options => __builtin_compiler_options, &
     compiler_version => __builtin_compiler_version
 
+  use iso_fortran_env_impl, only: &
+    selectedInt8, selectedInt16, selectedInt32, selectedInt64, selectedInt128, &
+    safeInt8, safeInt16, safeInt32, safeInt64, safeInt128, &
+    int8, int16, int32, int64, int128, &
+    logical8, logical16, logical32, logical64, &
+    selectedReal16, selectedBfloat16, selectedReal32, &
+    selectedReal64, selectedReal80, selectedReal64x2, &
+    selectedReal128, &
+    safeReal16, safeBfloat16, safeReal32, &
+    safeReal64, safeReal80, safeReal64x2, &
+    safeReal128, &
+    real16, bfloat16, real32, real64, &
+    real80, real64x2, real128, &
+    integer_kinds => __builtin_integer_kinds, &
+    real_kinds => __builtin_real_kinds, &
+    logical_kinds => __builtin_logical_kinds
+
   implicit none
   private
 
@@ -38,95 +55,22 @@ module iso_fortran_env
     pack([selectedUCS_2], selectedUCS_2 >= 0), &
     pack([selectedUnicode], selectedUnicode >= 0)]
 
-  integer, parameter :: &
-    selectedInt8 = selected_int_kind(2), &
-    selectedInt16 = selected_int_kind(4), &
-    selectedInt32 = selected_int_kind(9), &
-    selectedInt64 = selected_int_kind(18),&
-    selectedInt128 = selected_int_kind(38), &
-    safeInt8 = merge(selectedInt8, selected_int_kind(0), &
-                     selectedInt8 >= 0), &
-    safeInt16 = merge(selectedInt16, selected_int_kind(0), &
-                      selectedInt16 >= 0), &
-    safeInt32 = merge(selectedInt32, selected_int_kind(0), &
-                      selectedInt32 >= 0), &
-    safeInt64 = merge(selectedInt64, selected_int_kind(0), &
-                      selectedInt64 >= 0), &
-    safeInt128 = merge(selectedInt128, selected_int_kind(0), &
-                       selectedInt128 >= 0)
-  integer, parameter, public :: &
-    int8 = merge(selectedInt8, merge(-2, -1, selectedInt8 >= 0), &
-                 digits(int(0,kind=safeInt8)) == 7), &
-    int16 = merge(selectedInt16, merge(-2, -1, selectedInt16 >= 0), &
-                  digits(int(0,kind=safeInt16)) == 15), &
-    int32 = merge(selectedInt32, merge(-2, -1, selectedInt32 >= 0), &
-                  digits(int(0,kind=safeInt32)) == 31), &
-    int64 = merge(selectedInt64, merge(-2, -1, selectedInt64 >= 0), &
-                  digits(int(0,kind=safeInt64)) == 63), &
-    int128 = merge(selectedInt128, merge(-2, -1, selectedInt128 >= 0), &
-                   digits(int(0,kind=safeInt128)) == 127)
-
-  integer, parameter, public :: integer_kinds(*) = [ &
-    selected_int_kind(0), &
-    [(pack([selected_int_kind(k)], &
-           selected_int_kind(k) >= 0 .and. &
-             selected_int_kind(k) /= selected_int_kind(k-1)), &
-      integer :: k=1, 39)]]
+  public :: selectedInt8, selectedInt16, selectedInt32, selectedInt64, selectedInt128, &
+    safeInt8, safeInt16, safeInt32, safeInt64, safeInt128, &
+    int8, int16, int32, int64, int128
 
-  integer, parameter, public :: &
-    logical8 = int8, logical16 = int16, logical32 = int32, logical64 = int64
-  integer, parameter, public :: logical_kinds(*) = [ &
-    pack([logical8],  logical8 >= 0), &
-    pack([logical16], logical16 >= 0), &
-    pack([logical32], logical32 >= 0), &
-    pack([logical64], logical64 >= 0)]
+  public :: logical8, logical16, logical32, logical64
 
-  integer, parameter :: &
-    selectedReal16 = selected_real_kind(3, 4), &      ! IEEE half
-    selectedBfloat16 = selected_real_kind(2, 37), &   ! truncated IEEE single
-    selectedReal32 = selected_real_kind(6, 37), &     ! IEEE single
-    selectedReal64 = selected_real_kind(15, 307), &   ! IEEE double
-    selectedReal80 = selected_real_kind(18, 4931), &  ! 80x87 extended
-    selectedReal64x2 = selected_real_kind(31, 307), & ! "double-double"
-    selectedReal128 = selected_real_kind(33, 4931), & ! IEEE quad
-    safeReal16 = merge(selectedReal16, selected_real_kind(0,0), &
-                       selectedReal16 >= 0), &
-    safeBfloat16 = merge(selectedBfloat16, selected_real_kind(0,0), &
-                         selectedBfloat16 >= 0), &
-    safeReal32 = merge(selectedReal32, selected_real_kind(0,0), &
-                       selectedReal32 >= 0), &
-    safeReal64 = merge(selectedReal64, selected_real_kind(0,0), &
-                       selectedReal64 >= 0), &
-    safeReal80 = merge(selectedReal80, selected_real_kind(0,0), &
-                       selectedReal80 >= 0), &
-    safeReal64x2 = merge(selectedReal64x2, selected_real_kind(0,0), &
-                         selectedReal64x2 >= 0), &
-    safeReal128 = merge(selectedReal128, selected_real_kind(0,0), &
-                        selectedReal128 >= 0)
-  integer, parameter, public :: &
-    real16 = merge(selectedReal16, merge(-2, -1, selectedReal16 >= 0), &
-                   digits(real(0,kind=safeReal16)) == 11), &
-    bfloat16 = merge(selectedBfloat16, merge(-2, -1, selectedBfloat16 >= 0), &
-                     digits(real(0,kind=safeBfloat16)) == 8), &
-    real32 = merge(selectedReal32, merge(-2, -1, selectedReal32 >= 0), &
-                   digits(real(0,kind=safeReal32)) == 24), &
-    real64 = merge(selectedReal64, merge(-2, -1, selectedReal64 >= 0), &
-                   digits(real(0,kind=safeReal64)) == 53), &
-    real80 = merge(selectedReal80, merge(-2, -1, selectedReal80 >= 0), &
-                   digits(real(0,kind=safeReal80)) == 64), &
-    real64x2 = merge(selectedReal64x2, merge(-2, -1, selectedReal64x2 >= 0), &
-                     digits(real(0,kind=safeReal64x2)) == 106), &
-    real128 = merge(selectedReal128, merge(-2, -1, selectedReal128 >= 0), &
-                    digits(real(0,kind=safeReal128)) == 113)
-
-  integer, parameter, public :: real_kinds(*) = [ &
-    pack([real16], real16 >= 0), &
-    pack([bfloat16], bfloat16 >= 0), &
-    pack([real32], real32 >= 0), &
-    pack([real64], real64 >= 0), &
-    pack([real80], real80 >= 0), &
-    pack([real64x2], real64x2 >= 0), &
-    pack([real128], real128 >= 0)]
+  public :: selectedReal16, selectedBfloat16, selectedReal32, &
+    selectedReal64, selectedReal80, selectedReal64x2, &
+    selectedReal128, &
+    safeReal16, safeBfloat16, safeReal32, &
+    safeReal64, safeReal80, safeReal64x2, &
+    safeReal128, &
+    real16, bfloat16, real32, real64, &
+    real80, real64x2, real128
+
+  public :: integer_kinds, real_kinds, logical_kinds
 
   integer, parameter, public :: current_team = -1, &
     initial_team = -2, &
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang/module/iso_fortran_env_impl.f90
new file mode 100644
index 0000000000000..4de54dda7bab1
--- /dev/null
+++ b/flang/module/iso_fortran_env_impl.f90
@@ -0,0 +1,114 @@
+!===-- module/iso_fortran_env_impl.f90 --=--------------------------------===!
+!
+! Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+! See https://llvm.org/LICENSE.txt for license information.
+! SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+!
+!===------------------------------------------------------------------------===!
+
+! This MODULE implements part of the ISO_FORTRAN_ENV module file, which
+! partially requires linkable symbols for some entities defined
+! (e.g., real_kinds).
+
+module iso_fortran_env_impl
+  implicit none
+  private
+
+  ! INTEGER types
+  integer, parameter, public :: &
+    selectedInt8 = selected_int_kind(2), &
+    selectedInt16 = selected_int_kind(4), &
+    selectedInt32 = selected_int_kind(9), &
+    selectedInt64 = selected_int_kind(18),&
+    selectedInt128 = selected_int_kind(38), &
+    safeInt8 = merge(selectedInt8, selected_int_kind(0), &
+                     selectedInt8 >= 0), &
+    safeInt16 = merge(selectedInt16, selected_int_kind(0), &
+                      selectedInt16 >= 0), &
+    safeInt32 = merge(selectedInt32, selected_int_kind(0), &
+                      selectedInt32 >= 0), &
+    safeInt64 = merge(selectedInt64, selected_int_kind(0), &
+                      selectedInt64 >= 0), &
+    safeInt128 = merge(selectedInt128, selected_int_kind(0), &
+                       selectedInt128 >= 0)
+
+  integer, parameter, public :: &
+    int8 = merge(selectedInt8, merge(-2, -1, selectedInt8 >= 0), &
+                 digits(int(0,kind=safeInt8)) == 7), &
+    int16 = merge(selectedInt16, merge(-2, -1, selectedInt16 >= 0), &
+                  digits(int(0,kind=safeInt16)) == 15), &
+    int32 = merge(selectedInt32, merge(-2, -1, selectedInt32 >= 0), &
+                  digits(int(0,kind=safeInt32)) == 31), &
+    int64 = merge(selectedInt64, merge(-2, -1, selectedInt64 >= 0), &
+                  digits(int(0,kind=safeInt64)) == 63), &
+    int128 = merge(selectedInt128, merge(-2, -1, selectedInt128 >= 0), &
+                   digits(int(0,kind=safeInt128)) == 127)
+
+  integer, parameter, dimension(*), public :: __builtin_integer_kinds = [ &
+      selected_int_kind(0), &
+      [(pack([selected_int_kind(k)], &
+             selected_int_kind(k) >= 0 .and. &
+               selected_int_kind(k) /= selected_int_kind(k-1)), &
+        integer :: k=1, 39)]]
+
+  ! LOGICAL TYPES
+  integer, parameter, public :: &
+    logical8 = int8, logical16 = int16, logical32 = int32, logical64 = int64
+
+  integer, parameter, dimension(*), public :: __builtin_logical_kinds = [ &
+      pack([logical8],  logical8 >= 0), &
+      pack([logical16], logical16 >= 0), &
+      pack([logical32], logical32 >= 0), &
+      pack([logical64], logical64 >= 0) &
+    ]
+
+  ! REAL types
+  integer, parameter, public :: &
+    selectedReal16 = selected_real_kind(3, 4), &      ! IEEE half
+    selectedBfloat16 = selected_real_kind(2, 37), &   ! truncated IEEE single
+    selectedReal32 = selected_real_kind(6, 37), &     ! IEEE single
+    selectedReal64 = selected_real_kind(15, 307), &   ! IEEE double
+    selectedReal80 = selected_real_kind(18, 4931), &  ! 80x87 extended
+    selectedReal64x2 = selected_real_kind(31, 307), & ! "double-double"
+    selectedReal128 = selected_real_kind(33, 4931), & ! IEEE quad
+    safeReal16 = merge(selectedReal16, selected_real_kind(0,0), &
+                       selectedReal16 >= 0), &
+    safeBfloat16 = merge(selectedBfloat16, selected_real_kind(0,0), &
+                         selectedBfloat16 >= 0), &
+    safeReal32 = merge(selectedReal32, selected_real_kind(0,0), &
+                       selectedReal32 >= 0), &
+    safeReal64 = merge(selectedReal64, selected_real_kind(0,0), &
+                       selectedReal64 >= 0), &
+    safeReal80 = merge(selectedReal80, selected_real_kind(0,0), &
+                       selectedReal80 >= 0), &
+    safeReal64x2 = merge(selectedReal64x2, selected_real_kind(0,0), &
+                         selectedReal64x2 >= 0), &
+    safeReal128 = merge(selectedReal128, selected_real_kind(0,0), &
+                        selectedReal128 >= 0)
+
+  integer, parameter, public :: &
+    real16 = merge(selectedReal16, merge(-2, -1, selectedReal16 >= 0), &
+                   digits(real(0,kind=safeReal16)) == 11), &
+    bfloat16 = merge(selectedBfloat16, merge(-2, -1, selectedBfloat16 >= 0), &
+                     digits(real(0,kind=safeBfloat16)) == 8), &
+    real32 = merge(selectedReal32, merge(-2, -1, selectedReal32 >= 0), &
+                   digits(real(0,kind=safeReal32)) == 24), &
+    real64 = merge(selectedReal64, merge(-2, -1, selectedReal64 >= 0), &
+                   digits(real(0,kind=safeReal64)) == 53), &
+    real80 = merge(selectedReal80, merge(-2, -1, selectedReal80 >= 0), &
+                   digits(real(0,kind=safeReal80)) == 64), &
+    real64x2 = merge(selectedReal64x2, merge(-2, -1, selectedReal64x2 >= 0), &
+                     digits(real(0,kind=safeReal64x2)) == 106), &
+    real128 = merge(selectedReal128, merge(-2, -1, selectedReal128 >= 0), &
+                    digits(real(0,kind=safeReal128)) == 113)
+
+  integer, parameter, dimension(*), public :: __builtin_real_kinds = [ &
+      pack([real16], real16 >= 0), &
+      pack([bfloat16], bfloat16 >= 0), &
+      pack([real32], real32 >= 0), &
+      pack([real64], real64 >= 0), &
+      pack([real80], real80 >= 0), &
+      pack([real64x2], real64x2 >= 0), &
+      pack([real128], real128 >= 0) &
+    ]
+end module iso_fortran_env_impl
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index a826980e19411..8588af7e16eec 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -169,6 +169,7 @@ set(sources
   unit-map.cpp
   unit.cpp
   utf.cpp
+  ${FORTRAN_MODULE_OBJECTS}
 )
 
 include(AddFlangOffloadRuntime)
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index 4771199651602..cec4e2d810720 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -4,7 +4,16 @@ set(LLVM_LINK_COMPONENTS
   Support
   )
 
-set(MODULES
+# Define the list of Fortran module files that need to be compiled
+# to produce an object file for inclusion into the FortranRuntime
+# library.
+set(MODULES_WITH_IMPLEMENTATION
+  "iso_fortran_env_impl"
+)
+
+# Define the list of Fortran module files for which it is
+# sufficient to generate the module file via -fsyntax-only.
+set(MODULES_WITHOUT_IMPLEMENTATION
   "__fortran_builtins"
   "__fortran_ieee_exceptions"
   "__fortran_type_info"
@@ -20,6 +29,12 @@ set(MODULES
   "iso_fortran_env"
 )
 
+set(MODULES ${MODULES_WITH_IMPLEMENTATION} ${MODULES_WITHOUT_IMPLEMENTATION})
+
+# Init variable to hold extra object files coming from the Fortran modules;
+# these module files will be contributed from the CMakeLists in flang/tools/f18.
+set(module_objects "")
+
 # Create module files directly from the top-level module source directory.
 # If CMAKE_CROSSCOMPILING, then the newly built flang-new executable was
 # cross compiled, and thus can't be executed on the build system and thus
@@ -41,6 +56,9 @@ if (NOT CMAKE_CROSSCOMPILING)
       if(NOT ${filename} STREQUAL "__fortran_type_info")
         set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
       endif()
+      if(${filename} STREQUAL "iso_fortran_env")
+        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
+      endif()
       if(${filename} STREQUAL "ieee_arithmetic" OR
          ${filename} STREQUAL "ieee_exceptions")
         set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
@@ -58,18 +76,41 @@ if (NOT CMAKE_CROSSCOMPILING)
       endif()
     endif()
 
+
+    # Some modules have an implementation part that needs to be added to the
+    # FortranRuntime library.
+    set(compile_with "-fsyntax-only")
+    set(object_output "")
+    set(include_in_link FALSE)
+    if(${filename} IN_LIST MODULES_WITH_IMPLEMENTATION)
+      set(object_output "${CMAKE_CURRENT_BINARY_DIR}/${filename}${CMAKE_CXX_OUTPUT_EXTENSION}")
+      set(compile_with -c -o ${object_output})
+      set(include_in_link TRUE)
+    endif()
+
     set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
     # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod
+    add_custom_command(OUTPUT ${base}.mod ${object_output}
       COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang-new ${opts} -cpp -fsyntax-only -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
+      COMMAND flang-new ${opts} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
         ${FLANG_SOURCE_DIR}/module/${filename}.f90
       DEPENDS flang-new ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
     )
     list(APPEND MODULE_FILES ${base}.mod)
     install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang")
+
+    # If a module has been compiled into an object file, add the file to
+    # the link line for the FortranRuntime library.
+    if(include_in_link)
+      list(APPEND module_objects ${object_output})
+    endif()
   endforeach()
 
+  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
+  # the compiled Fortran modules can be added to the link line of the FortranRuntime
+  # library.
+  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
+
   # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
   # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
   # files only if OpenMP support has been configured.
diff --git a/flang/tools/flang-driver/CMakeLists.txt b/flang/tools/flang-driver/CMakeLists.txt
index ce30ecff028dc..9f33cdfe3fa90 100644
--- a/flang/tools/flang-driver/CMakeLists.txt
+++ b/flang/tools/flang-driver/CMakeLists.txt
@@ -14,13 +14,6 @@ set( LLVM_LINK_COMPONENTS
 add_flang_tool(flang-new
   driver.cpp
   fc1_main.cpp
-
-  DEPENDS
-  # These libraries are used in the linker invocation generated by the driver
-  # (i.e. when constructing the linker job). Without them the driver would be
-  # unable to generate executables.
-  FortranRuntime
-  FortranDecimal
 )
 
 target_link_libraries(flang-new

From 9ba524427321b931bad156860755adf420aeec6a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 19 Jul 2024 18:48:15 +0100
Subject: [PATCH 223/486] [LV] Add option to still enable the legacy cost
 model. (#99536)

This patch adds a new temporary option to still use the legacy cost
model after https://github.com/llvm/llvm-project/pull/92555. It defaults
to false and the only intended use is to adjust the default to true in
the soon-to-be-cut release branch.

PR: https://github.com/llvm/llvm-project/pull/99536
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 30 ++++++++++++-------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index cceed75aa50b0..ff60bd894cd40 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -205,6 +205,11 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
     "vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
     cl::desc("The maximum allowed number of runtime memory checks"));
 
+static cl::opt<bool> UseLegacyCostModel(
+    "vectorize-use-legacy-cost-model", cl::init(false), cl::Hidden,
+    cl::desc("Use the legacy cost model instead of the VPlan-based cost model. "
+             "This option will be removed in the future."));
+
 // Option prefer-predicate-over-epilogue indicates that an epilogue is undesired,
 // that predication is preferred, and this lists all options. I.e., the
 // vectorizer will try to fold the tail-loop (epilogue) into the vector body
@@ -10319,8 +10324,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
       InnerLoopUnroller Unroller(L, PSE, LI, DT, TLI, TTI, AC, ORE, IC, &LVL,
                                  &CM, BFI, PSI, Checks);
 
-      VPlan &BestPlan = LVP.getBestPlan();
-      assert(BestPlan.hasScalarVFOnly() &&
+      VPlan &BestPlan =
+          UseLegacyCostModel ? LVP.getBestPlanFor(VF.Width) : LVP.getBestPlan();
+      assert((UseLegacyCostModel || BestPlan.hasScalarVFOnly()) &&
              "VPlan cost model and legacy cost model disagreed");
       LVP.executePlan(VF.Width, IC, BestPlan, Unroller, DT, false);
 
@@ -10437,14 +10443,18 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         if (!MainILV.areSafetyChecksAdded())
           DisableRuntimeUnroll = true;
       } else {
-        VPlan &BestPlan = LVP.getBestPlan();
-        assert(size(BestPlan.vectorFactors()) == 1 &&
-               "Plan should have a single VF");
-        ElementCount Width = *BestPlan.vectorFactors().begin();
-        LLVM_DEBUG(dbgs() << "VF picked by VPlan cost model: " << Width
-                          << "\n");
-        assert(VF.Width == Width &&
-               "VPlan cost model and legacy cost model disagreed");
+        ElementCount Width = VF.Width;
+        VPlan &BestPlan =
+            UseLegacyCostModel ? LVP.getBestPlanFor(Width) : LVP.getBestPlan();
+        if (!UseLegacyCostModel) {
+          assert(size(BestPlan.vectorFactors()) == 1 &&
+                 "Plan should have a single VF");
+          Width = *BestPlan.vectorFactors().begin();
+          LLVM_DEBUG(dbgs()
+                     << "VF picked by VPlan cost model: " << Width << "\n");
+          assert(VF.Width == Width &&
+                 "VPlan cost model and legacy cost model disagreed");
+        }
         InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, Width,
                                VF.MinProfitableTripCount, IC, &LVL, &CM, BFI,
                                PSI, Checks);

From a56f37d3bc46f711a1bb8eda6839eb51d1af7d11 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Fri, 19 Jul 2024 19:01:27 +0100
Subject: [PATCH 224/486] [flang][Lower] get ultimate symbol when querying if
 pointer or allocatable (#99528)

This fixes a bug in OpenMP privatisation. The privatised variables are
created as though they are host associated clones of the original
variables. These privatised variables do not contain the allocatable
attribute themselves and so we need to check if the ultimate symbol is
allocatable. Having or not having this flag influences whether lowering
determines that this is a whole allocatable assignment, which then
causes hlfir.assign not to get the realloc flag, which cases the
allocatable not to be allocated when it is assigned to (leading to a
segfault running the newly added test).

I also did the same for pointer variables because I would imagine they
could experience the same issue.

There is no fallout on tests outside of OpenMP, and the gfortran test
suite still passes, so I think this doesn't break host other kinds of
host associated symbols.
---
 flang/lib/Lower/Allocatable.cpp               |  4 +--
 .../Lower/OpenMP/firstprivate-allocatable.f90 | 25 +++++++++++++++++++
 ...oop-reduction-allocatable-array-minmax.f90 |  6 ++---
 .../OpenMP/wsloop-reduction-allocatable.f90   |  4 +--
 4 files changed, 30 insertions(+), 9 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/firstprivate-allocatable.f90

diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp
index 77e02898ac9fb..f40c15db93db8 100644
--- a/flang/lib/Lower/Allocatable.cpp
+++ b/flang/lib/Lower/Allocatable.cpp
@@ -1104,14 +1104,14 @@ void Fortran::lower::associateMutableBox(
 bool Fortran::lower::isWholeAllocatable(const Fortran::lower::SomeExpr &expr) {
   if (const Fortran::semantics::Symbol *sym =
           Fortran::evaluate::UnwrapWholeSymbolOrComponentDataRef(expr))
-    return Fortran::semantics::IsAllocatable(*sym);
+    return Fortran::semantics::IsAllocatable(sym->GetUltimate());
   return false;
 }
 
 bool Fortran::lower::isWholePointer(const Fortran::lower::SomeExpr &expr) {
   if (const Fortran::semantics::Symbol *sym =
           Fortran::evaluate::UnwrapWholeSymbolOrComponentDataRef(expr))
-    return Fortran::semantics::IsPointer(*sym);
+    return Fortran::semantics::IsPointer(sym->GetUltimate());
   return false;
 }
 
diff --git a/flang/test/Lower/OpenMP/firstprivate-allocatable.f90 b/flang/test/Lower/OpenMP/firstprivate-allocatable.f90
new file mode 100644
index 0000000000000..9a16555543cff
--- /dev/null
+++ b/flang/test/Lower/OpenMP/firstprivate-allocatable.f90
@@ -0,0 +1,25 @@
+! RUN: bbc -emit-hlfir -fopenmp -o - %s | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+
+program firstprivateallocatable
+  Integer, Allocatable :: a,u
+  a = 137
+
+  !$omp parallel firstprivate(a,u)
+  u = a**2
+  !$omp end parallel
+end program
+
+
+! CHECK-LABEL:   func.func @_QQmain()
+! [...]
+! CHECK:           omp.parallel {
+! [...]
+! CHECK:             %[[VAL_50:.*]] = arith.constant 2 : i32
+! CHECK:             %[[VAL_51:.*]] = math.ipowi %{{.*}}, %[[VAL_50]] : i32
+! this is what we are really checking: the hlfir.assign must have realloc so that
+! u is allocated when the assignment occurs
+! CHECK:             hlfir.assign %[[VAL_51]] to %{{.*}}#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
+! [...]
+! CHECK:             omp.terminator
+! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index bc22c8b05b967..cda7593b217ab 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -244,8 +244,7 @@ program reduce15
 ! CHECK:                   %[[VAL_85:.*]] = arith.select %[[VAL_84]], %[[VAL_76]], %[[VAL_83]] : i32
 ! CHECK:                   hlfir.yield_element %[[VAL_85]] : i32
 ! CHECK:                 }
-! CHECK:                 %[[VAL_86:.*]] = fir.load %[[VAL_62]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:                 hlfir.assign %[[VAL_68]] to %[[VAL_86]] : !hlfir.expr<?xi32>, !fir.box<!fir.heap<!fir.array<?xi32>>>
+! CHECK:                 hlfir.assign %[[VAL_68]] to %[[VAL_62]]#0 realloc : !hlfir.expr<?xi32>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:                 hlfir.destroy %[[VAL_68]] : !hlfir.expr<?xi32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
@@ -288,8 +287,7 @@ program reduce15
 ! CHECK:                   %[[VAL_117:.*]] = arith.select %[[VAL_116]], %[[VAL_108]], %[[VAL_115]] : i32
 ! CHECK:                   hlfir.yield_element %[[VAL_117]] : i32
 ! CHECK:                 }
-! CHECK:                 %[[VAL_118:.*]] = fir.load %[[VAL_94]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:                 hlfir.assign %[[VAL_100]] to %[[VAL_118]] : !hlfir.expr<?xi32>, !fir.box<!fir.heap<!fir.array<?xi32>>>
+! CHECK:                 hlfir.assign %[[VAL_100]] to %[[VAL_94]]#0 realloc : !hlfir.expr<?xi32>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:                 hlfir.destroy %[[VAL_100]] : !hlfir.expr<?xi32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
index 13858ff2f34e6..3c5388b7e5d90 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
@@ -85,9 +85,7 @@ program reduce
 ! CHECK:                 %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_14]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEr"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
 ! CHECK:                 fir.store %[[VAL_15]] to %[[VAL_10]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_17:.*]] = fir.load %[[VAL_10]]#0 : !fir.ref<i32>
-! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_16]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
-! CHECK:                 %[[VAL_19:.*]] = fir.box_addr %[[VAL_18]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
-! CHECK:                 hlfir.assign %[[VAL_17]] to %[[VAL_19]] : i32, !fir.heap<i32>
+! CHECK:                 hlfir.assign %[[VAL_17]] to %[[VAL_16]]#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK:                 omp.yield
 ! CHECK:               }
 ! CHECK:               omp.terminator

From 54dab7dfcfdffe7bd8697737fbd65fda8385d77e Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Fri, 19 Jul 2024 11:02:57 -0700
Subject: [PATCH 225/486] [msan] Implement support for Arm NEON vst{2,3,4}
 instructions (#99360)

This adds support for vst{2,3,4}, which are not correctly handled by
handleUnknownIntrinsic/handleVector{Load,Store}Intrinsic.

This patch also updates the tests introduced in
https://github.com/llvm/llvm-project/pull/98247 and
https://github.com/llvm/llvm-project/pull/99555

---------

Co-authored-by: Vitaly Buka <vitalybuka@gmail.com>
---
 .../Instrumentation/MemorySanitizer.cpp       |   80 +
 .../MemorySanitizer/AArch64/neon_vst.ll       | 1864 ++++++++++-------
 .../AArch64/neon_vst_origins.ll               |  244 ++-
 3 files changed, 1367 insertions(+), 821 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 9bda288242299..c979e81ac1a3f 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -181,6 +181,7 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsAArch64.h"
 #include "llvm/IR/IntrinsicsX86.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
@@ -2498,6 +2499,15 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         MSV->setOrigin(I, Origin);
       }
     }
+
+    /// Store the current combined value at the specified origin
+    /// location.
+    void DoneAndStoreOrigin(TypeSize TS, Value *OriginPtr) {
+      if (MSV->MS.TrackOrigins) {
+        assert(Origin);
+        MSV->paintOrigin(IRB, Origin, OriginPtr, TS, kMinOriginAlignment);
+      }
+    }
   };
 
   using ShadowAndOriginCombiner = Combiner<true>;
@@ -3865,6 +3875,69 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOriginForNaryOp(I);
   }
 
+  /// Handle Arm NEON vector store intrinsics (vst{2,3,4}).
+  ///
+  /// Arm NEON vector store intrinsics have the output address (pointer) as the
+  /// last argument, with the initial arguments being the inputs. They return
+  /// void.
+  void handleNEONVectorStoreIntrinsic(IntrinsicInst &I) {
+    IRBuilder<> IRB(&I);
+
+    // Don't use getNumOperands() because it includes the callee
+    int numArgOperands = I.arg_size();
+    assert(numArgOperands >= 1);
+
+    // The last arg operand is the output
+    Value *Addr = I.getArgOperand(numArgOperands - 1);
+    assert(Addr->getType()->isPointerTy());
+
+    if (ClCheckAccessAddress)
+      insertShadowCheck(Addr, &I);
+
+    // Every arg operand, other than the last one, is an input vector
+    IntrinsicInst *ShadowI = cast<IntrinsicInst>(I.clone());
+    for (int i = 0; i < numArgOperands - 1; i++) {
+      assert(isa<FixedVectorType>(I.getArgOperand(i)->getType()));
+      ShadowI->setArgOperand(i, getShadow(&I, i));
+    }
+
+    // MSan's GetShadowTy assumes the LHS is the type we want the shadow for
+    // e.g., for:
+    //     [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+    // we know the type of the output (and its shadow) is <16 x i8>.
+    //
+    // Arm NEON VST is unusual because the last argument is the output address:
+    //     define void @st2_16b(<16 x i8> %A, <16 x i8> %B, ptr %P) {
+    //         call void @llvm.aarch64.neon.st2.v16i8.p0
+    //                   (<16 x i8> [[A]], <16 x i8> [[B]], ptr [[P]])
+    // and we have no type information about P's operand. We must manually
+    // compute the type (<16 x i8> x 2).
+    FixedVectorType *OutputVectorTy = FixedVectorType::get(
+        cast<FixedVectorType>(I.getArgOperand(0)->getType())->getElementType(),
+        cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements() *
+            (numArgOperands - 1));
+    Type *ShadowTy = getShadowTy(OutputVectorTy);
+    Value *ShadowPtr, *OriginPtr;
+    // AArch64 NEON does not need alignment (unless OS requires it)
+    std::tie(ShadowPtr, OriginPtr) =
+        getShadowOriginPtr(Addr, IRB, ShadowTy, Align(1), /*isStore*/ true);
+    ShadowI->setArgOperand(numArgOperands - 1, ShadowPtr);
+    ShadowI->insertAfter(&I);
+
+    if (MS.TrackOrigins) {
+      // TODO: if we modelled the vst* instruction more precisely, we could
+      // more accurately track the origins (e.g., if both inputs are
+      // uninitialized for vst2, we currently blame the second input, even
+      // though part of the output depends only on the first input).
+      OriginCombiner OC(this, IRB);
+      for (int i = 0; i < numArgOperands - 1; i++)
+        OC.Add(I.getArgOperand(i));
+
+      const DataLayout &DL = F.getDataLayout();
+      OC.DoneAndStoreOrigin(DL.getTypeStoreSize(OutputVectorTy), OriginPtr);
+    }
+  }
+
   void visitIntrinsicInst(IntrinsicInst &I) {
     switch (I.getIntrinsicID()) {
     case Intrinsic::uadd_with_overflow:
@@ -4204,6 +4277,13 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       setOrigin(&I, getCleanOrigin());
       break;
 
+    case Intrinsic::aarch64_neon_st2:
+    case Intrinsic::aarch64_neon_st3:
+    case Intrinsic::aarch64_neon_st4: {
+      handleNEONVectorStoreIntrinsic(I);
+      break;
+    }
+
     default:
       if (!handleUnknownIntrinsic(I))
         visitInstruction(I);
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst.ll
index 1c4ca47b60c13..df3bfbaf13bbb 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; Test memory sanitizer instrumentation for Arm NEON VST instructions.
 ;
 ; RUN: opt < %s -passes=msan -S | FileCheck %s
@@ -11,25 +11,25 @@ target triple = "aarch64--linux-android9001"
 ; -----------------------------------------------------------------------------------------------------------------------------------------------
 
 define void @st2_8b(<8 x i8> %A, <8 x i8> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_8b
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i8> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0:![0-9]+]]
-; CHECK:       6:
+;
+;
+; CHECK-LABEL: define void @st2_8b(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> [[A]], <8 x i8> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> %A, <8 x i8> %B, ptr %P)
@@ -37,13 +37,24 @@ define void @st2_8b(<8 x i8> %A, <8 x i8> %B, ptr %P) nounwind sanitize_memory {
 }
 
 define void @st2_8b_undefA(<8 x i8> %A, <8 x i8> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_8b_undefA
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st2_8b_undefA(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> undef, <8 x i8> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP2]], ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> undef, <8 x i8> %B, ptr %P)
@@ -51,13 +62,24 @@ define void @st2_8b_undefA(<8 x i8> %A, <8 x i8> %B, ptr %P) nounwind sanitize_m
 }
 
 define void @st2_8b_undefB(<8 x i8> %A, <8 x i8> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_8b_undefB
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st2_8b_undefB(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> [[A]], <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> %A, <8 x i8> undef, ptr %P)
@@ -65,12 +87,23 @@ define void @st2_8b_undefB(<8 x i8> %A, <8 x i8> %B, ptr %P) nounwind sanitize_m
 }
 
 define void @st2_8b_undefAB(<8 x i8> %A, <8 x i8> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_8b_undefAB
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+;
+;
+; CHECK-LABEL: define void @st2_8b_undefAB(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 193514046488576
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> undef, <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP4]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> undef, <8 x i8> undef, ptr %P)
@@ -78,29 +111,26 @@ define void @st2_8b_undefAB(<8 x i8> %A, <8 x i8> %B, ptr %P) nounwind sanitize_
 }
 
 define void @st3_8b(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_8b
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i8> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+;
+;
+; CHECK-LABEL: define void @st3_8b(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
 ; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> [[A]], <8 x i8> [[B]], <8 x i8> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> [[TMP3]], <8 x i8> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P)
@@ -108,14 +138,25 @@ define void @st3_8b(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwind sani
 }
 
 define void @st3_8b_undefA(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_8b_undefA
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st3_8b_undefA(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> undef, <8 x i8> [[B]], <8 x i8> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP2]], <8 x i8> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> undef, <8 x i8> %B, <8 x i8> %C, ptr %P)
@@ -123,14 +164,25 @@ define void @st3_8b_undefA(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwi
 }
 
 define void @st3_8b_undefB(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_8b_undefB
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st3_8b_undefB(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> [[A]], <8 x i8> undef, <8 x i8> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> %A, <8 x i8> undef, <8 x i8> %C, ptr %P)
@@ -138,19 +190,25 @@ define void @st3_8b_undefB(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwi
 }
 
 define void @st3_8b_undefC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_8b_undefC
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i8> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+;
+;
+; CHECK-LABEL: define void @st3_8b_undefC(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> [[A]], <8 x i8> [[B]], <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> [[TMP3]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> %A, <8 x i8> %B, <8 x i8> undef, ptr %P)
@@ -158,13 +216,24 @@ define void @st3_8b_undefC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwi
 }
 
 define void @st3_8b_undefAB(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_8b_undefAB
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st3_8b_undefAB(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP2]], ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> %C, ptr %P)
@@ -172,13 +241,24 @@ define void @st3_8b_undefAB(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounw
 }
 
 define void @st3_8b_undefAC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_8b_undefAC
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st3_8b_undefAC(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> undef, <8 x i8> [[B]], <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> undef, <8 x i8> %B, <8 x i8> undef, ptr %P)
@@ -186,13 +266,24 @@ define void @st3_8b_undefAC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounw
 }
 
 define void @st3_8b_undefBC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_8b_undefBC
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st3_8b_undefBC(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> [[A]], <8 x i8> undef, <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> %A, <8 x i8> undef, <8 x i8> undef, ptr %P)
@@ -200,12 +291,23 @@ define void @st3_8b_undefBC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounw
 }
 
 define void @st3_8b_undefABC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_8b_undefABC
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+;
+;
+; CHECK-LABEL: define void @st3_8b_undefABC(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 193514046488576
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP4]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> undef, ptr %P)
@@ -213,33 +315,27 @@ define void @st3_8b_undefABC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, ptr %P) noun
 }
 
 define void @st4_8b(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i8> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i8> [[TMP4]] to i64
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
-; CHECK:       10:
+;
+;
+; CHECK-LABEL: define void @st4_8b(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF0]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       11:
+; CHECK:       10:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[A]], <8 x i8> [[B]], <8 x i8> [[C]], <8 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> [[TMP3]], <8 x i8> [[TMP4]], <8 x i8> [[TMP5]], ptr [[TMP8]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P)
@@ -247,15 +343,26 @@ define void @st4_8b(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P)
 }
 
 define void @st4_8b_undefA(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefA
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefA(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> [[B]], <8 x i8> [[C]], <8 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP2]], <8 x i8> [[TMP3]], <8 x i8> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P)
@@ -263,15 +370,26 @@ define void @st4_8b_undefA(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, p
 }
 
 define void @st4_8b_undefB(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefB
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefB(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[A]], <8 x i8> undef, <8 x i8> [[C]], <8 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP3]], <8 x i8> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> %A, <8 x i8> undef, <8 x i8> %C, <8 x i8> %D, ptr %P)
@@ -279,20 +397,26 @@ define void @st4_8b_undefB(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, p
 }
 
 define void @st4_8b_undefC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefC
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefC(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i8> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[A]], <8 x i8> [[B]], <8 x i8> undef, <8 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> [[TMP3]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> %A, <8 x i8> %B, <8 x i8> undef, <8 x i8> %D, ptr %P)
@@ -300,23 +424,26 @@ define void @st4_8b_undefC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, p
 }
 
 define void @st4_8b_undefD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefD
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i8> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefD(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[A]], <8 x i8> [[B]], <8 x i8> [[C]], <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> [[TMP3]], <8 x i8> [[TMP4]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> undef, ptr %P)
@@ -324,14 +451,25 @@ define void @st4_8b_undefD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, p
 }
 
 define void @st4_8b_undefAB(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefAB
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefAB(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> [[C]], <8 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP2]], <8 x i8> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> %C, <8 x i8> %D, ptr %P)
@@ -339,14 +477,25 @@ define void @st4_8b_undefAB(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D,
 }
 
 define void @st4_8b_undefAC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefAC
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefAC(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> [[B]], <8 x i8> undef, <8 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> %B, <8 x i8> undef, <8 x i8> %D, ptr %P)
@@ -354,14 +503,25 @@ define void @st4_8b_undefAC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D,
 }
 
 define void @st4_8b_undefBC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefBC
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefBC(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[A]], <8 x i8> undef, <8 x i8> undef, <8 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> %A, <8 x i8> undef, <8 x i8> undef, <8 x i8> %D, ptr %P)
@@ -369,14 +529,25 @@ define void @st4_8b_undefBC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D,
 }
 
 define void @st4_8b_undefBD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefBD
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefBD(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[A]], <8 x i8> undef, <8 x i8> [[C]], <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP3]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> %A, <8 x i8> undef, <8 x i8> %C, <8 x i8> undef, ptr %P)
@@ -384,13 +555,24 @@ define void @st4_8b_undefBD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D,
 }
 
 define void @st4_8b_undefABC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefABC
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefABC(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> undef, <8 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP2]], ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> undef, <8 x i8> %D, ptr %P)
@@ -398,13 +580,24 @@ define void @st4_8b_undefABC(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D,
 }
 
 define void @st4_8b_undefABD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefABD
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefABD(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> [[C]], <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> %C, <8 x i8> undef, ptr %P)
@@ -412,13 +605,24 @@ define void @st4_8b_undefABD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D,
 }
 
 define void @st4_8b_undefACD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefACD
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefACD(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> [[B]], <8 x i8> undef, <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> %B, <8 x i8> undef, <8 x i8> undef, ptr %P)
@@ -426,13 +630,24 @@ define void @st4_8b_undefACD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D,
 }
 
 define void @st4_8b_undefBCD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefBCD
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefBCD(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[A]], <8 x i8> undef, <8 x i8> undef, <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[TMP2]], <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> %A, <8 x i8> undef, <8 x i8> undef, <8 x i8> undef, ptr %P)
@@ -440,12 +655,23 @@ define void @st4_8b_undefBCD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D,
 }
 
 define void @st4_8b_undefABCD(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8b_undefABCD
-; CHECK-SAME: (<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+;
+;
+; CHECK-LABEL: define void @st4_8b_undefABCD(
+; CHECK-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <8 x i8> [[C:%.*]], <8 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 193514046488576
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> undef, <8 x i8> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, ptr [[TMP4]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> undef, <8 x i8> undef, <8 x i8> undef, <8 x i8> undef, ptr %P)
@@ -459,25 +685,25 @@ declare void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8>, <8 x i8>, <8 x i8>, ptr) n
 declare void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, ptr) nounwind sanitize_memory readonly
 
 define void @st2_16b(<16 x i8> %A, <16 x i8> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_16b
-; CHECK-SAME: (<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
-; CHECK:       6:
+;
+;
+; CHECK-LABEL: define void @st2_16b(
+; CHECK-SAME: <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> [[A]], <16 x i8> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> %A, <16 x i8> %B, ptr %P)
@@ -485,29 +711,26 @@ define void @st2_16b(<16 x i8> %A, <16 x i8> %B, ptr %P) nounwind sanitize_memor
 }
 
 define void @st3_16b(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_16b
-; CHECK-SAME: (<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <16 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+;
+;
+; CHECK-LABEL: define void @st3_16b(
+; CHECK-SAME: <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <16 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
 ; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8> [[A]], <16 x i8> [[B]], <16 x i8> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]], <16 x i8> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, ptr %P)
@@ -515,33 +738,27 @@ define void @st3_16b(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, ptr %P) nounwind
 }
 
 define void @st4_16b(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_16b
-; CHECK-SAME: (<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <16 x i8> [[C:%.*]], <16 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP4]] to i128
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
-; CHECK:       10:
+;
+;
+; CHECK-LABEL: define void @st4_16b(
+; CHECK-SAME: <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <16 x i8> [[C:%.*]], <16 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF0]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       11:
+; CHECK:       10:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> [[A]], <16 x i8> [[B]], <16 x i8> [[C]], <16 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]], <16 x i8> [[TMP4]], <16 x i8> [[TMP5]], ptr [[TMP8]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, ptr %P)
@@ -555,25 +772,25 @@ declare void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8>, <16 x i8>, <16 x i8>, pt
 declare void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, ptr) nounwind sanitize_memory readonly
 
 define void @st2_4h(<4 x i16> %A, <4 x i16> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_4h
-; CHECK-SAME: (<4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
-; CHECK:       6:
+;
+;
+; CHECK-LABEL: define void @st2_4h(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v4i16.p0(<4 x i16> [[A]], <4 x i16> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v4i16.p0(<4 x i16> [[TMP2]], <4 x i16> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v4i16.p0(<4 x i16> %A, <4 x i16> %B, ptr %P)
@@ -581,29 +798,26 @@ define void @st2_4h(<4 x i16> %A, <4 x i16> %B, ptr %P) nounwind sanitize_memory
 }
 
 define void @st3_4h(<4 x i16> %A, <4 x i16> %B, <4 x i16> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_4h
-; CHECK-SAME: (<4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i16> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+;
+;
+; CHECK-LABEL: define void @st3_4h(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i16> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
 ; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v4i16.p0(<4 x i16> [[A]], <4 x i16> [[B]], <4 x i16> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v4i16.p0(<4 x i16> [[TMP2]], <4 x i16> [[TMP3]], <4 x i16> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v4i16.p0(<4 x i16> %A, <4 x i16> %B, <4 x i16> %C, ptr %P)
@@ -611,33 +825,27 @@ define void @st3_4h(<4 x i16> %A, <4 x i16> %B, <4 x i16> %C, ptr %P) nounwind s
 }
 
 define void @st4_4h(<4 x i16> %A, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_4h
-; CHECK-SAME: (<4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i16> [[C:%.*]], <4 x i16> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i16> [[TMP4]] to i64
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
-; CHECK:       10:
+;
+;
+; CHECK-LABEL: define void @st4_4h(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i16> [[C:%.*]], <4 x i16> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF0]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       11:
+; CHECK:       10:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v4i16.p0(<4 x i16> [[A]], <4 x i16> [[B]], <4 x i16> [[C]], <4 x i16> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v4i16.p0(<4 x i16> [[TMP2]], <4 x i16> [[TMP3]], <4 x i16> [[TMP4]], <4 x i16> [[TMP5]], ptr [[TMP8]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v4i16.p0(<4 x i16> %A, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, ptr %P)
@@ -651,25 +859,25 @@ declare void @llvm.aarch64.neon.st3.v4i16.p0(<4 x i16>, <4 x i16>, <4 x i16>, pt
 declare void @llvm.aarch64.neon.st4.v4i16.p0(<4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, ptr) nounwind sanitize_memory readonly
 
 define void @st2_8h(<8 x i16> %A, <8 x i16> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_8h
-; CHECK-SAME: (<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
-; CHECK:       6:
+;
+;
+; CHECK-LABEL: define void @st2_8h(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i16.p0(<8 x i16> [[A]], <8 x i16> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i16.p0(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v8i16.p0(<8 x i16> %A, <8 x i16> %B, ptr %P)
@@ -677,29 +885,26 @@ define void @st2_8h(<8 x i16> %A, <8 x i16> %B, ptr %P) nounwind sanitize_memory
 }
 
 define void @st3_8h(<8 x i16> %A, <8 x i16> %B, <8 x i16> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_8h
-; CHECK-SAME: (<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i16> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+;
+;
+; CHECK-LABEL: define void @st3_8h(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i16> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
 ; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i16.p0(<8 x i16> [[A]], <8 x i16> [[B]], <8 x i16> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i16.p0(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v8i16.p0(<8 x i16> %A, <8 x i16> %B, <8 x i16> %C, ptr %P)
@@ -707,33 +912,27 @@ define void @st3_8h(<8 x i16> %A, <8 x i16> %B, <8 x i16> %C, ptr %P) nounwind s
 }
 
 define void @st4_8h(<8 x i16> %A, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_8h
-; CHECK-SAME: (<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i16> [[C:%.*]], <8 x i16> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
-; CHECK:       10:
+;
+;
+; CHECK-LABEL: define void @st4_8h(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i16> [[C:%.*]], <8 x i16> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF0]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       11:
+; CHECK:       10:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i16.p0(<8 x i16> [[A]], <8 x i16> [[B]], <8 x i16> [[C]], <8 x i16> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i16.p0(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <8 x i16> [[TMP5]], ptr [[TMP8]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v8i16.p0(<8 x i16> %A, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, ptr %P)
@@ -747,25 +946,25 @@ declare void @llvm.aarch64.neon.st3.v8i16.p0(<8 x i16>, <8 x i16>, <8 x i16>, pt
 declare void @llvm.aarch64.neon.st4.v8i16.p0(<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>, ptr) nounwind sanitize_memory readonly
 
 define void @st2_2s(<2 x i32> %A, <2 x i32> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_2s
-; CHECK-SAME: (<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
-; CHECK:       6:
+;
+;
+; CHECK-LABEL: define void @st2_2s(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v2i32.p0(<2 x i32> [[A]], <2 x i32> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v2i32.p0(<2 x i32> [[TMP2]], <2 x i32> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v2i32.p0(<2 x i32> %A, <2 x i32> %B, ptr %P)
@@ -773,29 +972,26 @@ define void @st2_2s(<2 x i32> %A, <2 x i32> %B, ptr %P) nounwind sanitize_memory
 }
 
 define void @st3_2s(<2 x i32> %A, <2 x i32> %B, <2 x i32> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_2s
-; CHECK-SAME: (<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <2 x i32> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+;
+;
+; CHECK-LABEL: define void @st3_2s(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <2 x i32> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
 ; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i32.p0(<2 x i32> [[A]], <2 x i32> [[B]], <2 x i32> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i32.p0(<2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v2i32.p0(<2 x i32> %A, <2 x i32> %B, <2 x i32> %C, ptr %P)
@@ -803,33 +999,27 @@ define void @st3_2s(<2 x i32> %A, <2 x i32> %B, <2 x i32> %C, ptr %P) nounwind s
 }
 
 define void @st4_2s(<2 x i32> %A, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2s
-; CHECK-SAME: (<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <2 x i32> [[C:%.*]], <2 x i32> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i32> [[TMP4]] to i64
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
-; CHECK:       10:
+;
+;
+; CHECK-LABEL: define void @st4_2s(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <2 x i32> [[C:%.*]], <2 x i32> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF0]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       11:
+; CHECK:       10:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i32.p0(<2 x i32> [[A]], <2 x i32> [[B]], <2 x i32> [[C]], <2 x i32> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i32.p0(<2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], ptr [[TMP8]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i32.p0(<2 x i32> %A, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, ptr %P)
@@ -841,25 +1031,25 @@ declare void @llvm.aarch64.neon.st3.v2i32.p0(<2 x i32>, <2 x i32>, <2 x i32>, pt
 declare void @llvm.aarch64.neon.st4.v2i32.p0(<2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, ptr) nounwind sanitize_memory readonly
 
 define void @st2_4s(<4 x i32> %A, <4 x i32> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_4s
-; CHECK-SAME: (<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
-; CHECK:       6:
+;
+;
+; CHECK-LABEL: define void @st2_4s(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32> [[A]], <4 x i32> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32> %A, <4 x i32> %B, ptr %P)
@@ -867,29 +1057,26 @@ define void @st2_4s(<4 x i32> %A, <4 x i32> %B, ptr %P) nounwind sanitize_memory
 }
 
 define void @st3_4s(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_4s
-; CHECK-SAME: (<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+;
+;
+; CHECK-LABEL: define void @st3_4s(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
 ; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C, ptr %P)
@@ -897,33 +1084,27 @@ define void @st3_4s(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C, ptr %P) nounwind s
 }
 
 define void @st4_4s(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_4s
-; CHECK-SAME: (<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]], <4 x i32> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP4]] to i128
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
-; CHECK:       10:
+;
+;
+; CHECK-LABEL: define void @st4_4s(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]], <4 x i32> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF0]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       11:
+; CHECK:       10:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> [[C]], <4 x i32> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], ptr [[TMP8]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, ptr %P)
@@ -938,25 +1119,25 @@ declare void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32>, <4 x i32>, <4 x i32>, <4
 
 ; If there's only one element, st2/3/4 don't make much sense, stick to st1.
 define void @st2_1d(<1 x i64> %A, <1 x i64> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_1d
-; CHECK-SAME: (<1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
-; CHECK:       6:
+;
+;
+; CHECK-LABEL: define void @st2_1d(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v1i64.p0(<1 x i64> [[A]], <1 x i64> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v1i64.p0(<1 x i64> [[TMP2]], <1 x i64> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v1i64.p0(<1 x i64> %A, <1 x i64> %B, ptr %P)
@@ -964,29 +1145,26 @@ define void @st2_1d(<1 x i64> %A, <1 x i64> %B, ptr %P) nounwind sanitize_memory
 }
 
 define void @st3_1d(<1 x i64> %A, <1 x i64> %B, <1 x i64> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_1d
-; CHECK-SAME: (<1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]], <1 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+;
+;
+; CHECK-LABEL: define void @st3_1d(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]], <1 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
 ; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v1i64.p0(<1 x i64> [[A]], <1 x i64> [[B]], <1 x i64> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v1i64.p0(<1 x i64> [[TMP2]], <1 x i64> [[TMP3]], <1 x i64> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v1i64.p0(<1 x i64> %A, <1 x i64> %B, <1 x i64> %C, ptr %P)
@@ -994,33 +1172,27 @@ define void @st3_1d(<1 x i64> %A, <1 x i64> %B, <1 x i64> %C, ptr %P) nounwind s
 }
 
 define void @st4_1d(<1 x i64> %A, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_1d
-; CHECK-SAME: (<1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]], <1 x i64> [[C:%.*]], <1 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[TMP2]] to i64
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP4]] to i64
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
-; CHECK:       10:
+;
+;
+; CHECK-LABEL: define void @st4_1d(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]], <1 x i64> [[C:%.*]], <1 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF0]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       11:
+; CHECK:       10:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v1i64.p0(<1 x i64> [[A]], <1 x i64> [[B]], <1 x i64> [[C]], <1 x i64> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v1i64.p0(<1 x i64> [[TMP2]], <1 x i64> [[TMP3]], <1 x i64> [[TMP4]], <1 x i64> [[TMP5]], ptr [[TMP8]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v1i64.p0(<1 x i64> %A, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, ptr %P)
@@ -1034,25 +1206,25 @@ declare void @llvm.aarch64.neon.st3.v1i64.p0(<1 x i64>, <1 x i64>, <1 x i64>, pt
 declare void @llvm.aarch64.neon.st4.v1i64.p0(<1 x i64>, <1 x i64>, <1 x i64>, <1 x i64>, ptr) nounwind sanitize_memory readonly
 
 define void @st2_2d(<2 x i64> %A, <2 x i64> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_2d
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
-; CHECK:       6:
+;
+;
+; CHECK-LABEL: define void @st2_2d(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> [[A]], <2 x i64> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> %A, <2 x i64> %B, ptr %P)
@@ -1060,13 +1232,24 @@ define void @st2_2d(<2 x i64> %A, <2 x i64> %B, ptr %P) nounwind sanitize_memory
 }
 
 define void @st2_2d_undefA(<2 x i64> %A, <2 x i64> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_2d_undefA
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st2_2d_undefA(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> undef, <2 x i64> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> undef, <2 x i64> %B, ptr %P)
@@ -1074,13 +1257,24 @@ define void @st2_2d_undefA(<2 x i64> %A, <2 x i64> %B, ptr %P) nounwind sanitize
 }
 
 define void @st2_2d_undefB(<2 x i64> %A, <2 x i64> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_2d_undefB
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st2_2d_undefB(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> [[A]], <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> %A, <2 x i64> undef, ptr %P)
@@ -1088,12 +1282,23 @@ define void @st2_2d_undefB(<2 x i64> %A, <2 x i64> %B, ptr %P) nounwind sanitize
 }
 
 define void @st2_2d_undefAB(<2 x i64> %A, <2 x i64> %B, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st2_2d_undefAB
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+;
+;
+; CHECK-LABEL: define void @st2_2d_undefAB(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 193514046488576
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> undef, <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, ptr [[TMP4]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> undef, <2 x i64> undef, ptr %P)
@@ -1101,29 +1306,26 @@ define void @st2_2d_undefAB(<2 x i64> %A, <2 x i64> %B, ptr %P) nounwind sanitiz
 }
 
 define void @st3_2d(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_2d
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+;
+;
+; CHECK-LABEL: define void @st3_2d(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
 ; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> [[A]], <2 x i64> [[B]], <2 x i64> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P)
@@ -1131,14 +1333,25 @@ define void @st3_2d(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nounwind s
 }
 
 define void @st3_2d_undefA(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_2d_undefA
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st3_2d_undefA(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> undef, <2 x i64> [[B]], <2 x i64> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> undef, <2 x i64> %B, <2 x i64> %C, ptr %P)
@@ -1146,14 +1359,25 @@ define void @st3_2d_undefA(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nou
 }
 
 define void @st3_2d_undefB(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_2d_undefB
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st3_2d_undefB(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> [[A]], <2 x i64> undef, <2 x i64> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> %A, <2 x i64> undef, <2 x i64> %C, ptr %P)
@@ -1161,19 +1385,25 @@ define void @st3_2d_undefB(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nou
 }
 
 define void @st3_2d_undefC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_2d_undefC
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+;
+;
+; CHECK-LABEL: define void @st3_2d_undefC(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> [[A]], <2 x i64> [[B]], <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> <i64 -1, i64 -1>, ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> %A, <2 x i64> %B, <2 x i64> undef, ptr %P)
@@ -1181,13 +1411,24 @@ define void @st3_2d_undefC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nou
 }
 
 define void @st3_2d_undefAB(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_2d_undefAB
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st3_2d_undefAB(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> %C, ptr %P)
@@ -1195,13 +1436,24 @@ define void @st3_2d_undefAB(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) no
 }
 
 define void @st3_2d_undefAC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_2d_undefAC
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st3_2d_undefAC(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> undef, <2 x i64> [[B]], <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> undef, <2 x i64> %B, <2 x i64> undef, ptr %P)
@@ -1209,13 +1461,24 @@ define void @st3_2d_undefAC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) no
 }
 
 define void @st3_2d_undefBC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_2d_undefBC
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st3_2d_undefBC(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> [[A]], <2 x i64> undef, <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> %A, <2 x i64> undef, <2 x i64> undef, ptr %P)
@@ -1223,12 +1486,23 @@ define void @st3_2d_undefBC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) no
 }
 
 define void @st3_2d_undefABC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st3_2d_undefABC
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+;
+;
+; CHECK-LABEL: define void @st3_2d_undefABC(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 193514046488576
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, ptr [[TMP4]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> undef, ptr %P)
@@ -1236,33 +1510,27 @@ define void @st3_2d_undefABC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, ptr %P) n
 }
 
 define void @st4_2d(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[TMP4]] to i128
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
-; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
-; CHECK:       10:
+;
+;
+; CHECK-LABEL: define void @st4_2d(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF0]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       11:
+; CHECK:       10:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[A]], <2 x i64> [[B]], <2 x i64> [[C]], <2 x i64> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], ptr [[TMP8]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P)
@@ -1274,15 +1542,26 @@ declare void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64>, <2 x i64>, <2 x i64>, pt
 declare void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, ptr) nounwind sanitize_memory readonly
 
 define void @st4_2d_undefA(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefA
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefA(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> [[B]], <2 x i64> [[C]], <2 x i64> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P)
@@ -1290,15 +1569,26 @@ define void @st4_2d_undefA(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %
 }
 
 define void @st4_2d_undefB(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefB
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefB(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[A]], <2 x i64> undef, <2 x i64> [[C]], <2 x i64> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> %A, <2 x i64> undef, <2 x i64> %C, <2 x i64> %D, ptr %P)
@@ -1306,20 +1596,26 @@ define void @st4_2d_undefB(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %
 }
 
 define void @st4_2d_undefC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefC
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefC(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[A]], <2 x i64> [[B]], <2 x i64> undef, <2 x i64> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP4]], ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> %A, <2 x i64> %B, <2 x i64> undef, <2 x i64> %D, ptr %P)
@@ -1327,23 +1623,26 @@ define void @st4_2d_undefC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %
 }
 
 define void @st4_2d_undefD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefD
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefD(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[A]], <2 x i64> [[B]], <2 x i64> [[C]], <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64> <i64 -1, i64 -1>, ptr [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> undef, ptr %P)
@@ -1351,14 +1650,25 @@ define void @st4_2d_undefD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %
 }
 
 define void @st4_2d_undefAB(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefAB
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefAB(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> [[C]], <2 x i64> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> %C, <2 x i64> %D, ptr %P)
@@ -1366,14 +1676,25 @@ define void @st4_2d_undefAB(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64>
 }
 
 define void @st4_2d_undefAC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefAC
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefAC(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> [[B]], <2 x i64> undef, <2 x i64> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> %B, <2 x i64> undef, <2 x i64> %D, ptr %P)
@@ -1381,14 +1702,25 @@ define void @st4_2d_undefAC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64>
 }
 
 define void @st4_2d_undefAD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefAD
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefAD(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> [[B]], <2 x i64> [[C]], <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> <i64 -1, i64 -1>, ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> %B, <2 x i64> %C, <2 x i64> undef, ptr %P)
@@ -1396,14 +1728,25 @@ define void @st4_2d_undefAD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64>
 }
 
 define void @st4_2d_undefBC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefBC
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefBC(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[A]], <2 x i64> undef, <2 x i64> undef, <2 x i64> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP3]], ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> %A, <2 x i64> undef, <2 x i64> undef, <2 x i64> %D, ptr %P)
@@ -1411,14 +1754,25 @@ define void @st4_2d_undefBC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64>
 }
 
 define void @st4_2d_undefBD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefBD
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefBD(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[A]], <2 x i64> undef, <2 x i64> [[C]], <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP3]], <2 x i64> <i64 -1, i64 -1>, ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> %A, <2 x i64> undef, <2 x i64> %C, <2 x i64> undef, ptr %P)
@@ -1426,19 +1780,25 @@ define void @st4_2d_undefBD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64>
 }
 
 define void @st4_2d_undefCD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefCD
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefCD(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[A]], <2 x i64> [[B]], <2 x i64> undef, <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> %A, <2 x i64> %B, <2 x i64> undef, <2 x i64> undef, ptr %P)
@@ -1446,13 +1806,24 @@ define void @st4_2d_undefCD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64>
 }
 
 define void @st4_2d_undefABC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefABC
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefABC(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> undef, <2 x i64> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> undef, <2 x i64> %D, ptr %P)
@@ -1460,13 +1831,24 @@ define void @st4_2d_undefABC(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64>
 }
 
 define void @st4_2d_undefABD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefABD
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefABD(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> [[C]], <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> %C, <2 x i64> undef, ptr %P)
@@ -1474,13 +1856,24 @@ define void @st4_2d_undefABD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64>
 }
 
 define void @st4_2d_undefACD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefACD
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefACD(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> [[B]], <2 x i64> undef, <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> %B, <2 x i64> undef, <2 x i64> undef, ptr %P)
@@ -1488,13 +1881,24 @@ define void @st4_2d_undefACD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64>
 }
 
 define void @st4_2d_undefBCD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefBCD
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefBCD(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 193514046488576
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[A]], <2 x i64> undef, <2 x i64> undef, <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[TMP2]], <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, ptr [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> %A, <2 x i64> undef, <2 x i64> undef, <2 x i64> undef, ptr %P)
@@ -1502,14 +1906,28 @@ define void @st4_2d_undefBCD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64>
 }
 
 define void @st4_2d_undefABCD(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, ptr %P) nounwind sanitize_memory {
-; CHECK-LABEL: define void @st4_2d_undefABCD
-; CHECK-SAME: (<2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+;
+;
+; CHECK-LABEL: define void @st4_2d_undefABCD(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i64> [[C:%.*]], <2 x i64> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 193514046488576
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> undef, <2 x i64> undef, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>, ptr [[TMP4]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> undef, <2 x i64> undef, <2 x i64> undef, <2 x i64> undef, ptr %P)
   ret void
 }
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst_origins.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst_origins.ll
index ff4c4c24bf37a..818da89e3eccd 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst_origins.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vst_origins.ll
@@ -14,36 +14,48 @@ target triple = "aarch64--linux-android9001"
 
 define void @st2_16b(<16 x i8> %A, <16 x i8> %B, ptr %P) nounwind sanitize_memory {
 ;
+;
 ; CHECK-LABEL: define void @st2_16b
 ; CHECK-SAME: (<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 16) to ptr), align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 32) to ptr), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 32) to ptr), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 16) to ptr), align 4
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0:![0-9]+]]
-; CHECK:       8:
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP8]], 35184372088832
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], -4
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i8> [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP6]], i32 [[TMP4]]
+; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[TMP12]], i32 1
+; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP16]], align 4
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[TMP12]], i32 2
+; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP17]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i32, ptr [[TMP12]], i32 3
+; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP18]], align 4
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i32, ptr [[TMP12]], i32 4
+; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP19]], align 4
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i32, ptr [[TMP12]], i32 5
+; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP20]], align 4
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i32, ptr [[TMP12]], i32 6
+; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP21]], align 4
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i32, ptr [[TMP12]], i32 7
+; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP22]], align 4
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       23:
 ; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP2]]) #[[ATTR4:[0-9]+]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP10]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF0]]
-; CHECK:       11:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP4]]) #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       12:
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF0]]
-; CHECK:       13:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP6]]) #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       14:
+; CHECK:       24:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> [[A]], <16 x i8> [[B]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> [[TMP3]], <16 x i8> [[TMP5]], ptr [[TMP9]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> %A, <16 x i8> %B, ptr %P)
@@ -52,45 +64,61 @@ define void @st2_16b(<16 x i8> %A, <16 x i8> %B, ptr %P) nounwind sanitize_memor
 
 define void @st3_16b(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, ptr %P) nounwind sanitize_memory {
 ;
+;
 ; CHECK-LABEL: define void @st3_16b
 ; CHECK-SAME: (<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <16 x i8> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 16) to ptr), align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 32) to ptr), align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 48) to ptr), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 48) to ptr), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 16) to ptr), align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 32) to ptr), align 4
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP9]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
-; CHECK:       10:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP2]]) #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       11:
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF0]]
-; CHECK:       13:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP4]]) #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       14:
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 193514046488576
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP10]], 35184372088832
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], -4
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x i8> [[TMP5]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP15]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF0]]
-; CHECK:       16:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP6]]) #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       17:
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
-; CHECK:       18:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP8]]) #[[ATTR4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP6]], i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i8> [[TMP7]] to i128
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne i128 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP8]], i32 [[TMP17]]
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP14]], align 4
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i32, ptr [[TMP14]], i32 1
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP21]], align 4
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i32, ptr [[TMP14]], i32 2
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP22]], align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr i32, ptr [[TMP14]], i32 3
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP23]], align 4
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i32, ptr [[TMP14]], i32 4
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP24]], align 4
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i32, ptr [[TMP14]], i32 5
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i32, ptr [[TMP14]], i32 6
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP26]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i32, ptr [[TMP14]], i32 7
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP27]], align 4
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i32, ptr [[TMP14]], i32 8
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP28]], align 4
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i32, ptr [[TMP14]], i32 9
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP29]], align 4
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i32, ptr [[TMP14]], i32 10
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP30]], align 4
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i32, ptr [[TMP14]], i32 11
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP31]], align 4
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP32:%.*]], label [[TMP33:%.*]], !prof [[PROF0]]
+; CHECK:       32:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP2]]) #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       19:
+; CHECK:       33:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8> [[A]], <16 x i8> [[B]], <16 x i8> [[C]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8> [[TMP3]], <16 x i8> [[TMP5]], <16 x i8> [[TMP7]], ptr [[TMP11]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, ptr %P)
@@ -99,54 +127,74 @@ define void @st3_16b(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, ptr %P) nounwind
 
 define void @st4_16b(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, ptr %P) nounwind sanitize_memory {
 ;
+;
 ; CHECK-LABEL: define void @st4_16b
 ; CHECK-SAME: (<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <16 x i8> [[C:%.*]], <16 x i8> [[D:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 16) to ptr), align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 32) to ptr), align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
-; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 48) to ptr), align 4
-; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 64) to ptr), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 64) to ptr), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 16) to ptr), align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 32) to ptr), align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 48) to ptr), align 4
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP11]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF0]]
-; CHECK:       12:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP2]]) #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       13:
-; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP14]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF0]]
-; CHECK:       15:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP4]]) #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       16:
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP12]], 35184372088832
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], -4
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <16 x i8> [[TMP5]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP17]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
-; CHECK:       18:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP6]]) #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       19:
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne i128 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP6]], i32 [[TMP4]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <16 x i8> [[TMP7]] to i128
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP20]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP21:%.*]], label [[TMP22:%.*]], !prof [[PROF0]]
-; CHECK:       21:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP8]]) #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       22:
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP9]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF0]]
-; CHECK:       23:
-; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP10]]) #[[ATTR4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne i128 [[TMP20]], 0
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP8]], i32 [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <16 x i8> [[TMP9]] to i128
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp ne i128 [[TMP23]], 0
+; CHECK-NEXT:    [[TMP25:%.*]] = select i1 [[TMP24]], i32 [[TMP10]], i32 [[TMP22]]
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP16]], align 4
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i32, ptr [[TMP16]], i32 1
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP26]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i32, ptr [[TMP16]], i32 2
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP27]], align 4
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i32, ptr [[TMP16]], i32 3
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP28]], align 4
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i32, ptr [[TMP16]], i32 4
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP29]], align 4
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i32, ptr [[TMP16]], i32 5
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP30]], align 4
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i32, ptr [[TMP16]], i32 6
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP31]], align 4
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i32, ptr [[TMP16]], i32 7
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP32]], align 4
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i32, ptr [[TMP16]], i32 8
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP33]], align 4
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr i32, ptr [[TMP16]], i32 9
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP34]], align 4
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr i32, ptr [[TMP16]], i32 10
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP35]], align 4
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr i32, ptr [[TMP16]], i32 11
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP36]], align 4
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr i32, ptr [[TMP16]], i32 12
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP37]], align 4
+; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr i32, ptr [[TMP16]], i32 13
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP38]], align 4
+; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr i32, ptr [[TMP16]], i32 14
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP39]], align 4
+; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr i32, ptr [[TMP16]], i32 15
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP40]], align 4
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP41:%.*]], label [[TMP42:%.*]], !prof [[PROF0]]
+; CHECK:       41:
+; CHECK-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP2]]) #[[ATTR4]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       24:
+; CHECK:       42:
 ; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> [[A]], <16 x i8> [[B]], <16 x i8> [[C]], <16 x i8> [[D]], ptr [[P]])
+; CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> [[TMP3]], <16 x i8> [[TMP5]], <16 x i8> [[TMP7]], <16 x i8> [[TMP9]], ptr [[TMP13]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, ptr %P)

From 0f0cfcff2ca65e295cd84d3eda6f8e93b76cb3a8 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 19 Jul 2024 22:09:05 +0400
Subject: [PATCH 226/486] CodeGen: Avoid some references to MachineFunction's
 getMMI (#99652)

MachineFunction's probably should not include a backreference to
the owning MachineModuleInfo. Most of these references were used
just to query the MCContext, which MachineFunction already directly
stores. Other contexts are using it to query the LLVMContext, which
can already be accessed through the IR function reference.
---
 llvm/include/llvm/CodeGen/TailDuplicator.h     |  1 -
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp   |  3 +--
 llvm/lib/CodeGen/MachineInstr.cpp              |  2 +-
 llvm/lib/CodeGen/RegAllocBase.cpp              |  2 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp       | 18 +++++++-----------
 llvm/lib/CodeGen/TailDuplicator.cpp            |  1 -
 .../CodeGen/TargetLoweringObjectFileImpl.cpp   |  2 +-
 llvm/lib/Target/AArch64/AArch64PointerAuth.cpp |  2 +-
 llvm/lib/Target/ARC/ARCFrameLowering.cpp       |  3 +--
 llvm/lib/Target/ARM/ARMFrameLowering.cpp       |  6 ++----
 llvm/lib/Target/ARM/Thumb1FrameLowering.cpp    |  3 +--
 .../Target/Hexagon/HexagonFrameLowering.cpp    |  3 +--
 llvm/lib/Target/M68k/M68kFrameLowering.cpp     |  3 +--
 llvm/lib/Target/MSP430/MSP430FrameLowering.cpp |  3 +--
 llvm/lib/Target/Mips/Mips16FrameLowering.cpp   |  3 +--
 llvm/lib/Target/Mips/MipsSEFrameLowering.cpp   |  3 +--
 llvm/lib/Target/PowerPC/PPCFrameLowering.cpp   |  6 ++----
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp    |  2 +-
 llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp    |  2 +-
 .../Target/SystemZ/SystemZFrameLowering.cpp    |  6 ++----
 .../Target/WebAssembly/WebAssemblyFastISel.cpp |  2 +-
 llvm/lib/Target/X86/X86FrameLowering.cpp       |  9 +++------
 llvm/lib/Target/X86/X86ISelLowering.cpp        |  9 ++++-----
 llvm/lib/Target/X86/X86ISelLoweringCall.cpp    |  2 +-
 llvm/lib/Target/X86/X86PreTileConfig.cpp       |  2 +-
 llvm/lib/Target/XCore/XCoreFrameLowering.cpp   |  3 +--
 llvm/lib/Target/Xtensa/XtensaFrameLowering.cpp |  3 +--
 27 files changed, 39 insertions(+), 65 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TailDuplicator.h b/llvm/include/llvm/CodeGen/TailDuplicator.h
index 8fdce301c0ccb..8b1f67c416c22 100644
--- a/llvm/include/llvm/CodeGen/TailDuplicator.h
+++ b/llvm/include/llvm/CodeGen/TailDuplicator.h
@@ -40,7 +40,6 @@ class TailDuplicator {
   const TargetInstrInfo *TII;
   const TargetRegisterInfo *TRI;
   const MachineBranchProbabilityInfo *MBPI;
-  const MachineModuleInfo *MMI;
   MachineRegisterInfo *MRI;
   MachineFunction *MF;
   MBFIWrapper *MBFI;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 40a691af22748..68a8a273a1b47 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2451,8 +2451,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
 
       int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
       MCSymbol *FrameAllocSym =
-          MF->getMMI().getContext().getOrCreateFrameAllocSymbol(EscapedName,
-                                                                Idx);
+          MF->getContext().getOrCreateFrameAllocSymbol(EscapedName, Idx);
 
       // This should be inserted at the start of the entry block.
       auto LocalEscape =
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 2b083e1dc3210..be64e9c8452f6 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -2216,7 +2216,7 @@ void MachineInstr::emitError(StringRef Msg) const {
 
   if (const MachineBasicBlock *MBB = getParent())
     if (const MachineFunction *MF = MBB->getParent())
-      return MF->getMMI().getModule()->getContext().emitError(LocCookie, Msg);
+      return MF->getFunction().getContext().emitError(LocCookie, Msg);
   report_fatal_error(Msg);
 }
 
diff --git a/llvm/lib/CodeGen/RegAllocBase.cpp b/llvm/lib/CodeGen/RegAllocBase.cpp
index fb18f5a8a884a..60deb62bc908a 100644
--- a/llvm/lib/CodeGen/RegAllocBase.cpp
+++ b/llvm/lib/CodeGen/RegAllocBase.cpp
@@ -130,7 +130,7 @@ void RegAllocBase::allocatePhysRegs() {
         MI->emitError("inline assembly requires more registers than available");
       } else if (MI) {
         LLVMContext &Context =
-            MI->getParent()->getParent()->getMMI().getModule()->getContext();
+            MI->getParent()->getParent()->getFunction().getContext();
         Context.emitError("ran out of registers during register allocation");
       } else {
         report_fatal_error("ran out of registers during register allocation");
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index bbd4c3521d908..98a795edb7a03 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7375,8 +7375,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
   case Intrinsic::codeview_annotation: {
     // Emit a label associated with this metadata.
     MachineFunction &MF = DAG.getMachineFunction();
-    MCSymbol *Label =
-        MF.getMMI().getContext().createTempSymbol("annotation", true);
+    MCSymbol *Label = MF.getContext().createTempSymbol("annotation", true);
     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
     MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
     Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
@@ -7644,9 +7643,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
       assert(FuncInfo.StaticAllocaMap.count(Slot) &&
              "can only escape static allocas");
       int FI = FuncInfo.StaticAllocaMap[Slot];
-      MCSymbol *FrameAllocSym =
-          MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
-              GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
+      MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
+          GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
               TII->get(TargetOpcode::LOCAL_ESCAPE))
           .addSym(FrameAllocSym)
@@ -7665,9 +7663,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
     unsigned IdxVal =
         unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
-    MCSymbol *FrameAllocSym =
-        MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
-            GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
+    MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
+        GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
 
     Value *FP = I.getArgOperand(1);
     SDValue FPVal = getValue(FP);
@@ -8626,7 +8623,7 @@ SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
 
   // Insert a label before the invoke call to mark the try range.  This can be
   // used to detect deletion of the invoke via the MachineModuleInfo.
-  BeginLabel = MMI.getContext().createTempSymbol();
+  BeginLabel = MF.getContext().createTempSymbol();
 
   // For SjLj, keep track of which landing pads go with which invokes
   // so as to maintain the ordering of pads in the LSDA.
@@ -8648,11 +8645,10 @@ SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
   assert(BeginLabel && "BeginLabel should've been set");
 
   MachineFunction &MF = DAG.getMachineFunction();
-  MachineModuleInfo &MMI = MF.getMMI();
 
   // Insert a label at the end of the invoke call to mark the try range.  This
   // can be used to detect deletion of the invoke via the MachineModuleInfo.
-  MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
+  MCSymbol *EndLabel = MF.getContext().createTempSymbol();
   Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
 
   // Inform MachineModuleInfo of range.
diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp
index 6b860466a8875..c5fa4e6211a63 100644
--- a/llvm/lib/CodeGen/TailDuplicator.cpp
+++ b/llvm/lib/CodeGen/TailDuplicator.cpp
@@ -97,7 +97,6 @@ void TailDuplicator::initMF(MachineFunction &MFin, bool PreRegAlloc,
   TII = MF->getSubtarget().getInstrInfo();
   TRI = MF->getSubtarget().getRegisterInfo();
   MRI = &MF->getRegInfo();
-  MMI = &MF->getMMI();
   MBPI = MBPIin;
   MBFI = MBFIin;
   PSI = PSIin;
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 976691b7de448..0d3e4ba5662e0 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -2323,7 +2323,7 @@ bool TargetLoweringObjectFileXCOFF::ShouldSetSSPCanaryBitInTB(
 
 MCSymbol *
 TargetLoweringObjectFileXCOFF::getEHInfoTableSymbol(const MachineFunction *MF) {
-  MCSymbol *EHInfoSym = MF->getMMI().getContext().getOrCreateSymbol(
+  MCSymbol *EHInfoSym = MF->getContext().getOrCreateSymbol(
       "__ehinfo." + Twine(MF->getFunctionNumber()));
   cast<MCSymbolXCOFF>(EHInfoSym)->setEHInfo();
   return EHInfoSym;
diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
index eb0ff73200407..465e689d4a7a5 100644
--- a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
@@ -116,7 +116,7 @@ void AArch64PointerAuth::signLR(MachineFunction &MF,
   // PAuthLR authentication instructions need to know the value of PC at the
   // point of signing (PACI*).
   if (MFnI.branchProtectionPAuthLR()) {
-    MCSymbol *PACSym = MF.getMMI().getContext().createTempSymbol();
+    MCSymbol *PACSym = MF.getContext().createTempSymbol();
     MFnI.setSigningInstrLabel(PACSym);
   }
 
diff --git a/llvm/lib/Target/ARC/ARCFrameLowering.cpp b/llvm/lib/Target/ARC/ARCFrameLowering.cpp
index 6b5802d972eb7..1227fae13211a 100644
--- a/llvm/lib/Target/ARC/ARCFrameLowering.cpp
+++ b/llvm/lib/Target/ARC/ARCFrameLowering.cpp
@@ -116,8 +116,7 @@ void ARCFrameLowering::emitPrologue(MachineFunction &MF,
                                     MachineBasicBlock &MBB) const {
   LLVM_DEBUG(dbgs() << "Emit Prologue: " << MF.getName() << "\n");
   auto *AFI = MF.getInfo<ARCFunctionInfo>();
-  MachineModuleInfo &MMI = MF.getMMI();
-  MCContext &Context = MMI.getContext();
+  MCContext &Context = MF.getContext();
   const MCRegisterInfo *MRI = Context.getRegisterInfo();
   const ARCInstrInfo *TII = MF.getSubtarget<ARCSubtarget>().getInstrInfo();
   MachineBasicBlock::iterator MBBI = MBB.begin();
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index e94b0f6e1a44f..62d01b9f7e90b 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -735,8 +735,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo  &MFI = MF.getFrameInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
-  MachineModuleInfo &MMI = MF.getMMI();
-  MCContext &Context = MMI.getContext();
+  MCContext &Context = MF.getContext();
   const TargetMachine &TM = MF.getTarget();
   const MCRegisterInfo *MRI = Context.getRegisterInfo();
   const ARMBaseRegisterInfo *RegInfo = STI.getRegisterInfo();
@@ -2995,8 +2994,7 @@ void ARMFrameLowering::adjustForSegmentedStacks(
     report_fatal_error("Segmented stacks not supported on this platform.");
 
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  MachineModuleInfo &MMI = MF.getMMI();
-  MCContext &Context = MMI.getContext();
+  MCContext &Context = MF.getContext();
   const MCRegisterInfo *MRI = Context.getRegisterInfo();
   const ARMBaseInstrInfo &TII =
       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
index e908f1fb95124..cb9ded7dee57b 100644
--- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
+++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
@@ -149,8 +149,7 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
   const ThumbRegisterInfo *RegInfo =
       static_cast<const ThumbRegisterInfo *>(STI.getRegisterInfo());
   const Thumb1InstrInfo &TII =
diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
index f4f84beea734d..6ca18528591af 100644
--- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
@@ -1032,7 +1032,6 @@ void HexagonFrameLowering::insertCFIInstructionsAt(MachineBasicBlock &MBB,
       MachineBasicBlock::iterator At) const {
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  MachineModuleInfo &MMI = MF.getMMI();
   auto &HST = MF.getSubtarget<HexagonSubtarget>();
   auto &HII = *HST.getInstrInfo();
   auto &HRI = *HST.getRegisterInfo();
@@ -1043,7 +1042,7 @@ void HexagonFrameLowering::insertCFIInstructionsAt(MachineBasicBlock &MBB,
   DebugLoc DL;
   const MCInstrDesc &CFID = HII.get(TargetOpcode::CFI_INSTRUCTION);
 
-  MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
+  MCSymbol *FrameLabel = MF.getContext().createTempSymbol();
   bool HasFP = hasFP(MF);
 
   if (HasFP) {
diff --git a/llvm/lib/Target/M68k/M68kFrameLowering.cpp b/llvm/lib/Target/M68k/M68kFrameLowering.cpp
index 6347a52e19188..36443f9d33451 100644
--- a/llvm/lib/Target/M68k/M68kFrameLowering.cpp
+++ b/llvm/lib/Target/M68k/M68kFrameLowering.cpp
@@ -452,8 +452,7 @@ void M68kFrameLowering::emitPrologueCalleeSavedFrameMoves(
     const DebugLoc &DL) const {
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
 
   // Add callee saved registers to move list.
   const auto &CSI = MFI.getCalleeSavedInfo();
diff --git a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
index 176387d71fcb6..f4d703ebeeab2 100644
--- a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
@@ -59,8 +59,7 @@ void MSP430FrameLowering::emitCalleeSavedFrameMoves(
     const DebugLoc &DL, bool IsPrologue) const {
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
 
   // Add callee saved registers to move list.
   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
diff --git a/llvm/lib/Target/Mips/Mips16FrameLowering.cpp b/llvm/lib/Target/Mips/Mips16FrameLowering.cpp
index 10c953bb344a8..f72c49eb707db 100644
--- a/llvm/lib/Target/Mips/Mips16FrameLowering.cpp
+++ b/llvm/lib/Target/Mips/Mips16FrameLowering.cpp
@@ -54,8 +54,7 @@ void Mips16FrameLowering::emitPrologue(MachineFunction &MF,
   // No need to allocate space on the stack.
   if (StackSize == 0 && !MFI.adjustsStack()) return;
 
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
 
   // Adjust stack.
   TII.makeFrame(Mips::SP, StackSize, MBB, MBBI);
diff --git a/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp b/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp
index 38f6889a52358..ceb0668c455d6 100644
--- a/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp
@@ -427,8 +427,7 @@ void MipsSEFrameLowering::emitPrologue(MachineFunction &MF,
   // No need to allocate space on the stack.
   if (StackSize == 0 && !MFI.adjustsStack()) return;
 
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
 
   // Adjust stack.
   TII.adjustStackPtr(SP, -StackSize, MBB, MBBI);
diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
index 277d708013c78..1963582ce6863 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -613,8 +613,7 @@ void PPCFrameLowering::emitPrologue(MachineFunction &MF,
   const PPCRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
   const PPCTargetLowering &TLI = *Subtarget.getTargetLowering();
 
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
   DebugLoc dl;
   // AIX assembler does not support cfi directives.
   const bool needsCFI = MF.needsFrameMoves() && !Subtarget.isAIXABI();
@@ -1239,8 +1238,7 @@ void PPCFrameLowering::inlineStackProbe(MachineFunction &MF,
   const PPCTargetLowering &TLI = *Subtarget.getTargetLowering();
   const PPCInstrInfo &TII = *Subtarget.getInstrInfo();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
   // AIX assembler does not support cfi directives.
   const bool needsCFI = MF.needsFrameMoves() && !Subtarget.isAIXABI();
   auto StackAllocMIPos = llvm::find_if(PrologMBB, [](MachineInstr &MI) {
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index a11ab93b8db3c..898d1f80d0564 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -5559,7 +5559,7 @@ static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
       // C-linkage name. A Qualname is returned here because an external
       // function entry point is a csect with XTY_ER property.
       const auto getExternalFunctionEntryPointSymbol = [&](StringRef SymName) {
-        auto &Context = DAG.getMachineFunction().getMMI().getContext();
+        auto &Context = DAG.getMachineFunction().getContext();
         MCSectionXCOFF *Sec = Context.getXCOFFSection(
             (Twine(".") + Twine(SymName)).str(), SectionKind::getMetadata(),
             XCOFF::CsectProperties(XCOFF::XMC_PR, XCOFF::XTY_ER));
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 2344ec529e16d..ae4e039744286 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -196,7 +196,7 @@ static void validateGroupWaitEventsPtr(const SPIRVSubtarget &STI,
   if (!ElemType || ElemType->getOpcode() == SPIRV::OpTypeEvent)
     return;
   // Insert a bitcast before the instruction to keep SPIR-V code valid.
-  LLVMContext &Context = MF->getMMI().getModule()->getContext();
+  LLVMContext &Context = MF->getFunction().getContext();
   SPIRVType *NewPtrType =
       createNewPtrType(GR, I, OpType, false, true, nullptr,
                        TargetExtType::get(Context, "spirv.Event"));
diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
index a769fdeff5684..8c53b8dffc2fa 100644
--- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
@@ -517,8 +517,7 @@ static void buildDefCFAReg(MachineBasicBlock &MBB,
                            const DebugLoc &DL, unsigned Reg,
                            const SystemZInstrInfo *ZII) {
   MachineFunction &MF = *MBB.getParent();
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
   unsigned RegNum = MRI->getDwarfRegNum(Reg, true);
   unsigned CFIIndex = MF.addFrameInst(
                         MCCFIInstruction::createDefCfaRegister(nullptr, RegNum));
@@ -535,8 +534,7 @@ void SystemZELFFrameLowering::emitPrologue(MachineFunction &MF,
   auto *ZII = static_cast<const SystemZInstrInfo *>(STI.getInstrInfo());
   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
   MachineBasicBlock::iterator MBBI = MBB.begin();
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
   const std::vector<CalleeSavedInfo> &CSI = MFFrame.getCalleeSavedInfo();
   bool HasFP = hasFP(MF);
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
index aa3aa1b007a53..ea795cd00ed5b 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
@@ -886,7 +886,7 @@ bool WebAssemblyFastISel::selectCall(const Instruction *I) {
     MIB.addImm(0);
     // The table into which this call_indirect indexes.
     MCSymbolWasm *Table = WebAssembly::getOrCreateFunctionTableSymbol(
-        MF->getMMI().getContext(), Subtarget);
+        MF->getContext(), Subtarget);
     if (Subtarget->hasReferenceTypes()) {
       MIB.addSym(Table);
     } else {
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 8a263e1a0678b..89801783e9280 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -466,8 +466,7 @@ void X86FrameLowering::emitCalleeSavedFrameMovesFullCFA(
     emitCalleeSavedFrameMoves(MBB, MBBI, DebugLoc{}, true);
     return;
   }
-  const MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
   const Register FramePtr = TRI->getFrameRegister(MF);
   const Register MachineFramePtr =
       STI.isTarget64BitILP32() ? Register(getX86SubSuperRegister(FramePtr, 64))
@@ -485,8 +484,7 @@ void X86FrameLowering::emitCalleeSavedFrameMoves(
     const DebugLoc &DL, bool IsPrologue) const {
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
 
   // Add callee saved registers to move list.
@@ -3882,8 +3880,7 @@ bool X86FrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
   // If we may need to emit frameless compact unwind information, give
   // up as this is currently broken: PR25614.
   bool CompactUnwind =
-      MF.getMMI().getContext().getObjectFileInfo()->getCompactUnwindSection() !=
-      nullptr;
+      MF.getContext().getObjectFileInfo()->getCompactUnwindSection() != nullptr;
   return (MF.getFunction().hasFnAttribute(Attribute::NoUnwind) || hasFP(MF) ||
           !CompactUnwind) &&
          // The lowering of segmented stack and HiPE only support entry
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9d742be43408f..890728ba088dc 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -25446,9 +25446,8 @@ static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
 
   // Get an MCSymbol that will ultimately resolve to the frame offset of the EH
   // registration, or the .set_setframe offset.
-  MCSymbol *OffsetSym =
-      MF.getMMI().getContext().getOrCreateParentFrameOffsetSymbol(
-          GlobalValue::dropLLVMManglingEscape(Fn->getName()));
+  MCSymbol *OffsetSym = MF.getContext().getOrCreateParentFrameOffsetSymbol(
+      GlobalValue::dropLLVMManglingEscape(Fn->getName()));
   SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
   SDValue ParentFrameOffset =
       DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal);
@@ -26340,7 +26339,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     MachineFunction &MF = DAG.getMachineFunction();
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
-    auto &Context = MF.getMMI().getContext();
+    auto &Context = MF.getContext();
     MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") +
                                             Twine(MF.getFunctionNumber()));
     return DAG.getNode(getGlobalWrapperKind(nullptr, /*OpFlags=*/0), dl, VT,
@@ -26352,7 +26351,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     MachineFunction &MF = DAG.getMachineFunction();
     SDValue Op1 = Op.getOperand(1);
     auto *Fn = cast<Function>(cast<GlobalAddressSDNode>(Op1)->getGlobal());
-    MCSymbol *LSDASym = MF.getMMI().getContext().getOrCreateLSDASymbol(
+    MCSymbol *LSDASym = MF.getContext().getOrCreateLSDASymbol(
         GlobalValue::dropLLVMManglingEscape(Fn->getName()));
 
     // Generate a simple absolute symbol reference. This intrinsic is only
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 3c49b6ea8ec7b..f659c168b86e0 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -2016,7 +2016,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   bool HasNoCfCheck = (CB && CB->doesNoCfCheck());
   bool IsIndirectCall = (CB && isa<CallInst>(CB) && CB->isIndirectCall());
   bool IsCFICall = IsIndirectCall && CLI.CFIType;
-  const Module *M = MF.getMMI().getModule();
+  const Module *M = MF.getFunction().getParent();
   Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch");
 
   MachineFunction::CallSiteInfo CSInfo;
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index ee5fadf470dde..1d1885a3dcd24 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -45,7 +45,7 @@ using namespace llvm;
 #define DEBUG_TYPE "tile-pre-config"
 
 static void emitErrorMsg(MachineFunction &MF) {
-  LLVMContext &Context = MF.getMMI().getModule()->getContext();
+  LLVMContext &Context = MF.getFunction().getContext();
   Context.emitError(
       MF.getName() +
       ": Failed to config tile register, please define the shape earlier");
diff --git a/llvm/lib/Target/XCore/XCoreFrameLowering.cpp b/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
index 8cb9413f96526..b3753692ac2a0 100644
--- a/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
@@ -225,8 +225,7 @@ void XCoreFrameLowering::emitPrologue(MachineFunction &MF,
   assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  MachineModuleInfo *MMI = &MF.getMMI();
-  const MCRegisterInfo *MRI = MMI->getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
   const XCoreInstrInfo &TII = *MF.getSubtarget<XCoreSubtarget>().getInstrInfo();
   XCoreFunctionInfo *XFI = MF.getInfo<XCoreFunctionInfo>();
   // Debug location must be unknown since the first debug location is used
diff --git a/llvm/lib/Target/Xtensa/XtensaFrameLowering.cpp b/llvm/lib/Target/Xtensa/XtensaFrameLowering.cpp
index 445fd75e29f37..e24cb7714d364 100644
--- a/llvm/lib/Target/Xtensa/XtensaFrameLowering.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaFrameLowering.cpp
@@ -41,8 +41,7 @@ void XtensaFrameLowering::emitPrologue(MachineFunction &MF,
   DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
   MCRegister SP = Xtensa::SP;
   MCRegister FP = TRI->getFrameRegister(MF);
-  MachineModuleInfo &MMI = MF.getMMI();
-  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
 
   // First, compute final stack size.
   uint64_t StackSize = MFI.getStackSize();

From a8b90c835291df9a6c152fd32de666192aa0da77 Mon Sep 17 00:00:00 2001
From: aaryanshukla <53713108+aaryanshukla@users.noreply.github.com>
Date: Fri, 19 Jul 2024 11:18:25 -0700
Subject: [PATCH 227/486] [libc][newheadergen]: adding entry_point testing
 (#99587)

---
 libc/newhdrgen/tests/input/test_small.yaml |  9 +++++++
 libc/newhdrgen/tests/test_integration.py   | 29 +++++++++++++---------
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/libc/newhdrgen/tests/input/test_small.yaml b/libc/newhdrgen/tests/input/test_small.yaml
index f1f9f020ce6a4..a2e96939b4be0 100644
--- a/libc/newhdrgen/tests/input/test_small.yaml
+++ b/libc/newhdrgen/tests/input/test_small.yaml
@@ -54,4 +54,13 @@ functions:
     standards: 
       - stdc
     guard: LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128
+  - name: func_f
+    return_type: _Float16
+    arguments:
+      - type: int
+      - type: double
+      - type: float
+    standards: 
+      - stdc
+
 
diff --git a/libc/newhdrgen/tests/test_integration.py b/libc/newhdrgen/tests/test_integration.py
index 628a37b11c309..33353785af233 100644
--- a/libc/newhdrgen/tests/test_integration.py
+++ b/libc/newhdrgen/tests/test_integration.py
@@ -8,7 +8,6 @@
 
 class TestHeaderGenIntegration(unittest.TestCase):
     def setUp(self):
-
         self.output_dir = Path(
             args.output_dir if args.output_dir else "libc/newhdrgen/tests/output"
         )
@@ -17,19 +16,24 @@ def setUp(self):
 
         self.source_dir = Path(__file__).resolve().parent.parent.parent.parent
 
-    def run_script(self, yaml_file, h_def_file, output_dir):
+    def run_script(self, yaml_file, h_def_file, output_dir, entry_points):
         yaml_file = self.source_dir / yaml_file
         h_def_file = self.source_dir / h_def_file
+        command = [
+            "python3",
+            str(self.source_dir / "libc/newhdrgen/yaml_to_classes.py"),
+            str(yaml_file),
+            "--h_def_file",
+            str(h_def_file),
+            "--output_dir",
+            str(output_dir),
+        ]
+
+        for entry_point in entry_points:
+            command.extend(["--e", entry_point])
+
         result = subprocess.run(
-            [
-                "python3",
-                str(self.source_dir / "libc/newhdrgen/yaml_to_classes.py"),
-                str(yaml_file),
-                "--h_def_file",
-                str(h_def_file),
-                "--output_dir",
-                str(output_dir),
-            ],
+            command,
             capture_output=True,
             text=True,
         )
@@ -53,11 +57,12 @@ def test_generate_header(self):
             self.source_dir / "libc/newhdrgen/tests/expected_output/test_header.h"
         )
         output_file = self.output_dir / "test_small.h"
+        entry_points = {"func_b", "func_a", "func_c", "func_d", "func_e"}
 
         if not self.output_dir.exists():
             self.output_dir.mkdir(parents=True)
 
-        self.run_script(yaml_file, h_def_file, self.output_dir)
+        self.run_script(yaml_file, h_def_file, self.output_dir, entry_points)
 
         self.compare_files(output_file, expected_output_file)
 

From a96c906102e8d0284c7a402eac4fa1ad9ab3e871 Mon Sep 17 00:00:00 2001
From: Med Ismail Bennani <ismail@bennani.ma>
Date: Fri, 19 Jul 2024 11:39:56 -0700
Subject: [PATCH 228/486] [lldb/Target] Add GetStartSymbol method to
 DynamicLoader plugins (#99673)

This patch introduces a new method to the dynamic loader plugin, to
fetch its `start` symbol.

This can be useful to resolve the `start` symbol address for instance.

Signed-off-by: Med Ismail Bennani <ismail@bennani.ma>
---
 lldb/include/lldb/Target/DynamicLoader.h      |  9 ++++++++-
 .../MacOSX-DYLD/DynamicLoaderDarwin.cpp       | 20 +++++++++++++++++++
 .../MacOSX-DYLD/DynamicLoaderDarwin.h         |  2 ++
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/lldb/include/lldb/Target/DynamicLoader.h b/lldb/include/lldb/Target/DynamicLoader.h
index e508d192d2759..15384245194b0 100644
--- a/lldb/include/lldb/Target/DynamicLoader.h
+++ b/lldb/include/lldb/Target/DynamicLoader.h
@@ -10,6 +10,7 @@
 #define LLDB_TARGET_DYNAMICLOADER_H
 
 #include "lldb/Core/PluginInterface.h"
+#include "lldb/Symbol/Symbol.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/UUID.h"
@@ -24,7 +25,6 @@ namespace lldb_private {
 class ModuleList;
 class Process;
 class SectionList;
-class Symbol;
 class SymbolContext;
 class SymbolContextList;
 class Thread;
@@ -329,6 +329,13 @@ class DynamicLoader : public PluginInterface {
   /// safe to call certain APIs or SPIs.
   virtual bool IsFullyInitialized() { return true; }
 
+  /// Return the `start` function \b symbol in the dynamic loader module.
+  /// This is the symbol the process will begin executing with
+  /// `process launch --stop-at-entry`.
+  virtual std::optional<lldb_private::Symbol> GetStartSymbol() {
+    return std::nullopt;
+  }
+
 protected:
   // Utility methods for derived classes
 
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
index 0e17d57fa9c4f..5c6331735bde8 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
@@ -609,6 +609,26 @@ void DynamicLoaderDarwin::UpdateDYLDImageInfoFromNewImageInfo(
   }
 }
 
+std::optional<lldb_private::Symbol> DynamicLoaderDarwin::GetStartSymbol() {
+  Log *log = GetLog(LLDBLog::DynamicLoader);
+
+  auto log_err = [log](llvm::StringLiteral err_msg) -> std::nullopt_t {
+    LLDB_LOGV(log, "{}", err_msg);
+    return std::nullopt;
+  };
+
+  ModuleSP dyld_sp = GetDYLDModule();
+  if (!dyld_sp)
+    return log_err("Couldn't retrieve DYLD module. Cannot get `start` symbol.");
+
+  const Symbol *symbol =
+      dyld_sp->FindFirstSymbolWithNameAndType(ConstString("_dyld_start"));
+  if (!symbol)
+    return log_err("Cannot find `start` symbol in DYLD module.");
+
+  return *symbol;
+}
+
 void DynamicLoaderDarwin::SetDYLDModule(lldb::ModuleSP &dyld_module_sp) {
   m_dyld_module_wp = dyld_module_sp;
 }
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h
index 8f9a29c94173f..4ac55fdf6f3af 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h
@@ -67,6 +67,8 @@ class DynamicLoaderDarwin : public lldb_private::DynamicLoader {
   // Clear method for classes derived from this one
   virtual void DoClear() = 0;
 
+  std::optional<lldb_private::Symbol> GetStartSymbol() override;
+
   void SetDYLDModule(lldb::ModuleSP &dyld_module_sp);
 
   lldb::ModuleSP GetDYLDModule();

From 9b02b75c4d035192d9f24038f70107d7ffbb0f77 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Fri, 19 Jul 2024 20:44:35 +0200
Subject: [PATCH 229/486] [libc] Increase test timeout (#99678)

This patch increases the timeout of the libc test from 1s to 10s, mostly because rv32 runs in a qemu image and sometimes 1s is just not enough for the test to finish when there are other tests running in parallel
---
 libc/test/UnitTest/LibcDeathTestExecutors.cpp | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/libc/test/UnitTest/LibcDeathTestExecutors.cpp b/libc/test/UnitTest/LibcDeathTestExecutors.cpp
index e43dbb19ad152..77b0559c7acea 100644
--- a/libc/test/UnitTest/LibcDeathTestExecutors.cpp
+++ b/libc/test/UnitTest/LibcDeathTestExecutors.cpp
@@ -14,13 +14,18 @@
 
 #include <cassert>
 
+namespace {
+constexpr unsigned TIMEOUT_MS = 10000;
+} // Anonymous namespace
+
 namespace LIBC_NAMESPACE_DECL {
 namespace testing {
 
 bool Test::testProcessKilled(testutils::FunctionCaller *Func, int Signal,
                              const char *LHSStr, const char *RHSStr,
                              internal::Location Loc) {
-  testutils::ProcessStatus Result = testutils::invoke_in_subprocess(Func, 1000);
+  testutils::ProcessStatus Result =
+      testutils::invoke_in_subprocess(Func, TIMEOUT_MS);
 
   if (const char *error = Result.get_error()) {
     Ctx->markFail();
@@ -32,7 +37,7 @@ bool Test::testProcessKilled(testutils::FunctionCaller *Func, int Signal,
   if (Result.timed_out()) {
     Ctx->markFail();
     tlog << Loc;
-    tlog << "Process timed out after " << 1000 << " milliseconds.\n";
+    tlog << "Process timed out after " << TIMEOUT_MS << " milliseconds.\n";
     return false;
   }
 
@@ -63,7 +68,8 @@ bool Test::testProcessKilled(testutils::FunctionCaller *Func, int Signal,
 bool Test::testProcessExits(testutils::FunctionCaller *Func, int ExitCode,
                             const char *LHSStr, const char *RHSStr,
                             internal::Location Loc) {
-  testutils::ProcessStatus Result = testutils::invoke_in_subprocess(Func, 1000);
+  testutils::ProcessStatus Result =
+      testutils::invoke_in_subprocess(Func, TIMEOUT_MS);
 
   if (const char *error = Result.get_error()) {
     Ctx->markFail();
@@ -75,7 +81,7 @@ bool Test::testProcessExits(testutils::FunctionCaller *Func, int ExitCode,
   if (Result.timed_out()) {
     Ctx->markFail();
     tlog << Loc;
-    tlog << "Process timed out after " << 1000 << " milliseconds.\n";
+    tlog << "Process timed out after " << TIMEOUT_MS << " milliseconds.\n";
     return false;
   }
 

From 0ca98af7f96fcc5a1e16c314b3da8eb76c510207 Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Fri, 19 Jul 2024 21:22:33 +0100
Subject: [PATCH 230/486] Dump regs fbsd fix (#99676)

---
 .../lib/safestack/safestack_platform.h        |  2 +
 .../lib/sanitizer_common/sanitizer_linux.cpp  | 45 ++++++++++++++++---
 .../FreeBSD/dump_registers_x86_64.cpp         | 19 ++++++++
 3 files changed, 59 insertions(+), 7 deletions(-)
 create mode 100644 compiler-rt/test/sanitizer_common/TestCases/FreeBSD/dump_registers_x86_64.cpp

diff --git a/compiler-rt/lib/safestack/safestack_platform.h b/compiler-rt/lib/safestack/safestack_platform.h
index 68d26813bef4a..41c7c25fdaf4d 100644
--- a/compiler-rt/lib/safestack/safestack_platform.h
+++ b/compiler-rt/lib/safestack/safestack_platform.h
@@ -143,6 +143,8 @@ inline void *Mmap(void *addr, size_t length, int prot, int flags, int fd,
   return __mmap(addr, length, prot, flags, fd, 0, offset);
 #elif SANITIZER_FREEBSD && (defined(__aarch64__) || defined(__x86_64__))
   return (void *)__syscall(SYS_mmap, addr, length, prot, flags, fd, offset);
+#elif SANITIZER_FREEBSD && (defined(__i386__))
+  return (void *)syscall(SYS_mmap, addr, length, prot, flags, fd, offset);
 #elif SANITIZER_SOLARIS
   return _REAL64(mmap)(addr, length, prot, flags, fd, offset);
 #elif SANITIZER_LINUX_USES_64BIT_SYSCALLS
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index 47f9f0c4590fb..40c46a8949636 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -2121,7 +2121,8 @@ bool SignalContext::IsTrueFaultingAddress() const {
 UNUSED
 static const char *RegNumToRegName(int reg) {
   switch (reg) {
-#  if defined(__x86_64__)
+#  if SANITIZER_LINUX
+#    if defined(__x86_64__)
     case REG_RAX:
       return "rax";
     case REG_RBX:
@@ -2154,7 +2155,7 @@ static const char *RegNumToRegName(int reg) {
       return "r14";
     case REG_R15:
       return "r15";
-#  elif defined(__i386__)
+#    elif defined(__i386__)
     case REG_EAX:
       return "eax";
     case REG_EBX:
@@ -2171,6 +2172,7 @@ static const char *RegNumToRegName(int reg) {
       return "ebp";
     case REG_ESP:
       return "esp";
+#    endif
 #  endif
     default:
       return NULL;
@@ -2178,22 +2180,24 @@ static const char *RegNumToRegName(int reg) {
   return NULL;
 }
 
+#  if SANITIZER_LINUX
 UNUSED
 static void DumpSingleReg(ucontext_t *ctx, int RegNum) {
   const char *RegName = RegNumToRegName(RegNum);
-#  if defined(__x86_64__)
+#    if defined(__x86_64__)
   Printf("%s%s = 0x%016llx  ", internal_strlen(RegName) == 2 ? " " : "",
          RegName, ctx->uc_mcontext.gregs[RegNum]);
-#  elif defined(__i386__)
+#    elif defined(__i386__)
   Printf("%s = 0x%08x  ", RegName, ctx->uc_mcontext.gregs[RegNum]);
-#  else
+#    else
   (void)RegName;
-#  endif
+#    endif
 }
+#  endif
 
 void SignalContext::DumpAllRegisters(void *context) {
-#  if SANITIZER_LINUX
   ucontext_t *ucontext = (ucontext_t *)context;
+#  if SANITIZER_LINUX
 #    if defined(__x86_64__)
   Report("Register values:\n");
   DumpSingleReg(ucontext, REG_RAX);
@@ -2232,8 +2236,35 @@ void SignalContext::DumpAllRegisters(void *context) {
   DumpSingleReg(ucontext, REG_EBP);
   DumpSingleReg(ucontext, REG_ESP);
   Printf("\n");
+#    else
+  (void)ucontext;
 #    endif
+#  elif SANITIZER_FREEBSD
+#    if defined(__x86_64__)
+  Report("Register values:\n");
+  Printf("rax = 0x%016llx  ", ucontext->uc_mcontext.mc_rax);
+  Printf("rbx = 0x%016llx  ", ucontext->uc_mcontext.mc_rbx);
+  Printf("rcx = 0x%016llx  ", ucontext->uc_mcontext.mc_rcx);
+  Printf("rdx = 0x%016llx  ", ucontext->uc_mcontext.mc_rdx);
+  Printf("\n");
+  Printf("rdi = 0x%016llx  ", ucontext->uc_mcontext.mc_rdi);
+  Printf("rsi = 0x%016llx  ", ucontext->uc_mcontext.mc_rsi);
+  Printf("rbp = 0x%016llx  ", ucontext->uc_mcontext.mc_rbp);
+  Printf("rsp = 0x%016llx  ", ucontext->uc_mcontext.mc_rsp);
+  Printf("\n");
+  Printf(" r8 = 0x%016llx  ", ucontext->uc_mcontext.mc_r8);
+  Printf(" r9 = 0x%016llx  ", ucontext->uc_mcontext.mc_r9);
+  Printf("r10 = 0x%016llx  ", ucontext->uc_mcontext.mc_r10);
+  Printf("r11 = 0x%016llx  ", ucontext->uc_mcontext.mc_r11);
+  Printf("\n");
+  Printf("r12 = 0x%016llx  ", ucontext->uc_mcontext.mc_r12);
+  Printf("r13 = 0x%016llx  ", ucontext->uc_mcontext.mc_r13);
+  Printf("r14 = 0x%016llx  ", ucontext->uc_mcontext.mc_r14);
+  Printf("r15 = 0x%016llx  ", ucontext->uc_mcontext.mc_r15);
+  Printf("\n");
+#    else
   (void)ucontext;
+#    endif
 #  endif
   // FIXME: Implement this for other OSes and architectures.
 }
diff --git a/compiler-rt/test/sanitizer_common/TestCases/FreeBSD/dump_registers_x86_64.cpp b/compiler-rt/test/sanitizer_common/TestCases/FreeBSD/dump_registers_x86_64.cpp
new file mode 100644
index 0000000000000..3d11ef0e098f1
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/FreeBSD/dump_registers_x86_64.cpp
@@ -0,0 +1,19 @@
+// Check that sanitizer prints registers dump_registers on dump_registers=1
+// RUN: %clangxx  %s -o %t
+// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-NODUMP --strict-whitespace
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-DUMP --strict-whitespace
+//
+// REQUIRES: x86_64-target-arch
+
+#include <signal.h>
+
+int main() {
+  raise(SIGSEGV);
+  // CHECK-DUMP: Register values
+  // CHECK-DUMP-NEXT: rax = {{0x[0-9a-f]+}}  rbx = {{0x[0-9a-f]+}}  rcx = {{0x[0-9a-f]+}}  rdx = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: rdi = {{0x[0-9a-f]+}}  rsi = {{0x[0-9a-f]+}}  rbp = {{0x[0-9a-f]+}}  rsp = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT:  r8 = {{0x[0-9a-f]+}}   r9 = {{0x[0-9a-f]+}}  r10 = {{0x[0-9a-f]+}}  r11 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: r12 = {{0x[0-9a-f]+}}  r13 = {{0x[0-9a-f]+}}  r14 = {{0x[0-9a-f]+}}  r15 = {{0x[0-9a-f]+}}
+  // CHECK-NODUMP-NOT: Register values
+  return 0;
+}

From 49ef0a8dc2d3b8e9d7c515f5eb350d1cc0336ef0 Mon Sep 17 00:00:00 2001
From: Jorge Gorbe Moya <jgorbe@google.com>
Date: Fri, 19 Jul 2024 13:27:40 -0700
Subject: [PATCH 231/486] Revert "[NVPTX] Add Volta Load/Store Atomics
 (.relaxed, .acquire, .release) and Volatile (.mmio/.volatile) support"
 (#99695)

Reverts llvm/llvm-project#98022
---
 .../NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp   |  28 +-
 llvm/lib/Target/NVPTX/NVPTX.h                 |   8 -
 llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp   | 314 ++----
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td       | 144 +--
 llvm/lib/Target/NVPTX/NVPTXSubtarget.h        |   7 +-
 llvm/test/CodeGen/NVPTX/load-store-sm-70.ll   | 951 ------------------
 llvm/test/CodeGen/NVPTX/load-store.ll         | 553 +---------
 7 files changed, 157 insertions(+), 1848 deletions(-)
 delete mode 100644 llvm/test/CodeGen/NVPTX/load-store-sm-70.ll

diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index a004d64c21cc6..380d878c1f532 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -227,33 +227,9 @@ void NVPTXInstPrinter::printLdStCode(const MCInst *MI, int OpNum,
   if (Modifier) {
     const MCOperand &MO = MI->getOperand(OpNum);
     int Imm = (int) MO.getImm();
-    if (!strcmp(Modifier, "sem")) {
-      switch (Imm) {
-      case NVPTX::PTXLdStInstCode::NotAtomic:
-        break;
-      case NVPTX::PTXLdStInstCode::Volatile:
+    if (!strcmp(Modifier, "volatile")) {
+      if (Imm)
         O << ".volatile";
-        break;
-      case NVPTX::PTXLdStInstCode::Relaxed:
-        O << ".relaxed.sys";
-        break;
-      case NVPTX::PTXLdStInstCode::Acquire:
-        O << ".acquire.sys";
-        break;
-      case NVPTX::PTXLdStInstCode::Release:
-        O << ".release.sys";
-        break;
-      case NVPTX::PTXLdStInstCode::RelaxedMMIO:
-        O << ".mmio.relaxed.sys";
-        break;
-      default:
-        SmallString<256> Msg;
-        raw_svector_ostream OS(Msg);
-        OS << "NVPTX LdStCode Printer does not support \"" << Imm
-           << "\" sem modifier.";
-        report_fatal_error(OS.str());
-        break;
-      }
     } else if (!strcmp(Modifier, "addsp")) {
       switch (Imm) {
       case NVPTX::PTXLdStInstCode::GLOBAL:
diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h
index 3c7167b157025..b0cb24c63c3ce 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.h
+++ b/llvm/lib/Target/NVPTX/NVPTX.h
@@ -107,14 +107,6 @@ enum LoadStore {
 };
 
 namespace PTXLdStInstCode {
-enum MemorySemantic {
-  NotAtomic = 0, // PTX calls these: "Weak"
-  Volatile = 1,
-  Relaxed = 2,
-  Acquire = 3,
-  Release = 4,
-  RelaxedMMIO = 5
-};
 enum AddressSpace {
   GENERIC = 0,
   GLOBAL = 1,
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 371ec8596ef63..11193c11ede3b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -714,170 +714,6 @@ static unsigned int getCodeAddrSpace(MemSDNode *N) {
   return NVPTX::PTXLdStInstCode::GENERIC;
 }
 
-static unsigned int getCodeMemorySemantic(MemSDNode *N,
-                                          const NVPTXSubtarget *Subtarget) {
-  AtomicOrdering Ordering = N->getSuccessOrdering();
-  auto CodeAddrSpace = getCodeAddrSpace(N);
-
-  bool HasMemoryOrdering = Subtarget->hasMemoryOrdering();
-  bool HasRelaxedMMIO = Subtarget->hasRelaxedMMIO();
-
-  // TODO: lowering for SequentiallyConsistent Operations: for now, we error.
-  // TODO: lowering for AcquireRelease Operations: for now, we error.
-  //
-
-  // clang-format off
-
-  // Lowering for non-SequentiallyConsistent Operations
-  //
-  // | Atomic  | Volatile | Statespace         | PTX sm_60- | PTX sm_70+                   |
-  // |---------|----------|--------------------|------------|------------------------------|
-  // | No      | No       | All                | plain      | .weak                        |
-  // | No      | Yes      | Generic,Shared,    | .volatile  | .volatile                    |
-  // |         |          | Global [0]         |            |                              |
-  // | No      | Yes      | Local,Const,Param  | plain [1]  | .weak [1]                    |
-  // | Relaxed | No       | Generic,Shared,    |            |                              |
-  // |         |          | Global [0]         | .volatile  | <atomic sem>                 |
-  // | Other   | No       | Generic,Shared,    | Error [2]  | <atomic sem>                 |
-  // |         |          | Global [0]         |            |                              |
-  // | Yes     | No       | Local,Const,Param  | plain [1]  | .weak [1]                    |
-  // | Relaxed | Yes      | Generic,Shared [0] | .volatile  | .volatile                    |
-  // | Relaxed | Yes      | Global [0]         | .volatile  | .mmio.relaxed.sys (PTX 8.2+) |
-  // |         |          |                    |            |  or .volatile (PTX 8.1-)     |
-  // | Relaxed | Yes      | Local,Const,Param  | plain [1]  | .weak [1]                    |
-  // | Other   | Yes      | Generic, Shared,   | Error [2]  | <atomic sem> [3]             |
-  // |         |          | / Global [0]       |            |                              |
-
-  // clang-format on
-
-  // [0]: volatile and atomics are only supported on global or shared
-  //      memory locations, accessed via generic/shared/global pointers.
-  //      MMIO is only supported on global memory locations,
-  //      accessed via generic/global pointers.
-  // TODO: Implement MMIO access via generic pointer to global.
-  //       Currently implemented for global pointers only.
-
-  // [1]: Lowering volatile/atomic operations to non-volatile/non-atomic
-  //      PTX instructions fails to preserve their C++ side-effects.
-  //
-  //      Example (https://github.com/llvm/llvm-project/issues/62057):
-  //
-  //          void example() {
-  //              std::atomic<bool> True = true;
-  //              while (True.load(std::memory_order_relaxed));
-  //          }
-  //
-  //      A C++ program that calls "example" is well-defined: the infinite loop
-  //      performs an atomic operation. By lowering volatile/atomics to
-  //      "weak" memory operations, we are transforming the above into:
-  //
-  //          void undefined_behavior() {
-  //              bool True = true;
-  //              while (True);
-  //          }
-  //
-  //      which exhibits undefined behavior in both C++ and PTX.
-  //
-  //      Calling "example" in CUDA C++ compiled for sm_60- exhibits undefined
-  //      behavior due to lack of Independent Forward Progress. Lowering these
-  //      to weak memory operations in sm_60- is therefore fine.
-  //
-  //      TODO: lower atomic and volatile operations to memory locations
-  //      in local, const, and param to two PTX instructions in sm_70+:
-  //        - the "weak" memory instruction we are currently lowering to, and
-  //        - some other instruction that preserves the side-effect, e.g.,
-  //          a dead dummy volatile load.
-
-  if (CodeAddrSpace == NVPTX::PTXLdStInstCode::LOCAL ||
-      CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT ||
-      CodeAddrSpace == NVPTX::PTXLdStInstCode::PARAM) {
-    return NVPTX::PTXLdStInstCode::NotAtomic;
-  }
-
-  // [2]: Atomics with Ordering different than Relaxed are not supported on
-  //      sm_60 and older; this includes volatile atomics.
-  if (!(Ordering == AtomicOrdering::NotAtomic ||
-        Ordering == AtomicOrdering::Monotonic) &&
-      !HasMemoryOrdering) {
-    SmallString<256> Msg;
-    raw_svector_ostream OS(Msg);
-    OS << "PTX does not support \"atomic\" for orderings different than"
-          "\"NotAtomic\" or \"Monotonic\" for sm_60 or older, but order is: \""
-       << toIRString(Ordering) << "\".";
-    report_fatal_error(OS.str());
-  }
-
-  // [3]: TODO: these should eventually use .mmio<.atomic sem>; for now we drop
-  // the volatile semantics and preserve the atomic ones.
-
-  // PTX volatile and PTX atomics are not available for statespace that differ
-  // from .generic, .global, or .shared. The behavior of PTX volatile and PTX
-  // atomics is undefined if the generic address does not refer to a .global or
-  // .shared memory location.
-  bool AddrGenericOrGlobalOrShared =
-      (CodeAddrSpace == NVPTX::PTXLdStInstCode::GENERIC ||
-       CodeAddrSpace == NVPTX::PTXLdStInstCode::GLOBAL ||
-       CodeAddrSpace == NVPTX::PTXLdStInstCode::SHARED);
-  bool UseRelaxedMMIO =
-      HasRelaxedMMIO && CodeAddrSpace == NVPTX::PTXLdStInstCode::GLOBAL;
-
-  switch (Ordering) {
-  case AtomicOrdering::NotAtomic:
-    return N->isVolatile() && AddrGenericOrGlobalOrShared
-               ? NVPTX::PTXLdStInstCode::Volatile
-               : NVPTX::PTXLdStInstCode::NotAtomic;
-  case AtomicOrdering::Monotonic:
-    if (N->isVolatile())
-      return UseRelaxedMMIO                ? NVPTX::PTXLdStInstCode::RelaxedMMIO
-             : AddrGenericOrGlobalOrShared ? NVPTX::PTXLdStInstCode::Volatile
-                                           : NVPTX::PTXLdStInstCode::NotAtomic;
-    else
-      return HasMemoryOrdering             ? NVPTX::PTXLdStInstCode::Relaxed
-             : AddrGenericOrGlobalOrShared ? NVPTX::PTXLdStInstCode::Volatile
-                                           : NVPTX::PTXLdStInstCode::NotAtomic;
-  case AtomicOrdering::Acquire:
-    if (!N->readMem()) {
-      SmallString<256> Msg;
-      raw_svector_ostream OS(Msg);
-      OS << "PTX only supports Acquire Ordering on reads: "
-         << N->getOperationName();
-      N->print(OS);
-      report_fatal_error(OS.str());
-    }
-    return AddrGenericOrGlobalOrShared ? NVPTX::PTXLdStInstCode::Acquire
-                                       : NVPTX::PTXLdStInstCode::NotAtomic;
-  case AtomicOrdering::Release:
-    if (!N->writeMem()) {
-      SmallString<256> Msg;
-      raw_svector_ostream OS(Msg);
-      OS << "PTX only supports Release Ordering on writes: "
-         << N->getOperationName();
-      N->print(OS);
-      report_fatal_error(OS.str());
-    }
-    return AddrGenericOrGlobalOrShared ? NVPTX::PTXLdStInstCode::Release
-                                       : NVPTX::PTXLdStInstCode::NotAtomic;
-  case AtomicOrdering::AcquireRelease: {
-    SmallString<256> Msg;
-    raw_svector_ostream OS(Msg);
-    OS << "PTX only supports AcquireRelease Ordering on read-modify-write: "
-       << N->getOperationName();
-    N->print(OS);
-    report_fatal_error(OS.str());
-  }
-  case AtomicOrdering::SequentiallyConsistent:
-  case AtomicOrdering::Unordered:
-    // TODO: support AcquireRelease and SequentiallyConsistent
-    SmallString<256> Msg;
-    raw_svector_ostream OS(Msg);
-    OS << "NVPTX backend does not support AtomicOrdering \""
-       << toIRString(Ordering) << "\" yet.";
-    report_fatal_error(OS.str());
-  }
-
-  llvm_unreachable("unexpected unhandled case");
-}
-
 static bool canLowerToLDG(MemSDNode *N, const NVPTXSubtarget &Subtarget,
                           unsigned CodeAddrSpace, MachineFunction *F) {
   // We use ldg (i.e. ld.global.nc) for invariant loads from the global address
@@ -1080,18 +916,32 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
   if (!LoadedVT.isSimple())
     return false;
 
+  AtomicOrdering Ordering = LD->getSuccessOrdering();
+  // In order to lower atomic loads with stronger guarantees we would need to
+  // use load.acquire or insert fences. However these features were only added
+  // with PTX ISA 6.0 / sm_70.
+  // TODO: Check if we can actually use the new instructions and implement them.
+  if (isStrongerThanMonotonic(Ordering))
+    return false;
+
   // Address Space Setting
   unsigned int CodeAddrSpace = getCodeAddrSpace(LD);
   if (canLowerToLDG(LD, *Subtarget, CodeAddrSpace, MF)) {
     return tryLDGLDU(N);
   }
 
-  // Memory Semantic Setting
-  unsigned int CodeMemorySem = getCodeMemorySemantic(LD, Subtarget);
-
   unsigned int PointerSize =
       CurDAG->getDataLayout().getPointerSizeInBits(LD->getAddressSpace());
 
+  // Volatile Setting
+  // - .volatile is only available for .global and .shared
+  // - .volatile has the same memory synchronization semantics as .relaxed.sys
+  bool isVolatile = LD->isVolatile() || Ordering == AtomicOrdering::Monotonic;
+  if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
+      CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
+      CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
+    isVolatile = false;
+
   // Type Setting: fromType + fromTypeWidth
   //
   // Sign   : ISD::SEXTLOAD
@@ -1132,13 +982,9 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
                              NVPTX::LD_f32_avar, NVPTX::LD_f64_avar);
     if (!Opcode)
       return false;
-    SDValue Ops[] = {getI32Imm(CodeMemorySem, dl),
-                     getI32Imm(CodeAddrSpace, dl),
-                     getI32Imm(vecType, dl),
-                     getI32Imm(fromType, dl),
-                     getI32Imm(fromTypeWidth, dl),
-                     Addr,
-                     Chain};
+    SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
+                      getI32Imm(vecType, dl), getI32Imm(fromType, dl),
+                      getI32Imm(fromTypeWidth, dl), Addr, Chain };
     NVPTXLD = CurDAG->getMachineNode(*Opcode, dl, TargetVT, MVT::Other, Ops);
   } else if (PointerSize == 64 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
                                : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
@@ -1147,14 +993,9 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
                              NVPTX::LD_f32_asi, NVPTX::LD_f64_asi);
     if (!Opcode)
       return false;
-    SDValue Ops[] = {getI32Imm(CodeMemorySem, dl),
-                     getI32Imm(CodeAddrSpace, dl),
-                     getI32Imm(vecType, dl),
-                     getI32Imm(fromType, dl),
-                     getI32Imm(fromTypeWidth, dl),
-                     Base,
-                     Offset,
-                     Chain};
+    SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
+                      getI32Imm(vecType, dl), getI32Imm(fromType, dl),
+                      getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
     NVPTXLD = CurDAG->getMachineNode(*Opcode, dl, TargetVT, MVT::Other, Ops);
   } else if (PointerSize == 64 ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
                                : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
@@ -1169,14 +1010,9 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
                                NVPTX::LD_f32_ari, NVPTX::LD_f64_ari);
     if (!Opcode)
       return false;
-    SDValue Ops[] = {getI32Imm(CodeMemorySem, dl),
-                     getI32Imm(CodeAddrSpace, dl),
-                     getI32Imm(vecType, dl),
-                     getI32Imm(fromType, dl),
-                     getI32Imm(fromTypeWidth, dl),
-                     Base,
-                     Offset,
-                     Chain};
+    SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
+                      getI32Imm(vecType, dl), getI32Imm(fromType, dl),
+                      getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
     NVPTXLD = CurDAG->getMachineNode(*Opcode, dl, TargetVT, MVT::Other, Ops);
   } else {
     if (PointerSize == 64)
@@ -1190,13 +1026,9 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
                                NVPTX::LD_f32_areg, NVPTX::LD_f64_areg);
     if (!Opcode)
       return false;
-    SDValue Ops[] = {getI32Imm(CodeMemorySem, dl),
-                     getI32Imm(CodeAddrSpace, dl),
-                     getI32Imm(vecType, dl),
-                     getI32Imm(fromType, dl),
-                     getI32Imm(fromTypeWidth, dl),
-                     N1,
-                     Chain};
+    SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
+                      getI32Imm(vecType, dl), getI32Imm(fromType, dl),
+                      getI32Imm(fromTypeWidth, dl), N1, Chain };
     NVPTXLD = CurDAG->getMachineNode(*Opcode, dl, TargetVT, MVT::Other, Ops);
   }
 
@@ -1233,8 +1065,13 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
   unsigned int PointerSize =
       CurDAG->getDataLayout().getPointerSizeInBits(MemSD->getAddressSpace());
 
-  // Memory Semantic Setting
-  unsigned int CodeMemorySem = getCodeMemorySemantic(MemSD, Subtarget);
+  // Volatile Setting
+  // - .volatile is only availalble for .global and .shared
+  bool IsVolatile = MemSD->isVolatile();
+  if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
+      CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
+      CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
+    IsVolatile = false;
 
   // Vector Setting
   MVT SimpleVT = LoadedVT.getSimpleVT();
@@ -1301,13 +1138,9 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
     }
     if (!Opcode)
       return false;
-    SDValue Ops[] = {getI32Imm(CodeMemorySem, DL),
-                     getI32Imm(CodeAddrSpace, DL),
-                     getI32Imm(VecType, DL),
-                     getI32Imm(FromType, DL),
-                     getI32Imm(FromTypeWidth, DL),
-                     Addr,
-                     Chain};
+    SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
+                      getI32Imm(VecType, DL), getI32Imm(FromType, DL),
+                      getI32Imm(FromTypeWidth, DL), Addr, Chain };
     LD = CurDAG->getMachineNode(*Opcode, DL, N->getVTList(), Ops);
   } else if (PointerSize == 64
                  ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
@@ -1330,14 +1163,9 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
     }
     if (!Opcode)
       return false;
-    SDValue Ops[] = {getI32Imm(CodeMemorySem, DL),
-                     getI32Imm(CodeAddrSpace, DL),
-                     getI32Imm(VecType, DL),
-                     getI32Imm(FromType, DL),
-                     getI32Imm(FromTypeWidth, DL),
-                     Base,
-                     Offset,
-                     Chain};
+    SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
+                      getI32Imm(VecType, DL), getI32Imm(FromType, DL),
+                      getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
     LD = CurDAG->getMachineNode(*Opcode, DL, N->getVTList(), Ops);
   } else if (PointerSize == 64
                  ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
@@ -1380,14 +1208,9 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
     }
     if (!Opcode)
       return false;
-    SDValue Ops[] = {getI32Imm(CodeMemorySem, DL),
-                     getI32Imm(CodeAddrSpace, DL),
-                     getI32Imm(VecType, DL),
-                     getI32Imm(FromType, DL),
-                     getI32Imm(FromTypeWidth, DL),
-                     Base,
-                     Offset,
-                     Chain};
+    SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
+                      getI32Imm(VecType, DL), getI32Imm(FromType, DL),
+                      getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
 
     LD = CurDAG->getMachineNode(*Opcode, DL, N->getVTList(), Ops);
   } else {
@@ -1430,13 +1253,9 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
     }
     if (!Opcode)
       return false;
-    SDValue Ops[] = {getI32Imm(CodeMemorySem, DL),
-                     getI32Imm(CodeAddrSpace, DL),
-                     getI32Imm(VecType, DL),
-                     getI32Imm(FromType, DL),
-                     getI32Imm(FromTypeWidth, DL),
-                     Op1,
-                     Chain};
+    SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
+                      getI32Imm(VecType, DL), getI32Imm(FromType, DL),
+                      getI32Imm(FromTypeWidth, DL), Op1, Chain };
     LD = CurDAG->getMachineNode(*Opcode, DL, N->getVTList(), Ops);
   }
 
@@ -1879,13 +1698,27 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
   if (!StoreVT.isSimple())
     return false;
 
+  AtomicOrdering Ordering = ST->getSuccessOrdering();
+  // In order to lower atomic loads with stronger guarantees we would need to
+  // use store.release or insert fences. However these features were only added
+  // with PTX ISA 6.0 / sm_70.
+  // TODO: Check if we can actually use the new instructions and implement them.
+  if (isStrongerThanMonotonic(Ordering))
+    return false;
+
   // Address Space Setting
   unsigned int CodeAddrSpace = getCodeAddrSpace(ST);
   unsigned int PointerSize =
       CurDAG->getDataLayout().getPointerSizeInBits(ST->getAddressSpace());
 
-  // Memory Semantic Setting
-  unsigned int CodeMemorySem = getCodeMemorySemantic(ST, Subtarget);
+  // Volatile Setting
+  // - .volatile is only available for .global and .shared
+  // - .volatile has the same memory synchronization semantics as .relaxed.sys
+  bool isVolatile = ST->isVolatile() || Ordering == AtomicOrdering::Monotonic;
+  if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
+      CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
+      CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
+    isVolatile = false;
 
   // Vector Setting
   MVT SimpleVT = StoreVT.getSimpleVT();
@@ -1922,7 +1755,7 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
     if (!Opcode)
       return false;
     SDValue Ops[] = {Value,
-                     getI32Imm(CodeMemorySem, dl),
+                     getI32Imm(isVolatile, dl),
                      getI32Imm(CodeAddrSpace, dl),
                      getI32Imm(vecType, dl),
                      getI32Imm(toType, dl),
@@ -1939,7 +1772,7 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
     if (!Opcode)
       return false;
     SDValue Ops[] = {Value,
-                     getI32Imm(CodeMemorySem, dl),
+                     getI32Imm(isVolatile, dl),
                      getI32Imm(CodeAddrSpace, dl),
                      getI32Imm(vecType, dl),
                      getI32Imm(toType, dl),
@@ -1964,7 +1797,7 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
       return false;
 
     SDValue Ops[] = {Value,
-                     getI32Imm(CodeMemorySem, dl),
+                     getI32Imm(isVolatile, dl),
                      getI32Imm(CodeAddrSpace, dl),
                      getI32Imm(vecType, dl),
                      getI32Imm(toType, dl),
@@ -1986,7 +1819,7 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
     if (!Opcode)
       return false;
     SDValue Ops[] = {Value,
-                     getI32Imm(CodeMemorySem, dl),
+                     getI32Imm(isVolatile, dl),
                      getI32Imm(CodeAddrSpace, dl),
                      getI32Imm(vecType, dl),
                      getI32Imm(toType, dl),
@@ -2025,8 +1858,13 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
   unsigned int PointerSize =
       CurDAG->getDataLayout().getPointerSizeInBits(MemSD->getAddressSpace());
 
-  // Memory Semantic Setting
-  unsigned int CodeMemorySem = getCodeMemorySemantic(MemSD, Subtarget);
+  // Volatile Setting
+  // - .volatile is only availalble for .global and .shared
+  bool IsVolatile = MemSD->isVolatile();
+  if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
+      CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
+      CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
+    IsVolatile = false;
 
   // Type Setting: toType + toTypeWidth
   // - for integer type, always use 'u'
@@ -2068,7 +1906,7 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
     ToTypeWidth = 32;
   }
 
-  StOps.push_back(getI32Imm(CodeMemorySem, DL));
+  StOps.push_back(getI32Imm(IsVolatile, DL));
   StOps.push_back(getI32Imm(CodeAddrSpace, DL));
   StOps.push_back(getI32Imm(VecType, DL));
   StOps.push_back(getI32Imm(ToType, DL));
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 7f1ac8688007e..cd17a9de541ad 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -2958,39 +2958,39 @@ foreach vt = [v2f16, v2bf16, v2i16, v4i8] in {
 multiclass LD<NVPTXRegClass regclass> {
   def _avar : NVPTXInst<
     (outs regclass:$dst),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, imem:$addr),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t$dst, [$addr];", []>;
   def _areg : NVPTXInst<
     (outs regclass:$dst),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int32Regs:$addr),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t$dst, [$addr];", []>;
   def _areg_64 : NVPTXInst<
     (outs regclass:$dst),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int64Regs:$addr),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t$dst, [$addr];", []>;
   def _ari : NVPTXInst<
     (outs regclass:$dst),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int32Regs:$addr, i32imm:$offset),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t$dst, [$addr+$offset];", []>;
   def _ari_64 : NVPTXInst<
     (outs regclass:$dst),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec,
          LdStCode:$Sign, i32imm:$fromWidth, Int64Regs:$addr, i32imm:$offset),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t$dst, [$addr+$offset];", []>;
   def _asi : NVPTXInst<
     (outs regclass:$dst),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec,
          LdStCode:$Sign, i32imm:$fromWidth, imem:$addr, i32imm:$offset),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t$dst, [$addr+$offset];", []>;
 }
 
@@ -3006,39 +3006,39 @@ let mayLoad=1, hasSideEffects=0 in {
 multiclass ST<NVPTXRegClass regclass> {
   def _avar : NVPTXInst<
     (outs),
-    (ins regclass:$src, LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec,
+    (ins regclass:$src, LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec,
          LdStCode:$Sign, i32imm:$toWidth, imem:$addr),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
     " \t[$addr], $src;", []>;
   def _areg : NVPTXInst<
     (outs),
-    (ins regclass:$src, LdStCode:$sem, LdStCode:$addsp,
+    (ins regclass:$src, LdStCode:$isVol, LdStCode:$addsp,
          LdStCode:$Vec, LdStCode:$Sign, i32imm:$toWidth, Int32Regs:$addr),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
     " \t[$addr], $src;", []>;
   def _areg_64 : NVPTXInst<
     (outs),
-    (ins regclass:$src, LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec,
+    (ins regclass:$src, LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec,
          LdStCode:$Sign, i32imm:$toWidth, Int64Regs:$addr),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
     " \t[$addr], $src;", []>;
   def _ari : NVPTXInst<
     (outs),
-    (ins regclass:$src, LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec,
+    (ins regclass:$src, LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec,
          LdStCode:$Sign, i32imm:$toWidth, Int32Regs:$addr, i32imm:$offset),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
     " \t[$addr+$offset], $src;", []>;
   def _ari_64 : NVPTXInst<
     (outs),
-    (ins regclass:$src, LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec,
+    (ins regclass:$src, LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec,
          LdStCode:$Sign, i32imm:$toWidth, Int64Regs:$addr, i32imm:$offset),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
     " \t[$addr+$offset], $src;", []>;
   def _asi : NVPTXInst<
     (outs),
-    (ins regclass:$src, LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec,
+    (ins regclass:$src, LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec,
          LdStCode:$Sign, i32imm:$toWidth, imem:$addr, i32imm:$offset),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$toWidth"
     " \t[$addr+$offset], $src;", []>;
 }
 
@@ -3057,75 +3057,75 @@ let mayStore=1, hasSideEffects=0 in {
 multiclass LD_VEC<NVPTXRegClass regclass> {
   def _v2_avar : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, imem:$addr),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2}}, [$addr];", []>;
   def _v2_areg : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int32Regs:$addr),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2}}, [$addr];", []>;
   def _v2_areg_64 : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int64Regs:$addr),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2}}, [$addr];", []>;
   def _v2_ari : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int32Regs:$addr, i32imm:$offset),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2}}, [$addr+$offset];", []>;
   def _v2_ari_64 : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int64Regs:$addr, i32imm:$offset),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2}}, [$addr+$offset];", []>;
   def _v2_asi : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, imem:$addr, i32imm:$offset),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2}}, [$addr+$offset];", []>;
   def _v4_avar : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, imem:$addr),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr];", []>;
   def _v4_areg : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int32Regs:$addr),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr];", []>;
   def _v4_areg_64 : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int64Regs:$addr),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr];", []>;
   def _v4_ari : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int32Regs:$addr, i32imm:$offset),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr+$offset];", []>;
   def _v4_ari_64 : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int64Regs:$addr, i32imm:$offset),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr+$offset];", []>;
   def _v4_asi : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
-    (ins LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+    (ins LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, imem:$addr, i32imm:$offset),
-    "ld${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "ld${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr+$offset];", []>;
 }
 let mayLoad=1, hasSideEffects=0 in {
@@ -3140,84 +3140,84 @@ let mayLoad=1, hasSideEffects=0 in {
 multiclass ST_VEC<NVPTXRegClass regclass> {
   def _v2_avar : NVPTXInst<
     (outs),
-    (ins regclass:$src1, regclass:$src2, LdStCode:$sem, LdStCode:$addsp,
+    (ins regclass:$src1, regclass:$src2, LdStCode:$isVol, LdStCode:$addsp,
          LdStCode:$Vec, LdStCode:$Sign, i32imm:$fromWidth, imem:$addr),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr], {{$src1, $src2}};", []>;
   def _v2_areg : NVPTXInst<
     (outs),
-    (ins regclass:$src1, regclass:$src2, LdStCode:$sem, LdStCode:$addsp,
+    (ins regclass:$src1, regclass:$src2, LdStCode:$isVol, LdStCode:$addsp,
          LdStCode:$Vec, LdStCode:$Sign, i32imm:$fromWidth, Int32Regs:$addr),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr], {{$src1, $src2}};", []>;
   def _v2_areg_64 : NVPTXInst<
     (outs),
-    (ins regclass:$src1, regclass:$src2, LdStCode:$sem, LdStCode:$addsp,
+    (ins regclass:$src1, regclass:$src2, LdStCode:$isVol, LdStCode:$addsp,
          LdStCode:$Vec, LdStCode:$Sign, i32imm:$fromWidth, Int64Regs:$addr),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr], {{$src1, $src2}};", []>;
   def _v2_ari : NVPTXInst<
     (outs),
-    (ins regclass:$src1, regclass:$src2, LdStCode:$sem, LdStCode:$addsp,
+    (ins regclass:$src1, regclass:$src2, LdStCode:$isVol, LdStCode:$addsp,
          LdStCode:$Vec, LdStCode:$Sign, i32imm:$fromWidth, Int32Regs:$addr,
          i32imm:$offset),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr+$offset], {{$src1, $src2}};", []>;
   def _v2_ari_64 : NVPTXInst<
     (outs),
-    (ins regclass:$src1, regclass:$src2, LdStCode:$sem, LdStCode:$addsp,
+    (ins regclass:$src1, regclass:$src2, LdStCode:$isVol, LdStCode:$addsp,
          LdStCode:$Vec, LdStCode:$Sign, i32imm:$fromWidth, Int64Regs:$addr,
          i32imm:$offset),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr+$offset], {{$src1, $src2}};", []>;
   def _v2_asi : NVPTXInst<
     (outs),
-    (ins regclass:$src1, regclass:$src2, LdStCode:$sem, LdStCode:$addsp,
+    (ins regclass:$src1, regclass:$src2, LdStCode:$isVol, LdStCode:$addsp,
          LdStCode:$Vec, LdStCode:$Sign, i32imm:$fromWidth, imem:$addr,
          i32imm:$offset),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr+$offset], {{$src1, $src2}};", []>;
   def _v4_avar : NVPTXInst<
     (outs),
     (ins regclass:$src1, regclass:$src2, regclass:$src3, regclass:$src4,
-         LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+         LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, imem:$addr),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr], {{$src1, $src2, $src3, $src4}};", []>;
   def _v4_areg : NVPTXInst<
     (outs),
     (ins regclass:$src1, regclass:$src2, regclass:$src3, regclass:$src4,
-         LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+         LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int32Regs:$addr),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr], {{$src1, $src2, $src3, $src4}};", []>;
   def _v4_areg_64 : NVPTXInst<
     (outs),
     (ins regclass:$src1, regclass:$src2, regclass:$src3, regclass:$src4,
-         LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+         LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int64Regs:$addr),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr], {{$src1, $src2, $src3, $src4}};", []>;
   def _v4_ari : NVPTXInst<
     (outs),
     (ins regclass:$src1, regclass:$src2, regclass:$src3, regclass:$src4,
-         LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+         LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int32Regs:$addr, i32imm:$offset),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr+$offset], {{$src1, $src2, $src3, $src4}};", []>;
   def _v4_ari_64 : NVPTXInst<
     (outs),
     (ins regclass:$src1, regclass:$src2, regclass:$src3, regclass:$src4,
-         LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+         LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, Int64Regs:$addr, i32imm:$offset),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}$fromWidth "
     "\t[$addr+$offset], {{$src1, $src2, $src3, $src4}};", []>;
   def _v4_asi : NVPTXInst<
     (outs),
     (ins regclass:$src1, regclass:$src2, regclass:$src3, regclass:$src4,
-         LdStCode:$sem, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
+         LdStCode:$isVol, LdStCode:$addsp, LdStCode:$Vec, LdStCode:$Sign,
          i32imm:$fromWidth, imem:$addr, i32imm:$offset),
-    "st${sem:sem}${addsp:addsp}${Vec:vec}.${Sign:sign}"
+    "st${isVol:volatile}${addsp:addsp}${Vec:vec}.${Sign:sign}"
     "$fromWidth \t[$addr+$offset], {{$src1, $src2, $src3, $src4}};", []>;
 }
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index 8df41913ff12e..3ca4c1a24c79a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -78,18 +78,13 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
   bool hasAtomBitwise64() const { return SmVersion >= 32; }
   bool hasAtomMinMax64() const { return SmVersion >= 32; }
   bool hasLDG() const { return SmVersion >= 32; }
-  bool hasHWROT32() const { return SmVersion >= 32; }
+  inline bool hasHWROT32() const { return SmVersion >= 32; }
   bool hasImageHandles() const;
   bool hasFP16Math() const { return SmVersion >= 53; }
   bool hasBF16Math() const { return SmVersion >= 80; }
   bool allowFP16Math() const;
   bool hasMaskOperator() const { return PTXVersion >= 71; }
   bool hasNoReturn() const { return SmVersion >= 30 && PTXVersion >= 64; }
-  // Does SM & PTX support memory orderings (weak and atomic: relaxed, acquire,
-  // release, acq_rel, sc) ?
-  bool hasMemoryOrdering() const { return SmVersion >= 70 && PTXVersion >= 60; }
-  // Does SM & PTX support atomic relaxed MMIO operations ?
-  bool hasRelaxedMMIO() const { return SmVersion >= 70 && PTXVersion >= 82; }
   unsigned int getFullSmVersion() const { return FullSmVersion; }
   unsigned int getSmVersion() const { return getFullSmVersion() / 10; }
   // GPUs with "a" suffix have include architecture-accelerated features that
diff --git a/llvm/test/CodeGen/NVPTX/load-store-sm-70.ll b/llvm/test/CodeGen/NVPTX/load-store-sm-70.ll
deleted file mode 100644
index 7cdced1778a53..0000000000000
--- a/llvm/test/CodeGen/NVPTX/load-store-sm-70.ll
+++ /dev/null
@@ -1,951 +0,0 @@
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx82 | FileCheck %s
-; RUN: %if ptxas-12.2 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx82 | %ptxas-verify -arch=sm_70 %}
-
-; CHECK-LABEL: generic_plain
-define void @generic_plain(ptr %a, ptr %b, ptr %c, ptr %d) local_unnamed_addr {
-  ; CHECK: ld.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load i8, ptr %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i8 %a.add, ptr %a
-
-  ; CHECK: ld.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load i16, ptr %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i16 %b.add, ptr %b
-
-  ; CHECK: ld.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load i32, ptr %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store i32 %c.add, ptr %c
-
-  ; CHECK: ld.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load i64, ptr %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store i64 %d.add, ptr %d
-
-  ; CHECK: ld.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load float, ptr %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store float %e.add, ptr %c
-
-  ; CHECK: ld.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load double, ptr %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store double %f.add, ptr %c
-
-  ret void
-}
-
-; CHECK-LABEL: generic_volatile
-define void @generic_volatile(ptr %a, ptr %b, ptr %c, ptr %d) local_unnamed_addr {
-  ; CHECK: ld.volatile.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load volatile i8, ptr %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i8 %a.add, ptr %a
-
-  ; CHECK: ld.volatile.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load volatile i16, ptr %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i16 %b.add, ptr %b
-
-  ; CHECK: ld.volatile.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load volatile i32, ptr %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store volatile i32 %c.add, ptr %c
-
-  ; CHECK: ld.volatile.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load volatile i64, ptr %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store volatile i64 %d.add, ptr %d
-
-  ; CHECK: ld.volatile.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load volatile float, ptr %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.volatile.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store volatile float %e.add, ptr %c
-
-  ; CHECK: ld.volatile.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load volatile double, ptr %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store volatile double %f.add, ptr %c
-
-  ret void
-}
-
-; CHECK-LABEL: generic_monotonic
-define void @generic_monotonic(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {
-  ; CHECK: ld.relaxed.sys.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.relaxed.sys.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr %a monotonic, align 1
-
-  ; CHECK: ld.relaxed.sys.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.relaxed.sys.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr %b monotonic, align 2
-
-  ; CHECK: ld.relaxed.sys.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.relaxed.sys.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr %c monotonic, align 4
-
-  ; CHECK: ld.relaxed.sys.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.relaxed.sys.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr %d monotonic, align 8
-
-  ; CHECK: ld.relaxed.sys.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.relaxed.sys.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr %e monotonic, align 4
-
-  ; CHECK: ld.relaxed.sys.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.relaxed.sys.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr %e monotonic, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: generic_acq_rel
-define void @generic_acq_rel(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {
-  ; CHECK: ld.acquire.sys.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr %a acquire, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.release.sys.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr %a release, align 1
-
-  ; CHECK: ld.acquire.sys.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr %b acquire, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.release.sys.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr %b release, align 2
-
-  ; CHECK: ld.acquire.sys.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr %c acquire, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.release.sys.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr %c release, align 4
-
-  ; CHECK: ld.acquire.sys.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr %d acquire, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.release.sys.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr %d release, align 8
-
-  ; CHECK: ld.acquire.sys.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr %e acquire, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.release.sys.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr %e release, align 4
-
-  ; CHECK: ld.acquire.sys.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr %e acquire, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.release.sys.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr %e release, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: generic_monotonic_volatile
-define void @generic_monotonic_volatile(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {
-  ; CHECK: ld.volatile.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr %a monotonic, align 1
-
-  ; CHECK: ld.volatile.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr %b monotonic, align 2
-
-  ; CHECK: ld.volatile.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr %c monotonic, align 4
-
-  ; CHECK: ld.volatile.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr %d monotonic, align 8
-
-  ; CHECK: ld.volatile.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.volatile.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr %e monotonic, align 4
-
-  ; CHECK: ld.volatile.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr %e monotonic, align 8
-
-  ret void
-}
-
-;; global statespace
-
-; CHECK-LABEL: global_plain
-define void @global_plain(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d) local_unnamed_addr {
-  ; CHECK: ld.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load i8, ptr addrspace(1) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i8 %a.add, ptr addrspace(1) %a
-
-  ; CHECK: ld.global.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load i16, ptr addrspace(1) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.global.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i16 %b.add, ptr addrspace(1) %b
-
-  ; CHECK: ld.global.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load i32, ptr addrspace(1) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.global.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store i32 %c.add, ptr addrspace(1) %c
-
-  ; CHECK: ld.global.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load i64, ptr addrspace(1) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.global.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store i64 %d.add, ptr addrspace(1) %d
-
-  ; CHECK: ld.global.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load float, ptr addrspace(1) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.global.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store float %e.add, ptr addrspace(1) %c
-
-  ; CHECK: ld.global.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load double, ptr addrspace(1) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.global.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store double %f.add, ptr addrspace(1) %c
-
-  ret void
-}
-
-; CHECK-LABEL: global_volatile
-define void @global_volatile(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d) local_unnamed_addr {
-  ; CHECK: ld.volatile.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load volatile i8, ptr addrspace(1) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i8 %a.add, ptr addrspace(1) %a
-
-  ; CHECK: ld.volatile.global.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load volatile i16, ptr addrspace(1) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.global.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i16 %b.add, ptr addrspace(1) %b
-
-  ; CHECK: ld.volatile.global.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load volatile i32, ptr addrspace(1) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.global.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store volatile i32 %c.add, ptr addrspace(1) %c
-
-  ; CHECK: ld.volatile.global.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load volatile i64, ptr addrspace(1) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.global.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store volatile i64 %d.add, ptr addrspace(1) %d
-
-  ; CHECK: ld.volatile.global.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load volatile float, ptr addrspace(1) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.volatile.global.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store volatile float %e.add, ptr addrspace(1) %c
-
-  ; CHECK: ld.volatile.global.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load volatile double, ptr addrspace(1) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.global.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store volatile double %f.add, ptr addrspace(1) %c
-
-  ret void
-}
-
-; CHECK-LABEL: global_monotonic
-define void @global_monotonic(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {
-  ; CHECK: ld.relaxed.sys.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr addrspace(1) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.relaxed.sys.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr addrspace(1) %a monotonic, align 1
-
-  ; CHECK: ld.relaxed.sys.global.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr addrspace(1) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.relaxed.sys.global.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr addrspace(1) %b monotonic, align 2
-
-  ; CHECK: ld.relaxed.sys.global.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr addrspace(1) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.relaxed.sys.global.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr addrspace(1) %c monotonic, align 4
-
-  ; CHECK: ld.relaxed.sys.global.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr addrspace(1) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.relaxed.sys.global.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr addrspace(1) %d monotonic, align 8
-
-  ; CHECK: ld.relaxed.sys.global.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr addrspace(1) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.relaxed.sys.global.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr addrspace(1) %e monotonic, align 4
-
-  ; CHECK: ld.relaxed.sys.global.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr addrspace(1) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.relaxed.sys.global.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr addrspace(1) %e monotonic, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: global_monotonic_volatile
-define void @global_monotonic_volatile(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {
-  ; CHECK: ld.mmio.relaxed.sys.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr addrspace(1) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.mmio.relaxed.sys.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr addrspace(1) %a monotonic, align 1
-
-  ; CHECK: ld.mmio.relaxed.sys.global.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr addrspace(1) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.mmio.relaxed.sys.global.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr addrspace(1) %b monotonic, align 2
-
-  ; CHECK: ld.mmio.relaxed.sys.global.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr addrspace(1) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.mmio.relaxed.sys.global.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr addrspace(1) %c monotonic, align 4
-
-  ; CHECK: ld.mmio.relaxed.sys.global.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr addrspace(1) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.mmio.relaxed.sys.global.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr addrspace(1) %d monotonic, align 8
-
-  ; CHECK: ld.mmio.relaxed.sys.global.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr addrspace(1) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.mmio.relaxed.sys.global.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr addrspace(1) %e monotonic, align 4
-
-  ; CHECK: ld.mmio.relaxed.sys.global.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr addrspace(1) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.mmio.relaxed.sys.global.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr addrspace(1) %e monotonic, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: global_acq_rel
-define void @global_acq_rel(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {
-  ; CHECK: ld.acquire.sys.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr addrspace(1) %a acquire, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.release.sys.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr addrspace(1) %a release, align 1
-
-  ; CHECK: ld.acquire.sys.global.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr addrspace(1) %b acquire, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.release.sys.global.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr addrspace(1) %b release, align 2
-
-  ; CHECK: ld.acquire.sys.global.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr addrspace(1) %c acquire, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.release.sys.global.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr addrspace(1) %c release, align 4
-
-  ; CHECK: ld.acquire.sys.global.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr addrspace(1) %d acquire, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.release.sys.global.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr addrspace(1) %d release, align 8
-
-  ; CHECK: ld.acquire.sys.global.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr addrspace(1) %e acquire, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.release.sys.global.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr addrspace(1) %e release, align 4
-
-  ; CHECK: ld.acquire.sys.global.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr addrspace(1) %e acquire, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.release.sys.global.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr addrspace(1) %e release, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: global_acq_rel_volatile
-define void @global_acq_rel_volatile(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {
-  ; CHECK: ld.acquire.sys.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr addrspace(1) %a acquire, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.release.sys.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr addrspace(1) %a release, align 1
-
-  ; CHECK: ld.acquire.sys.global.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr addrspace(1) %b acquire, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.release.sys.global.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr addrspace(1) %b release, align 2
-
-  ; CHECK: ld.acquire.sys.global.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr addrspace(1) %c acquire, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.release.sys.global.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr addrspace(1) %c release, align 4
-
-  ; CHECK: ld.acquire.sys.global.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr addrspace(1) %d acquire, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.release.sys.global.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr addrspace(1) %d release, align 8
-
-  ; CHECK: ld.acquire.sys.global.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr addrspace(1) %e acquire, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.release.sys.global.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr addrspace(1) %e release, align 4
-
-  ; CHECK: ld.acquire.sys.global.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr addrspace(1) %e acquire, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.release.sys.global.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr addrspace(1) %e release, align 8
-
-  ret void
-}
-
-;; shared statespace
-
-; CHECK-LABEL: shared_plain
-define void @shared_plain(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) local_unnamed_addr {
-  ; CHECK: ld.shared.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load i8, ptr addrspace(3) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.shared.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i8 %a.add, ptr addrspace(3) %a
-
-  ; CHECK: ld.shared.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load i16, ptr addrspace(3) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.shared.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i16 %b.add, ptr addrspace(3) %b
-
-  ; CHECK: ld.shared.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load i32, ptr addrspace(3) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.shared.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store i32 %c.add, ptr addrspace(3) %c
-
-  ; CHECK: ld.shared.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load i64, ptr addrspace(3) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.shared.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store i64 %d.add, ptr addrspace(3) %d
-
-  ; CHECK: ld.shared.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load float, ptr addrspace(3) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.shared.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store float %e.add, ptr addrspace(3) %c
-
-  ; CHECK: ld.shared.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load double, ptr addrspace(3) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.shared.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store double %f.add, ptr addrspace(3) %c
-
-  ret void
-}
-
-; CHECK-LABEL: shared_volatile
-define void @shared_volatile(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) local_unnamed_addr {
-  ; CHECK: ld.volatile.shared.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load volatile i8, ptr addrspace(3) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.shared.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i8 %a.add, ptr addrspace(3) %a
-
-  ; CHECK: ld.volatile.shared.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load volatile i16, ptr addrspace(3) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.shared.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i16 %b.add, ptr addrspace(3) %b
-
-  ; CHECK: ld.volatile.shared.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load volatile i32, ptr addrspace(3) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.shared.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store volatile i32 %c.add, ptr addrspace(3) %c
-
-  ; CHECK: ld.volatile.shared.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load volatile i64, ptr addrspace(3) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.shared.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store volatile i64 %d.add, ptr addrspace(3) %d
-
-  ; CHECK: ld.volatile.shared.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load volatile float, ptr addrspace(3) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.volatile.shared.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store volatile float %e.add, ptr addrspace(3) %c
-
-  ; CHECK: ld.volatile.shared.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load volatile double, ptr addrspace(3) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.shared.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store volatile double %f.add, ptr addrspace(3) %c
-
-  ret void
-}
-
-; CHECK-LABEL: shared_monotonic
-define void @shared_monotonic(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {
-  ; CHECK: ld.relaxed.sys.shared.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr addrspace(3) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.relaxed.sys.shared.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr addrspace(3) %a monotonic, align 1
-
-  ; CHECK: ld.relaxed.sys.shared.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr addrspace(3) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.relaxed.sys.shared.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr addrspace(3) %b monotonic, align 2
-
-  ; CHECK: ld.relaxed.sys.shared.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr addrspace(3) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.relaxed.sys.shared.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr addrspace(3) %c monotonic, align 4
-
-  ; CHECK: ld.relaxed.sys.shared.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr addrspace(3) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.relaxed.sys.shared.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr addrspace(3) %d monotonic, align 8
-
-  ; CHECK: ld.relaxed.sys.shared.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr addrspace(3) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.relaxed.sys.shared.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr addrspace(3) %e monotonic, align 4
-
-  ; CHECK: ld.relaxed.sys.shared.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr addrspace(3) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.relaxed.sys.shared.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr addrspace(3) %e monotonic, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: shared_monotonic_volatile
-define void @shared_monotonic_volatile(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {
-  ; CHECK: ld.volatile.shared.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr addrspace(3) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.shared.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr addrspace(3) %a monotonic, align 1
-
-  ; CHECK: ld.volatile.shared.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr addrspace(3) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.shared.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr addrspace(3) %b monotonic, align 2
-
-  ; CHECK: ld.volatile.shared.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr addrspace(3) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.shared.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr addrspace(3) %c monotonic, align 4
-
-  ; CHECK: ld.volatile.shared.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr addrspace(3) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.shared.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr addrspace(3) %d monotonic, align 8
-
-  ; CHECK: ld.volatile.shared.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr addrspace(3) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.volatile.shared.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr addrspace(3) %e monotonic, align 4
-
-  ; CHECK: ld.volatile.shared.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr addrspace(3) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.shared.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr addrspace(3) %e monotonic, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: shared_acq_rel
-define void @shared_acq_rel(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {
-  ; CHECK: ld.acquire.sys.shared.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr addrspace(3) %a acquire, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.release.sys.shared.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr addrspace(3) %a release, align 1
-
-  ; CHECK: ld.acquire.sys.shared.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr addrspace(3) %b acquire, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.release.sys.shared.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr addrspace(3) %b release, align 2
-
-  ; CHECK: ld.acquire.sys.shared.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr addrspace(3) %c acquire, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.release.sys.shared.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr addrspace(3) %c release, align 4
-
-  ; CHECK: ld.acquire.sys.shared.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr addrspace(3) %d acquire, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.release.sys.shared.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr addrspace(3) %d release, align 8
-
-  ; CHECK: ld.acquire.sys.shared.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr addrspace(3) %e acquire, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.release.sys.shared.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr addrspace(3) %e release, align 4
-
-  ; CHECK: ld.acquire.sys.shared.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr addrspace(3) %e acquire, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.release.sys.shared.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr addrspace(3) %e release, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: shared_acq_rel_volatile
-define void @shared_acq_rel_volatile(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {
-  ; CHECK: ld.acquire.sys.shared.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr addrspace(3) %a acquire, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.release.sys.shared.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr addrspace(3) %a release, align 1
-
-  ; CHECK: ld.acquire.sys.shared.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr addrspace(3) %b acquire, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.release.sys.shared.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr addrspace(3) %b release, align 2
-
-  ; CHECK: ld.acquire.sys.shared.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr addrspace(3) %c acquire, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.release.sys.shared.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr addrspace(3) %c release, align 4
-
-  ; CHECK: ld.acquire.sys.shared.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr addrspace(3) %d acquire, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.release.sys.shared.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr addrspace(3) %d release, align 8
-
-  ; CHECK: ld.acquire.sys.shared.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr addrspace(3) %e acquire, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.release.sys.shared.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr addrspace(3) %e release, align 4
-
-  ; CHECK: ld.acquire.sys.shared.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr addrspace(3) %e acquire, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.release.sys.shared.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr addrspace(3) %e release, align 8
-
-  ret void
-}
-
-;; local statespace
-
-; CHECK-LABEL: local_plain
-define void @local_plain(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d) local_unnamed_addr {
-  ; CHECK: ld.local.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load i8, ptr addrspace(5) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.local.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i8 %a.add, ptr addrspace(5) %a
-
-  ; CHECK: ld.local.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load i16, ptr addrspace(5) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.local.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i16 %b.add, ptr addrspace(5) %b
-
-  ; CHECK: ld.local.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load i32, ptr addrspace(5) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.local.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store i32 %c.add, ptr addrspace(5) %c
-
-  ; CHECK: ld.local.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load i64, ptr addrspace(5) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.local.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store i64 %d.add, ptr addrspace(5) %d
-
-  ; CHECK: ld.local.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load float, ptr addrspace(5) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.local.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store float %e.add, ptr addrspace(5) %c
-
-  ; CHECK: ld.local.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load double, ptr addrspace(5) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.local.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store double %f.add, ptr addrspace(5) %c
-
-  ret void
-}
-
-; CHECK-LABEL: local_volatile
-define void @local_volatile(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d) local_unnamed_addr {
-  ; CHECK: ld.local.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load volatile i8, ptr addrspace(5) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.local.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i8 %a.add, ptr addrspace(5) %a
-
-  ; CHECK: ld.local.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load volatile i16, ptr addrspace(5) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.local.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i16 %b.add, ptr addrspace(5) %b
-
-  ; CHECK: ld.local.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load volatile i32, ptr addrspace(5) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.local.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store volatile i32 %c.add, ptr addrspace(5) %c
-
-  ; CHECK: ld.local.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load volatile i64, ptr addrspace(5) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.local.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store volatile i64 %d.add, ptr addrspace(5) %d
-
-  ; CHECK: ld.local.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load volatile float, ptr addrspace(5) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.local.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store volatile float %e.add, ptr addrspace(5) %c
-
-  ; CHECK: ld.local.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load volatile double, ptr addrspace(5) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.local.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store volatile double %f.add, ptr addrspace(5) %c
-
-  ret void
-}
-
-; CHECK-LABEL: local_monotonic
-define void @local_monotonic(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {
-  ; CHECK: ld.local.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr addrspace(5) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.local.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr addrspace(5) %a monotonic, align 1
-
-  ; CHECK: ld.local.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr addrspace(5) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.local.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr addrspace(5) %b monotonic, align 2
-
-  ; CHECK: ld.local.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr addrspace(5) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.local.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr addrspace(5) %c monotonic, align 4
-
-  ; CHECK: ld.local.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr addrspace(5) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.local.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr addrspace(5) %d monotonic, align 8
-
-  ; CHECK: ld.local.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr addrspace(5) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.local.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr addrspace(5) %e monotonic, align 4
-
-  ; CHECK: ld.local.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr addrspace(5) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.local.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr addrspace(5) %e monotonic, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: local_monotonic_volatile
-define void @local_monotonic_volatile(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {
-  ; CHECK: ld.local.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr addrspace(5) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.local.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr addrspace(5) %a monotonic, align 1
-
-  ; CHECK: ld.local.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr addrspace(5) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.local.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr addrspace(5) %b monotonic, align 2
-
-  ; CHECK: ld.local.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr addrspace(5) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.local.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr addrspace(5) %c monotonic, align 4
-
-  ; CHECK: ld.local.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr addrspace(5) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.local.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr addrspace(5) %d monotonic, align 8
-
-  ; CHECK: ld.local.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr addrspace(5) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.local.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr addrspace(5) %e monotonic, align 4
-
-  ; CHECK: ld.local.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr addrspace(5) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.local.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr addrspace(5) %e monotonic, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: local_acq_rel
-define void @local_acq_rel(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {
-  ; CHECK: ld.local.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr addrspace(5) %a acquire, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.local.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr addrspace(5) %a release, align 1
-
-  ; CHECK: ld.local.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr addrspace(5) %b acquire, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.local.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr addrspace(5) %b release, align 2
-
-  ; CHECK: ld.local.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr addrspace(5) %c acquire, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.local.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr addrspace(5) %c release, align 4
-
-  ; CHECK: ld.local.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr addrspace(5) %d acquire, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.local.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr addrspace(5) %d release, align 8
-
-  ; CHECK: ld.local.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr addrspace(5) %e acquire, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.local.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr addrspace(5) %e release, align 4
-
-  ; CHECK: ld.local.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr addrspace(5) %e acquire, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.local.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr addrspace(5) %e release, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: local_acq_rel_volatile
-define void @local_acq_rel_volatile(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {
-  ; CHECK: ld.local.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr addrspace(5) %a acquire, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.local.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr addrspace(5) %a release, align 1
-
-  ; CHECK: ld.local.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr addrspace(5) %b acquire, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.local.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr addrspace(5) %b release, align 2
-
-  ; CHECK: ld.local.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr addrspace(5) %c acquire, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.local.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr addrspace(5) %c release, align 4
-
-  ; CHECK: ld.local.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr addrspace(5) %d acquire, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.local.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr addrspace(5) %d release, align 8
-
-  ; CHECK: ld.local.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr addrspace(5) %e acquire, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.local.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr addrspace(5) %e release, align 4
-
-  ; CHECK: ld.local.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr addrspace(5) %e acquire, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.local.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr addrspace(5) %e release, align 8
-
-  ret void
-}
diff --git a/llvm/test/CodeGen/NVPTX/load-store.ll b/llvm/test/CodeGen/NVPTX/load-store.ll
index 27065f5eca9f4..c477bd9e744cd 100644
--- a/llvm/test/CodeGen/NVPTX/load-store.ll
+++ b/llvm/test/CodeGen/NVPTX/load-store.ll
@@ -1,10 +1,8 @@
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck %s
 ; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
 
-; generic statespace
-
-; CHECK-LABEL: generic_plain
-define void @generic_plain(ptr %a, ptr %b, ptr %c, ptr %d) local_unnamed_addr {
+; CHECK-LABEL: plain
+define void @plain(ptr %a, ptr %b, ptr %c, ptr %d) local_unnamed_addr {
   ; CHECK: ld.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
   %a.load = load i8, ptr %a
   %a.add = add i8 %a.load, 1
@@ -44,8 +42,8 @@ define void @generic_plain(ptr %a, ptr %b, ptr %c, ptr %d) local_unnamed_addr {
   ret void
 }
 
-; CHECK-LABEL: generic_volatile
-define void @generic_volatile(ptr %a, ptr %b, ptr %c, ptr %d) local_unnamed_addr {
+; CHECK-LABEL: volatile
+define void @volatile(ptr %a, ptr %b, ptr %c, ptr %d) local_unnamed_addr {
   ; CHECK: ld.volatile.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
   %a.load = load volatile i8, ptr %a
   %a.add = add i8 %a.load, 1
@@ -85,8 +83,8 @@ define void @generic_volatile(ptr %a, ptr %b, ptr %c, ptr %d) local_unnamed_addr
   ret void
 }
 
-; CHECK-LABEL: generic_monotonic
-define void @generic_monotonic(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {
+; CHECK-LABEL: monotonic
+define void @monotonic(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {
   ; CHECK: ld.volatile.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
   %a.load = load atomic i8, ptr %a monotonic, align 1
   %a.add = add i8 %a.load, 1
@@ -125,542 +123,3 @@ define void @generic_monotonic(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unn
 
   ret void
 }
-
-; CHECK-LABEL: generic_monotonic_volatile
-define void @generic_monotonic_volatile(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {
-  ; CHECK: ld.volatile.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr %a monotonic, align 1
-
-  ; CHECK: ld.volatile.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr %b monotonic, align 2
-
-  ; CHECK: ld.volatile.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr %c monotonic, align 4
-
-  ; CHECK: ld.volatile.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr %d monotonic, align 8
-
-  ; CHECK: ld.volatile.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.volatile.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr %e monotonic, align 4
-
-  ; CHECK: ld.volatile.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr %e monotonic, align 8
-
-  ret void
-}
-
-;; global statespace
-
-; CHECK-LABEL: global_plain
-define void @global_plain(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d) local_unnamed_addr {
-  ; CHECK: ld.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load i8, ptr addrspace(1) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i8 %a.add, ptr addrspace(1) %a
-
-  ; CHECK: ld.global.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load i16, ptr addrspace(1) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.global.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i16 %b.add, ptr addrspace(1) %b
-
-  ; CHECK: ld.global.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load i32, ptr addrspace(1) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.global.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store i32 %c.add, ptr addrspace(1) %c
-
-  ; CHECK: ld.global.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load i64, ptr addrspace(1) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.global.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store i64 %d.add, ptr addrspace(1) %d
-
-  ; CHECK: ld.global.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load float, ptr addrspace(1) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.global.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store float %e.add, ptr addrspace(1) %c
-
-  ; CHECK: ld.global.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load double, ptr addrspace(1) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.global.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store double %f.add, ptr addrspace(1) %c
-
-  ret void
-}
-
-; CHECK-LABEL: global_volatile
-define void @global_volatile(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d) local_unnamed_addr {
-  ; CHECK: ld.volatile.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load volatile i8, ptr addrspace(1) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i8 %a.add, ptr addrspace(1) %a
-
-  ; CHECK: ld.volatile.global.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load volatile i16, ptr addrspace(1) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.global.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i16 %b.add, ptr addrspace(1) %b
-
-  ; CHECK: ld.volatile.global.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load volatile i32, ptr addrspace(1) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.global.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store volatile i32 %c.add, ptr addrspace(1) %c
-
-  ; CHECK: ld.volatile.global.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load volatile i64, ptr addrspace(1) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.global.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store volatile i64 %d.add, ptr addrspace(1) %d
-
-  ; CHECK: ld.volatile.global.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load volatile float, ptr addrspace(1) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.volatile.global.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store volatile float %e.add, ptr addrspace(1) %c
-
-  ; CHECK: ld.volatile.global.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load volatile double, ptr addrspace(1) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.global.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store volatile double %f.add, ptr addrspace(1) %c
-
-  ret void
-}
-
-; CHECK-LABEL: global_monotonic
-define void @global_monotonic(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {
-  ; CHECK: ld.volatile.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr addrspace(1) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr addrspace(1) %a monotonic, align 1
-
-  ; CHECK: ld.volatile.global.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr addrspace(1) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.global.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr addrspace(1) %b monotonic, align 2
-
-  ; CHECK: ld.volatile.global.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr addrspace(1) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.global.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr addrspace(1) %c monotonic, align 4
-
-  ; CHECK: ld.volatile.global.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr addrspace(1) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.global.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr addrspace(1) %d monotonic, align 8
-
-  ; CHECK: ld.volatile.global.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr addrspace(1) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.volatile.global.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr addrspace(1) %e monotonic, align 4
-
-  ; CHECK: ld.volatile.global.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr addrspace(1) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.global.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr addrspace(1) %e monotonic, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: global_monotonic_volatile
-define void @global_monotonic_volatile(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {
-  ; CHECK: ld.volatile.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr addrspace(1) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr addrspace(1) %a monotonic, align 1
-
-  ; CHECK: ld.volatile.global.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr addrspace(1) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.global.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr addrspace(1) %b monotonic, align 2
-
-  ; CHECK: ld.volatile.global.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr addrspace(1) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.global.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr addrspace(1) %c monotonic, align 4
-
-  ; CHECK: ld.volatile.global.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr addrspace(1) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.global.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr addrspace(1) %d monotonic, align 8
-
-  ; CHECK: ld.volatile.global.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr addrspace(1) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.volatile.global.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr addrspace(1) %e monotonic, align 4
-
-  ; CHECK: ld.volatile.global.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr addrspace(1) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.global.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr addrspace(1) %e monotonic, align 8
-
-  ret void
-}
-
-;; shared statespace
-
-; CHECK-LABEL: shared_plain
-define void @shared_plain(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) local_unnamed_addr {
-  ; CHECK: ld.shared.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load i8, ptr addrspace(3) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.shared.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i8 %a.add, ptr addrspace(3) %a
-
-  ; CHECK: ld.shared.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load i16, ptr addrspace(3) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.shared.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i16 %b.add, ptr addrspace(3) %b
-
-  ; CHECK: ld.shared.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load i32, ptr addrspace(3) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.shared.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store i32 %c.add, ptr addrspace(3) %c
-
-  ; CHECK: ld.shared.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load i64, ptr addrspace(3) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.shared.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store i64 %d.add, ptr addrspace(3) %d
-
-  ; CHECK: ld.shared.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load float, ptr addrspace(3) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.shared.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store float %e.add, ptr addrspace(3) %c
-
-  ; CHECK: ld.shared.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load double, ptr addrspace(3) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.shared.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store double %f.add, ptr addrspace(3) %c
-
-  ret void
-}
-
-; CHECK-LABEL: shared_volatile
-define void @shared_volatile(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) local_unnamed_addr {
-  ; CHECK: ld.volatile.shared.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load volatile i8, ptr addrspace(3) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.shared.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i8 %a.add, ptr addrspace(3) %a
-
-  ; CHECK: ld.volatile.shared.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load volatile i16, ptr addrspace(3) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.shared.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i16 %b.add, ptr addrspace(3) %b
-
-  ; CHECK: ld.volatile.shared.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load volatile i32, ptr addrspace(3) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.shared.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store volatile i32 %c.add, ptr addrspace(3) %c
-
-  ; CHECK: ld.volatile.shared.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load volatile i64, ptr addrspace(3) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.shared.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store volatile i64 %d.add, ptr addrspace(3) %d
-
-  ; CHECK: ld.volatile.shared.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load volatile float, ptr addrspace(3) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.volatile.shared.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store volatile float %e.add, ptr addrspace(3) %c
-
-  ; CHECK: ld.volatile.shared.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load volatile double, ptr addrspace(3) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.shared.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store volatile double %f.add, ptr addrspace(3) %c
-
-  ret void
-}
-
-; CHECK-LABEL: shared_monotonic
-define void @shared_monotonic(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {
-  ; CHECK: ld.volatile.shared.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr addrspace(3) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.shared.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr addrspace(3) %a monotonic, align 1
-
-  ; CHECK: ld.volatile.shared.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr addrspace(3) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.shared.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr addrspace(3) %b monotonic, align 2
-
-  ; CHECK: ld.volatile.shared.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr addrspace(3) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.shared.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr addrspace(3) %c monotonic, align 4
-
-  ; CHECK: ld.volatile.shared.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr addrspace(3) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.shared.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr addrspace(3) %d monotonic, align 8
-
-  ; CHECK: ld.volatile.shared.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr addrspace(3) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.volatile.shared.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr addrspace(3) %e monotonic, align 4
-
-  ; CHECK: ld.volatile.shared.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr addrspace(3) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.shared.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr addrspace(3) %e monotonic, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: shared_monotonic_volatile
-define void @shared_monotonic_volatile(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {
-  ; CHECK: ld.volatile.shared.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr addrspace(3) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.volatile.shared.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr addrspace(3) %a monotonic, align 1
-
-  ; CHECK: ld.volatile.shared.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr addrspace(3) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.volatile.shared.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr addrspace(3) %b monotonic, align 2
-
-  ; CHECK: ld.volatile.shared.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr addrspace(3) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.volatile.shared.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr addrspace(3) %c monotonic, align 4
-
-  ; CHECK: ld.volatile.shared.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr addrspace(3) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.volatile.shared.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr addrspace(3) %d monotonic, align 8
-
-  ; CHECK: ld.volatile.shared.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr addrspace(3) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.volatile.shared.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr addrspace(3) %e monotonic, align 4
-
-  ; CHECK: ld.volatile.shared.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr addrspace(3) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.volatile.shared.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr addrspace(3) %e monotonic, align 8
-
-  ret void
-}
-
-;; local statespace
-
-; CHECK-LABEL: local_plain
-define void @local_plain(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d) local_unnamed_addr {
-  ; CHECK: ld.local.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load i8, ptr addrspace(5) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.local.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i8 %a.add, ptr addrspace(5) %a
-
-  ; CHECK: ld.local.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load i16, ptr addrspace(5) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.local.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store i16 %b.add, ptr addrspace(5) %b
-
-  ; CHECK: ld.local.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load i32, ptr addrspace(5) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.local.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store i32 %c.add, ptr addrspace(5) %c
-
-  ; CHECK: ld.local.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load i64, ptr addrspace(5) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.local.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store i64 %d.add, ptr addrspace(5) %d
-
-  ; CHECK: ld.local.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load float, ptr addrspace(5) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.local.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store float %e.add, ptr addrspace(5) %c
-
-  ; CHECK: ld.local.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load double, ptr addrspace(5) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.local.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store double %f.add, ptr addrspace(5) %c
-
-  ret void
-}
-
-; CHECK-LABEL: local_volatile
-define void @local_volatile(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d) local_unnamed_addr {
-  ; CHECK: ld.local.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load volatile i8, ptr addrspace(5) %a
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.local.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i8 %a.add, ptr addrspace(5) %a
-
-  ; CHECK: ld.local.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load volatile i16, ptr addrspace(5) %b
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.local.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store volatile i16 %b.add, ptr addrspace(5) %b
-
-  ; CHECK: ld.local.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load volatile i32, ptr addrspace(5) %c
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.local.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store volatile i32 %c.add, ptr addrspace(5) %c
-
-  ; CHECK: ld.local.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load volatile i64, ptr addrspace(5) %d
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.local.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store volatile i64 %d.add, ptr addrspace(5) %d
-
-  ; CHECK: ld.local.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load volatile float, ptr addrspace(5) %c
-  %e.add = fadd float %e.load, 1.
-  ; CHECK: st.local.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store volatile float %e.add, ptr addrspace(5) %c
-
-  ; CHECK: ld.local.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load volatile double, ptr addrspace(5) %c
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.local.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store volatile double %f.add, ptr addrspace(5) %c
-
-  ret void
-}
-
-; CHECK-LABEL: local_monotonic
-define void @local_monotonic(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {
-  ; CHECK: ld.local.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic i8, ptr addrspace(5) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.local.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i8 %a.add, ptr addrspace(5) %a monotonic, align 1
-
-  ; CHECK: ld.local.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic i16, ptr addrspace(5) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.local.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic i16 %b.add, ptr addrspace(5) %b monotonic, align 2
-
-  ; CHECK: ld.local.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic i32, ptr addrspace(5) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.local.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic i32 %c.add, ptr addrspace(5) %c monotonic, align 4
-
-  ; CHECK: ld.local.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic i64, ptr addrspace(5) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.local.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic i64 %d.add, ptr addrspace(5) %d monotonic, align 8
-
-  ; CHECK: ld.local.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic float, ptr addrspace(5) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.local.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic float %e.add, ptr addrspace(5) %e monotonic, align 4
-
-  ; CHECK: ld.local.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic double, ptr addrspace(5) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.local.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic double %f.add, ptr addrspace(5) %e monotonic, align 8
-
-  ret void
-}
-
-; CHECK-LABEL: local_monotonic_volatile
-define void @local_monotonic_volatile(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {
-  ; CHECK: ld.local.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %a.load = load atomic volatile i8, ptr addrspace(5) %a monotonic, align 1
-  %a.add = add i8 %a.load, 1
-  ; CHECK: st.local.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i8 %a.add, ptr addrspace(5) %a monotonic, align 1
-
-  ; CHECK: ld.local.u16 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %b.load = load atomic volatile i16, ptr addrspace(5) %b monotonic, align 2
-  %b.add = add i16 %b.load, 1
-  ; CHECK: st.local.u16 [%rd{{[0-9]+}}], %rs{{[0-9]+}}
-  store atomic volatile i16 %b.add, ptr addrspace(5) %b monotonic, align 2
-
-  ; CHECK: ld.local.u32 %r{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %c.load = load atomic volatile i32, ptr addrspace(5) %c monotonic, align 4
-  %c.add = add i32 %c.load, 1
-  ; CHECK: st.local.u32 [%rd{{[0-9]+}}], %r{{[0-9]+}}
-  store atomic volatile i32 %c.add, ptr addrspace(5) %c monotonic, align 4
-
-  ; CHECK: ld.local.u64 %rd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %d.load = load atomic volatile i64, ptr addrspace(5) %d monotonic, align 8
-  %d.add = add i64 %d.load, 1
-  ; CHECK: st.local.u64 [%rd{{[0-9]+}}], %rd{{[0-9]+}}
-  store atomic volatile i64 %d.add, ptr addrspace(5) %d monotonic, align 8
-
-  ; CHECK: ld.local.f32 %f{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %e.load = load atomic volatile float, ptr addrspace(5) %e monotonic, align 4
-  %e.add = fadd float %e.load, 1.0
-  ; CHECK: st.local.f32 [%rd{{[0-9]+}}], %f{{[0-9]+}}
-  store atomic volatile float %e.add, ptr addrspace(5) %e monotonic, align 4
-
-  ; CHECK: ld.local.f64 %fd{{[0-9]+}}, [%rd{{[0-9]+}}]
-  %f.load = load atomic volatile double, ptr addrspace(5) %e monotonic, align 8
-  %f.add = fadd double %f.load, 1.
-  ; CHECK: st.local.f64 [%rd{{[0-9]+}}], %fd{{[0-9]+}}
-  store atomic volatile double %f.add, ptr addrspace(5) %e monotonic, align 8
-
-  ret void
-}

From 296a9563690f2fea78e4d746a6d1896262b69da4 Mon Sep 17 00:00:00 2001
From: Shaw Young <58664393+shawbyoung@users.noreply.github.com>
Date: Fri, 19 Jul 2024 17:00:28 -0400
Subject: [PATCH 232/486] [BOLT] Match functions with call graph (#98125)

Implemented call graph function matching. First, two call graphs are
constructed for both profiled and binary functions. Then functions are
hashed based on the names of their callee/caller functions. Finally,
functions are matched based on these neighbor hashes and the
longest common prefix of their names. The `match-with-call-graph`
flag turns this matching on.

Test Plan: Added match-with-call-graph.test. Matched 164 functions
in a large binary with 10171 profiled functions.
---
 bolt/docs/CommandLineArgumentReference.md     |   4 +
 bolt/include/bolt/Profile/YAMLProfileReader.h |  56 +++++++
 bolt/lib/Profile/YAMLProfileReader.cpp        | 157 +++++++++++++++++-
 .../X86/match-functions-with-call-graph.test  | 103 ++++++++++++
 4 files changed, 313 insertions(+), 7 deletions(-)
 create mode 100644 bolt/test/X86/match-functions-with-call-graph.test

diff --git a/bolt/docs/CommandLineArgumentReference.md b/bolt/docs/CommandLineArgumentReference.md
index bd6e2ec01b53e..d2cbf0ca86449 100644
--- a/bolt/docs/CommandLineArgumentReference.md
+++ b/bolt/docs/CommandLineArgumentReference.md
@@ -686,6 +686,10 @@
   threshold means fewer functions to process. E.g threshold of 90 means only top
   10 percent of functions with profile will be processed.
 
+- `--match-with-call-graph`
+
+  Match functions with call graph
+
 - `--memcpy1-spec=<func1,func2:cs1:cs2,func3:cs1,...>`
 
   List of functions with call sites for which to specialize memcpy() for size 1
diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h
index 502a1ad612bb0..bd5a86fd676a5 100644
--- a/bolt/include/bolt/Profile/YAMLProfileReader.h
+++ b/bolt/include/bolt/Profile/YAMLProfileReader.h
@@ -43,6 +43,59 @@ class YAMLProfileReader : public ProfileReaderBase {
   using ProfileLookupMap =
       DenseMap<uint32_t, yaml::bolt::BinaryFunctionProfile *>;
 
+  /// A class for matching binary functions in functions in the YAML profile.
+  /// First, a call graph is constructed for both profiled and binary functions.
+  /// Then functions are hashed based on the names of their callee/caller
+  /// functions. Finally, functions are matched based on these neighbor hashes.
+  class CallGraphMatcher {
+  public:
+    /// Constructs the call graphs for binary and profiled functions and
+    /// computes neighbor hashes for binary functions.
+    CallGraphMatcher(BinaryContext &BC, yaml::bolt::BinaryProfile &YamlBP,
+                     ProfileLookupMap &IdToYAMLBF);
+
+    /// Returns the YamlBFs adjacent to the parameter YamlBF in the call graph.
+    std::optional<std::set<yaml::bolt::BinaryFunctionProfile *>>
+    getAdjacentYamlBFs(yaml::bolt::BinaryFunctionProfile &YamlBF) {
+      auto It = YamlBFAdjacencyMap.find(&YamlBF);
+      return It == YamlBFAdjacencyMap.end() ? std::nullopt
+                                            : std::make_optional(It->second);
+    }
+
+    /// Returns the binary functions with the parameter neighbor hash.
+    std::optional<std::vector<BinaryFunction *>>
+    getBFsWithNeighborHash(uint64_t NeighborHash) {
+      auto It = NeighborHashToBFs.find(NeighborHash);
+      return It == NeighborHashToBFs.end() ? std::nullopt
+                                           : std::make_optional(It->second);
+    }
+
+  private:
+    /// Adds edges to the binary function call graph given the callsites of the
+    /// parameter function.
+    void constructBFCG(BinaryContext &BC, yaml::bolt::BinaryProfile &YamlBP);
+
+    /// Using the constructed binary function call graph, computes and creates
+    /// mappings from "neighbor hash" (composed of the function names of callee
+    /// and caller functions of a function) to binary functions.
+    void computeBFNeighborHashes(BinaryContext &BC);
+
+    /// Constructs the call graph for profile functions.
+    void constructYAMLFCG(yaml::bolt::BinaryProfile &YamlBP,
+                          ProfileLookupMap &IdToYAMLBF);
+
+    /// Adjacency map for binary functions in the call graph.
+    DenseMap<BinaryFunction *, std::set<BinaryFunction *>> BFAdjacencyMap;
+
+    /// Maps neighbor hashes to binary functions.
+    DenseMap<uint64_t, std::vector<BinaryFunction *>> NeighborHashToBFs;
+
+    /// Adjacency map for profile functions in the call graph.
+    DenseMap<yaml::bolt::BinaryFunctionProfile *,
+             std::set<yaml::bolt::BinaryFunctionProfile *>>
+        YamlBFAdjacencyMap;
+  };
+
 private:
   /// Adjustments for basic samples profiles (without LBR).
   bool NormalizeByInsnCount{false};
@@ -100,6 +153,9 @@ class YAMLProfileReader : public ProfileReaderBase {
   /// Matches functions using exact hash.
   size_t matchWithHash(BinaryContext &BC);
 
+  /// Matches functions using the call graph.
+  size_t matchWithCallGraph(BinaryContext &BC);
+
   /// Matches functions with similarly named profiled functions.
   size_t matchWithNameSimilarity(BinaryContext &BC);
 
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index d9a6998bc5e9f..604a9fb4813be 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -41,6 +41,10 @@ llvm::cl::opt<bool>
     MatchProfileWithFunctionHash("match-profile-with-function-hash",
                                  cl::desc("Match profile with function hash"),
                                  cl::Hidden, cl::cat(BoltOptCategory));
+llvm::cl::opt<bool>
+    MatchWithCallGraph("match-with-call-graph",
+                       cl::desc("Match functions with call graph"), cl::Hidden,
+                       cl::cat(BoltOptCategory));
 
 llvm::cl::opt<bool> ProfileUseDFS("profile-use-dfs",
                                   cl::desc("use DFS order for YAML profile"),
@@ -50,6 +54,69 @@ llvm::cl::opt<bool> ProfileUseDFS("profile-use-dfs",
 namespace llvm {
 namespace bolt {
 
+YAMLProfileReader::CallGraphMatcher::CallGraphMatcher(
+    BinaryContext &BC, yaml::bolt::BinaryProfile &YamlBP,
+    ProfileLookupMap &IdToYAMLBF) {
+  constructBFCG(BC, YamlBP);
+  constructYAMLFCG(YamlBP, IdToYAMLBF);
+  computeBFNeighborHashes(BC);
+}
+
+void YAMLProfileReader::CallGraphMatcher::constructBFCG(
+    BinaryContext &BC, yaml::bolt::BinaryProfile &YamlBP) {
+  for (BinaryFunction *BF : BC.getAllBinaryFunctions()) {
+    for (const BinaryBasicBlock &BB : BF->blocks()) {
+      for (const MCInst &Instr : BB) {
+        if (!BC.MIB->isCall(Instr))
+          continue;
+        const MCSymbol *CallSymbol = BC.MIB->getTargetSymbol(Instr);
+        if (!CallSymbol)
+          continue;
+        BinaryData *BD = BC.getBinaryDataByName(CallSymbol->getName());
+        if (!BD)
+          continue;
+        BinaryFunction *CalleeBF = BC.getFunctionForSymbol(BD->getSymbol());
+        if (!CalleeBF)
+          continue;
+
+        BFAdjacencyMap[CalleeBF].insert(BF);
+        BFAdjacencyMap[BF].insert(CalleeBF);
+      }
+    }
+  }
+}
+
+void YAMLProfileReader::CallGraphMatcher::computeBFNeighborHashes(
+    BinaryContext &BC) {
+  for (BinaryFunction *BF : BC.getAllBinaryFunctions()) {
+    auto It = BFAdjacencyMap.find(BF);
+    if (It == BFAdjacencyMap.end())
+      continue;
+    auto &AdjacentBFs = It->second;
+    std::string HashStr;
+    for (BinaryFunction *BF : AdjacentBFs)
+      HashStr += BF->getOneName();
+    uint64_t Hash = std::hash<std::string>{}(HashStr);
+    NeighborHashToBFs[Hash].push_back(BF);
+  }
+}
+
+void YAMLProfileReader::CallGraphMatcher::constructYAMLFCG(
+    yaml::bolt::BinaryProfile &YamlBP, ProfileLookupMap &IdToYAMLBF) {
+
+  for (auto &CallerYamlBF : YamlBP.Functions) {
+    for (auto &YamlBB : CallerYamlBF.Blocks) {
+      for (auto &CallSite : YamlBB.CallSites) {
+        auto IdToYAMLBFIt = IdToYAMLBF.find(CallSite.DestId);
+        if (IdToYAMLBFIt == IdToYAMLBF.end())
+          continue;
+        YamlBFAdjacencyMap[&CallerYamlBF].insert(IdToYAMLBFIt->second);
+        YamlBFAdjacencyMap[IdToYAMLBFIt->second].insert(&CallerYamlBF);
+      }
+    }
+  }
+}
+
 bool YAMLProfileReader::isYAML(const StringRef Filename) {
   if (auto MB = MemoryBuffer::getFileOrSTDIN(Filename)) {
     StringRef Buffer = (*MB)->getBuffer();
@@ -350,7 +417,7 @@ bool YAMLProfileReader::profileMatches(
 }
 
 bool YAMLProfileReader::mayHaveProfileData(const BinaryFunction &BF) {
-  if (opts::MatchProfileWithFunctionHash)
+  if (opts::MatchProfileWithFunctionHash || opts::MatchWithCallGraph)
     return true;
   for (StringRef Name : BF.getNames())
     if (ProfileFunctionNames.contains(Name))
@@ -446,6 +513,79 @@ size_t YAMLProfileReader::matchWithLTOCommonName() {
   return MatchedWithLTOCommonName;
 }
 
+size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
+  if (!opts::MatchWithCallGraph)
+    return 0;
+
+  size_t MatchedWithCallGraph = 0;
+  CallGraphMatcher CGMatcher(BC, YamlBP, IdToYamLBF);
+
+  ItaniumPartialDemangler Demangler;
+  auto GetBaseName = [&](std::string &FunctionName) {
+    if (Demangler.partialDemangle(FunctionName.c_str()))
+      return std::string("");
+    size_t BufferSize = 1;
+    char *Buffer = static_cast<char *>(std::malloc(BufferSize));
+    char *BaseName = Demangler.getFunctionBaseName(Buffer, &BufferSize);
+    if (!BaseName) {
+      std::free(Buffer);
+      return std::string("");
+    }
+    if (Buffer != BaseName)
+      Buffer = BaseName;
+    std::string BaseNameStr(Buffer, BufferSize);
+    std::free(Buffer);
+    return BaseNameStr;
+  };
+
+  // Matches YAMLBF to BFs with neighbor hashes.
+  for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
+    if (YamlBF.Used)
+      continue;
+    auto AdjacentYamlBFsOpt = CGMatcher.getAdjacentYamlBFs(YamlBF);
+    if (!AdjacentYamlBFsOpt)
+      continue;
+    std::set<yaml::bolt::BinaryFunctionProfile *> AdjacentYamlBFs =
+        AdjacentYamlBFsOpt.value();
+    std::string AdjacentYamlBFsHashStr;
+    for (auto *AdjacentYamlBF : AdjacentYamlBFs)
+      AdjacentYamlBFsHashStr += AdjacentYamlBF->Name;
+    uint64_t Hash = std::hash<std::string>{}(AdjacentYamlBFsHashStr);
+    auto BFsWithSameHashOpt = CGMatcher.getBFsWithNeighborHash(Hash);
+    if (!BFsWithSameHashOpt)
+      continue;
+    std::vector<BinaryFunction *> BFsWithSameHash = BFsWithSameHashOpt.value();
+    // Finds the binary function with the longest common prefix to the profiled
+    // function and matches.
+    BinaryFunction *ClosestBF = nullptr;
+    size_t LCP = 0;
+    std::string YamlBFBaseName = GetBaseName(YamlBF.Name);
+    for (BinaryFunction *BF : BFsWithSameHash) {
+      if (ProfiledFunctions.count(BF))
+        continue;
+      std::string BFName = std::string(BF->getOneName());
+      std::string BFBaseName = GetBaseName(BFName);
+      size_t PrefixLength = 0;
+      size_t N = std::min(YamlBFBaseName.size(), BFBaseName.size());
+      for (size_t I = 0; I < N; ++I) {
+        if (YamlBFBaseName[I] != BFBaseName[I])
+          break;
+        ++PrefixLength;
+      }
+      if (PrefixLength >= LCP) {
+        LCP = PrefixLength;
+        ClosestBF = BF;
+      }
+    }
+    if (ClosestBF) {
+      matchProfileToFunction(YamlBF, *ClosestBF);
+      ++MatchedWithCallGraph;
+    }
+  }
+
+  return MatchedWithCallGraph;
+}
+
 size_t YAMLProfileReader::matchWithNameSimilarity(BinaryContext &BC) {
   if (opts::NameSimilarityFunctionMatchingThreshold == 0)
     return 0;
@@ -581,9 +721,14 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
     }
   }
 
+  // Map profiled function ids to names.
+  for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions)
+    IdToYamLBF[YamlBF.Id] = &YamlBF;
+
   const size_t MatchedWithExactName = matchWithExactName();
   const size_t MatchedWithHash = matchWithHash(BC);
   const size_t MatchedWithLTOCommonName = matchWithLTOCommonName();
+  const size_t MatchedWithCallGraph = matchWithCallGraph(BC);
   const size_t MatchedWithNameSimilarity = matchWithNameSimilarity(BC);
 
   for (auto [YamlBF, BF] : llvm::zip_equal(YamlBP.Functions, ProfileBFs))
@@ -603,6 +748,8 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
            << " functions with hash\n";
     outs() << "BOLT-INFO: matched " << MatchedWithLTOCommonName
            << " functions with matching LTO common names\n";
+    outs() << "BOLT-INFO: matched " << MatchedWithCallGraph
+           << " functions with call graph\n";
     outs() << "BOLT-INFO: matched " << MatchedWithNameSimilarity
            << " functions with similar names\n";
   }
@@ -610,11 +757,6 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
   // Set for parseFunctionProfile().
   NormalizeByInsnCount = usesEvent("cycles") || usesEvent("instructions");
   NormalizeByCalls = usesEvent("branches");
-
-  // Map profiled function ids to names.
-  for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions)
-    IdToYamLBF[YamlBF.Id] = &YamlBF;
-
   uint64_t NumUnused = 0;
   for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
     if (YamlBF.Id >= YamlProfileToFunction.size()) {
@@ -630,7 +772,8 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
 
   BC.setNumUnusedProfiledObjects(NumUnused);
 
-  if (opts::Lite && opts::MatchProfileWithFunctionHash) {
+  if (opts::Lite &&
+      (opts::MatchProfileWithFunctionHash || opts::MatchWithCallGraph)) {
     for (BinaryFunction *BF : BC.getAllBinaryFunctions())
       if (!BF->hasProfile())
         BF->setIgnored();
diff --git a/bolt/test/X86/match-functions-with-call-graph.test b/bolt/test/X86/match-functions-with-call-graph.test
new file mode 100644
index 0000000000000..e826c57f35312
--- /dev/null
+++ b/bolt/test/X86/match-functions-with-call-graph.test
@@ -0,0 +1,103 @@
+## Tests blocks matching by called function names in inferStaleProfile.
+
+# REQUIRES: system-linux
+# RUN: split-file %s %t
+# RUN: %clang %cflags %t/main.cpp -o %t.exe -Wl,-q -nostdlib
+# RUN: llvm-bolt %t.exe -o %t.out --data %t/yaml --profile-ignore-hash -v=1 \
+# RUN:   --dyno-stats --print-cfg --infer-stale-profile=1 --match-with-call-graph 2>&1 | FileCheck %s
+
+# CHECK: BOLT-INFO: matched 1 functions with call graph
+
+#--- main.cpp
+void foo() {}
+
+void bar() {}
+
+void qux() {
+    foo();
+    bar();
+}
+
+void fred() {
+    foo();
+    qux();
+    bar();
+    bar();
+    foo();
+}
+
+int main() {
+    return 0;
+}
+
+#--- yaml
+---
+header:
+  profile-version: 1
+  binary-name:     'match-functions-with-calls-as-anchors.s.tmp.exe'
+  binary-build-id: '<unknown>'
+  profile-flags:   [ lbr ]
+  profile-origin:  branch profile reader
+  profile-events:  ''
+  dfs-order:       false
+  hash-func:       xxh3
+functions:
+  - name:            main
+    fid:             0
+    hash:            0x0000000000000001
+    exec:            1
+    nblocks:         6
+    blocks:
+      - bid:             1
+        hash:            0x0000000000000001
+        insns:           1
+        succ:            [ { bid: 3, cnt: 1} ]
+  - name:            _Z3foov
+    fid:             1
+    hash:            0x0000000000000002
+    exec:            1
+    nblocks:         6
+    blocks:
+      - bid:             1
+        hash:            0x0000000000000002
+        insns:           1
+        succ:            [ { bid: 3, cnt: 1} ]
+
+  - name:            _Z3barv
+    fid:             2
+    hash:            0x0000000000000003
+    exec:            1
+    nblocks:         6
+    blocks:
+      - bid:             1
+        hash:            0x0000000000000003
+        insns:           1
+        succ:            [ { bid: 3, cnt: 1} ]
+  - name:            _Z3quxv
+    fid:             3
+    hash:            0x0000000000000004
+    exec:            4
+    nblocks:         6
+    blocks:
+      - bid:             1
+        hash:            0x0000000000000004
+        insns:           1
+        succ:            [ { bid: 3, cnt: 1} ]
+        calls:           [ { off : 0, fid : 1, cnt : 0},
+                           { off : 0, fid : 2, cnt : 0} ]
+  - name:            _Z4bazv
+    fid:             4
+    hash:            0x0000000000000005
+    exec:            1
+    nblocks:         6
+    blocks:
+      - bid:             1
+        hash:            0x0000000000000005
+        insns:           1
+        succ:            [ { bid: 3, cnt: 1} ]
+        calls:           [ { off : 0, fid : 3, cnt : 0},
+                           { off : 0, fid : 1, cnt : 0},
+                           { off : 0, fid : 2, cnt : 0},
+                           { off : 0, fid : 1, cnt : 0},
+                           { off : 0, fid : 2, cnt : 0} ]
+...

From ffda5212282b0b3ff943690738b314e3ac5254c3 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 19 Jul 2024 16:00:40 -0500
Subject: [PATCH 233/486] [libc++] Make libc++ forward-compatible with
 AppleClang's definition of __builtin_verbose_trap (#99529)

AppleClang as included in the Xcode 16 beta implements
`__builtin_verbose_trap`, but it implements slightly different semantics
from the ones implemented upstream. This patch makes libc++ compatible
with either definition until we drop support for the version of
AppleClang that differs from upstream.
---
 libcxx/vendor/llvm/default_assertion_handler.in | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/libcxx/vendor/llvm/default_assertion_handler.in b/libcxx/vendor/llvm/default_assertion_handler.in
index 9bd831c227798..3b6d6b2cca53c 100644
--- a/libcxx/vendor/llvm/default_assertion_handler.in
+++ b/libcxx/vendor/llvm/default_assertion_handler.in
@@ -24,7 +24,13 @@
 #else
 
 #  if __has_builtin(__builtin_verbose_trap)
-#    define _LIBCPP_ASSERTION_HANDLER(message) __builtin_verbose_trap("libc++", message)
+// AppleClang shipped a slightly different version of __builtin_verbose_trap from the upstream
+// version before upstream Clang actually got the builtin.
+#    if defined(_LIBCPP_APPLE_CLANG_VER) && _LIBCPP_APPLE_CLANG_VER < 17000
+#      define _LIBCPP_ASSERTION_HANDLER(message) __builtin_verbose_trap(message)
+#    else
+#      define _LIBCPP_ASSERTION_HANDLER(message) __builtin_verbose_trap("libc++", message)
+#    endif
 #  else
 #    define _LIBCPP_ASSERTION_HANDLER(message) ((void)message, __builtin_trap())
 #  endif

From d54ec64f6743dee346bd22298577edf1985ce0b8 Mon Sep 17 00:00:00 2001
From: Sayhaan Siddiqui <49014204+sayhaan@users.noreply.github.com>
Date: Fri, 19 Jul 2024 14:03:50 -0700
Subject: [PATCH 234/486] [BOLT][DWARF] Remove deprecated opt (#99575)

Remove deprecated DeterministicDebugInfo option and its uses.
---
 bolt/docs/CommandLineArgumentReference.md |  5 ---
 bolt/lib/Rewrite/DWARFRewriter.cpp        | 37 +++++------------------
 2 files changed, 7 insertions(+), 35 deletions(-)

diff --git a/bolt/docs/CommandLineArgumentReference.md b/bolt/docs/CommandLineArgumentReference.md
index d2cbf0ca86449..ea51b77d838a1 100644
--- a/bolt/docs/CommandLineArgumentReference.md
+++ b/bolt/docs/CommandLineArgumentReference.md
@@ -113,11 +113,6 @@
 
   Prints out offsets for abbrev and debug_info of Skeleton CUs that get patched.
 
-- `--deterministic-debuginfo`
-
-  Disables parallel execution of tasks that may produce nondeterministic debug
-  info
-
 - `--dot-tooltip-code`
 
   Add basic block instructions as tool tips on nodes
diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp
index 4ba6344925856..1ec216b39e95c 100644
--- a/bolt/lib/Rewrite/DWARFRewriter.cpp
+++ b/bolt/lib/Rewrite/DWARFRewriter.cpp
@@ -326,12 +326,6 @@ static cl::opt<bool> KeepARanges(
         "keep or generate .debug_aranges section if .gdb_index is written"),
     cl::Hidden, cl::cat(BoltCategory));
 
-static cl::opt<bool> DeterministicDebugInfo(
-    "deterministic-debuginfo",
-    cl::desc("disables parallel execution of tasks that may produce "
-             "nondeterministic debug info"),
-    cl::init(true), cl::cat(BoltCategory));
-
 static cl::opt<std::string> DwarfOutputPath(
     "dwarf-output-path",
     cl::desc("Path to where .dwo files or dwp file will be written out to."),
@@ -607,11 +601,6 @@ void DWARFRewriter::updateDebugInfo() {
   StrWriter = std::make_unique<DebugStrWriter>(*BC.DwCtx, false);
   StrOffstsWriter = std::make_unique<DebugStrOffsetsWriter>(BC);
 
-  if (!opts::DeterministicDebugInfo) {
-    opts::DeterministicDebugInfo = true;
-    errs() << "BOLT-WARNING: --deterministic-debuginfo is being deprecated\n";
-  }
-
   /// Stores and serializes information that will be put into the
   /// .debug_addr DWARF section.
   std::unique_ptr<DebugAddrWriter> FinalAddrWriter;
@@ -759,25 +748,13 @@ void DWARFRewriter::updateDebugInfo() {
   CUOffsetMap OffsetMap =
       finalizeTypeSections(DIEBlder, *Streamer, GDBIndexSection);
 
-  const bool SingleThreadedMode =
-      opts::NoThreads || opts::DeterministicDebugInfo;
-  if (!SingleThreadedMode)
-    DIEBlder.buildCompileUnits();
-  if (SingleThreadedMode) {
-    CUPartitionVector PartVec = partitionCUs(*BC.DwCtx);
-    for (std::vector<DWARFUnit *> &Vec : PartVec) {
-      DIEBlder.buildCompileUnits(Vec);
-      for (DWARFUnit *CU : DIEBlder.getProcessedCUs())
-        processUnitDIE(CU, &DIEBlder);
-      finalizeCompileUnits(DIEBlder, *Streamer, OffsetMap,
-                           DIEBlder.getProcessedCUs(), *FinalAddrWriter);
-    }
-  } else {
-    // Update unit debug info in parallel
-    ThreadPoolInterface &ThreadPool = ParallelUtilities::getThreadPool();
-    for (std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units())
-      ThreadPool.async(processUnitDIE, CU.get(), &DIEBlder);
-    ThreadPool.wait();
+  CUPartitionVector PartVec = partitionCUs(*BC.DwCtx);
+  for (std::vector<DWARFUnit *> &Vec : PartVec) {
+    DIEBlder.buildCompileUnits(Vec);
+    for (DWARFUnit *CU : DIEBlder.getProcessedCUs())
+      processUnitDIE(CU, &DIEBlder);
+    finalizeCompileUnits(DIEBlder, *Streamer, OffsetMap,
+                         DIEBlder.getProcessedCUs(), *FinalAddrWriter);
   }
 
   DebugNamesTable.emitAccelTable();

From 63625f4406ac9ed419d72f11e701ec94dd72e317 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Fri, 19 Jul 2024 14:11:51 -0700
Subject: [PATCH 235/486] [SandboxIR] Implement LoadInst (#99597)

This patch implements a `LoadInst` instruction in SandboxIR. It mirrors
`llvm::LoadInst`.
---
 llvm/include/llvm/SandboxIR/SandboxIR.h       | 66 ++++++++++++++++--
 .../llvm/SandboxIR/SandboxIRValues.def        |  1 +
 llvm/lib/SandboxIR/SandboxIR.cpp              | 69 +++++++++++++++++--
 llvm/unittests/SandboxIR/SandboxIRTest.cpp    | 31 +++++++++
 4 files changed, 156 insertions(+), 11 deletions(-)

diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index a9f0177eb9338..f168fdf8b1056 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -59,6 +59,7 @@
 #define LLVM_SANDBOXIR_SANDBOXIR_H
 
 #include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
 #include "llvm/SandboxIR/Tracker.h"
@@ -74,6 +75,7 @@ class BasicBlock;
 class Context;
 class Function;
 class Instruction;
+class LoadInst;
 class User;
 class Value;
 
@@ -170,9 +172,10 @@ class Value {
   /// order.
   llvm::Value *Val = nullptr;
 
-  friend class Context; // For getting `Val`.
-  friend class User;    // For getting `Val`.
-  friend class Use;     // For getting `Val`.
+  friend class Context;  // For getting `Val`.
+  friend class User;     // For getting `Val`.
+  friend class Use;      // For getting `Val`.
+  friend class LoadInst; // For getting `Val`.
 
   /// All values point to the context.
   Context &Ctx;
@@ -262,11 +265,14 @@ class Value {
                          llvm::function_ref<bool(const Use &)> ShouldReplace);
   void replaceAllUsesWith(Value *Other);
 
+  /// \Returns the LLVM IR name of the bottom-most LLVM value.
+  StringRef getName() const { return Val->getName(); }
+
 #ifndef NDEBUG
   /// Should crash if there is something wrong with the instruction.
   virtual void verify() const = 0;
-  /// Returns the name in the form 'SB<number>.' like 'SB1.'
-  std::string getName() const;
+  /// Returns the unique id in the form 'SB<number>.' like 'SB1.'
+  std::string getUid() const;
   virtual void dumpCommonHeader(raw_ostream &OS) const;
   void dumpCommonFooter(raw_ostream &OS) const;
   void dumpCommonPrefix(raw_ostream &OS) const;
@@ -489,6 +495,7 @@ class Instruction : public sandboxir::User {
   /// A SandboxIR Instruction may map to multiple LLVM IR Instruction. This
   /// returns its topmost LLVM IR instruction.
   llvm::Instruction *getTopmostLLVMInstruction() const;
+  friend class LoadInst; // For getTopmostLLVMInstruction().
 
   /// \Returns the LLVM IR Instructions that this SandboxIR maps to in program
   /// order.
@@ -553,6 +560,45 @@ class Instruction : public sandboxir::User {
 #endif
 };
 
+class LoadInst final : public Instruction {
+  /// Use LoadInst::create() instead of calling the constructor.
+  LoadInst(llvm::LoadInst *LI, Context &Ctx)
+      : Instruction(ClassID::Load, Opcode::Load, LI, Ctx) {}
+  friend Context; // for LoadInst()
+  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
+    return getOperandUseDefault(OpIdx, Verify);
+  }
+  SmallVector<llvm::Instruction *, 1> getLLVMInstrs() const final {
+    return {cast<llvm::Instruction>(Val)};
+  }
+
+public:
+  unsigned getUseOperandNo(const Use &Use) const final {
+    return getUseOperandNoDefault(Use);
+  }
+
+  unsigned getNumOfIRInstrs() const final { return 1u; }
+  static LoadInst *create(Type *Ty, Value *Ptr, MaybeAlign Align,
+                          Instruction *InsertBefore, Context &Ctx,
+                          const Twine &Name = "");
+  static LoadInst *create(Type *Ty, Value *Ptr, MaybeAlign Align,
+                          BasicBlock *InsertAtEnd, Context &Ctx,
+                          const Twine &Name = "");
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From);
+  Value *getPointerOperand() const;
+  Align getAlign() const { return cast<llvm::LoadInst>(Val)->getAlign(); }
+  bool isUnordered() const { return cast<llvm::LoadInst>(Val)->isUnordered(); }
+  bool isSimple() const { return cast<llvm::LoadInst>(Val)->isSimple(); }
+#ifndef NDEBUG
+  void verify() const final {
+    assert(isa<llvm::LoadInst>(Val) && "Expected LoadInst!");
+  }
+  void dump(raw_ostream &OS) const override;
+  LLVM_DUMP_METHOD void dump() const override;
+#endif
+};
+
 /// An LLLVM Instruction that has no SandboxIR equivalent class gets mapped to
 /// an OpaqueInstr.
 class OpaqueInst : public sandboxir::Instruction {
@@ -683,8 +729,16 @@ class Context {
 
   friend class BasicBlock; // For getOrCreateValue().
 
+  IRBuilder<ConstantFolder> LLVMIRBuilder;
+  auto &getLLVMIRBuilder() { return LLVMIRBuilder; }
+
+  LoadInst *createLoadInst(llvm::LoadInst *LI);
+  friend LoadInst; // For createLoadInst()
+
 public:
-  Context(LLVMContext &LLVMCtx) : LLVMCtx(LLVMCtx), IRTracker(*this) {}
+  Context(LLVMContext &LLVMCtx)
+      : LLVMCtx(LLVMCtx), IRTracker(*this),
+        LLVMIRBuilder(LLVMCtx, ConstantFolder()) {}
 
   Tracker &getTracker() { return IRTracker; }
   /// Convenience function for `getTracker().save()`
diff --git a/llvm/include/llvm/SandboxIR/SandboxIRValues.def b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
index b090ade3ea0ca..e1ed3cdac6bba 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIRValues.def
+++ b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
@@ -25,6 +25,7 @@ DEF_USER(Constant, Constant)
 #endif
 //       ClassID, Opcode(s),  Class
 DEF_INSTR(Opaque, OP(Opaque), OpaqueInst)
+DEF_INSTR(Load, OP(Load), LoadInst)
 
 #ifdef DEF_VALUE
 #undef DEF_VALUE
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 87134995a1538..f392704a6d27e 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -140,14 +140,14 @@ void Value::replaceAllUsesWith(Value *Other) {
 }
 
 #ifndef NDEBUG
-std::string Value::getName() const {
+std::string Value::getUid() const {
   std::stringstream SS;
   SS << "SB" << UID << ".";
   return SS.str();
 }
 
 void Value::dumpCommonHeader(raw_ostream &OS) const {
-  OS << getName() << " " << getSubclassIDStr(SubclassID) << " ";
+  OS << getUid() << " " << getSubclassIDStr(SubclassID) << " ";
 }
 
 void Value::dumpCommonFooter(raw_ostream &OS) const {
@@ -167,7 +167,7 @@ void Value::dumpCommonPrefix(raw_ostream &OS) const {
 }
 
 void Value::dumpCommonSuffix(raw_ostream &OS) const {
-  OS << " ; " << getName() << " (" << getSubclassIDStr(SubclassID) << ")";
+  OS << " ; " << getUid() << " (" << getSubclassIDStr(SubclassID) << ")";
 }
 
 void Value::printAsOperandCommon(raw_ostream &OS) const {
@@ -453,6 +453,49 @@ void Instruction::dump() const {
   dump(dbgs());
   dbgs() << "\n";
 }
+#endif // NDEBUG
+
+LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
+                           Instruction *InsertBefore, Context &Ctx,
+                           const Twine &Name) {
+  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(BeforeIR);
+  auto *NewLI = Builder.CreateAlignedLoad(Ty, Ptr->Val, Align,
+                                          /*isVolatile=*/false, Name);
+  auto *NewSBI = Ctx.createLoadInst(NewLI);
+  return NewSBI;
+}
+
+LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
+                           BasicBlock *InsertAtEnd, Context &Ctx,
+                           const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  auto *NewLI = Builder.CreateAlignedLoad(Ty, Ptr->Val, Align,
+                                          /*isVolatile=*/false, Name);
+  auto *NewSBI = Ctx.createLoadInst(NewLI);
+  return NewSBI;
+}
+
+bool LoadInst::classof(const Value *From) {
+  return From->getSubclassID() == ClassID::Load;
+}
+
+Value *LoadInst::getPointerOperand() const {
+  return Ctx.getValue(cast<llvm::LoadInst>(Val)->getPointerOperand());
+}
+
+#ifndef NDEBUG
+void LoadInst::dump(raw_ostream &OS) const {
+  dumpCommonPrefix(OS);
+  dumpCommonSuffix(OS);
+}
+
+void LoadInst::dump() const {
+  dump(dbgs());
+  dbgs() << "\n";
+}
 
 void OpaqueInst::dump(raw_ostream &OS) const {
   dumpCommonPrefix(OS);
@@ -538,8 +581,8 @@ Value *Context::registerValue(std::unique_ptr<Value> &&VPtr) {
   assert(VPtr->getSubclassID() != Value::ClassID::User &&
          "Can't register a user!");
   Value *V = VPtr.get();
-  llvm::Value *Key = V->Val;
-  LLVMValueToValueMap[Key] = std::move(VPtr);
+  auto Pair = LLVMValueToValueMap.insert({VPtr->Val, std::move(VPtr)});
+  assert(Pair.second && "Already exists!");
   return V;
 }
 
@@ -568,6 +611,17 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
     return nullptr;
   }
   assert(isa<llvm::Instruction>(LLVMV) && "Expected Instruction");
+
+  switch (cast<llvm::Instruction>(LLVMV)->getOpcode()) {
+  case llvm::Instruction::Load: {
+    auto *LLVMLd = cast<llvm::LoadInst>(LLVMV);
+    It->second = std::unique_ptr<LoadInst>(new LoadInst(LLVMLd, *this));
+    return It->second.get();
+  }
+  default:
+    break;
+  }
+
   It->second = std::unique_ptr<OpaqueInst>(
       new OpaqueInst(cast<llvm::Instruction>(LLVMV), *this));
   return It->second.get();
@@ -582,6 +636,11 @@ BasicBlock *Context::createBasicBlock(llvm::BasicBlock *LLVMBB) {
   return BB;
 }
 
+LoadInst *Context::createLoadInst(llvm::LoadInst *LI) {
+  auto NewPtr = std::unique_ptr<LoadInst>(new LoadInst(LI, *this));
+  return cast<LoadInst>(registerValue(std::move(NewPtr)));
+}
+
 Value *Context::getValue(llvm::Value *V) const {
   auto It = LLVMValueToValueMap.find(V);
   if (It != LLVMValueToValueMap.end())
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index ec68ed0afeb2f..04beb429502bc 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -560,3 +560,34 @@ define void @foo(i8 %v1) {
   EXPECT_EQ(I0->getNumUses(), 0u);
   EXPECT_EQ(I0->getNextNode(), Ret);
 }
+
+TEST_F(SandboxIRTest, LoadInst) {
+  parseIR(C, R"IR(
+define void @foo(ptr %arg0, ptr %arg1) {
+  %ld = load i8, ptr %arg0, align 64
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  sandboxir::Function *F = Ctx.createFunction(LLVMF);
+  auto *Arg0 = F->getArg(0);
+  auto *Arg1 = F->getArg(1);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *Ld = cast<sandboxir::LoadInst>(&*It++);
+  auto *Ret = &*It++;
+
+  // Check getPointerOperand()
+  EXPECT_EQ(Ld->getPointerOperand(), Arg0);
+  // Check getAlign()
+  EXPECT_EQ(Ld->getAlign(), 64);
+  // Check create(InsertBefore)
+  sandboxir::LoadInst *NewLd =
+      sandboxir::LoadInst::create(Ld->getType(), Arg1, Align(8),
+                                  /*InsertBefore=*/Ret, Ctx, "NewLd");
+  EXPECT_EQ(NewLd->getType(), Ld->getType());
+  EXPECT_EQ(NewLd->getPointerOperand(), Arg1);
+  EXPECT_EQ(NewLd->getAlign(), 8);
+  EXPECT_EQ(NewLd->getName(), "NewLd");
+}

From 2b371003d101ccc50081c08c7ed5fd7dcbfdef71 Mon Sep 17 00:00:00 2001
From: Angel Zhang <angel.zhang@amd.com>
Date: Fri, 19 Jul 2024 17:12:33 -0400
Subject: [PATCH 236/486] [mlir][docs] Update documentation for
 `vector.multi_reduction`. NFC (#99668)

---
 mlir/include/mlir/Dialect/Vector/IR/VectorOps.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 2019eb5a9fd7f..39ad03c801140 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -300,9 +300,9 @@ def Vector_MultiDimReductionOp :
 
     ```mlir
     %1 = vector.multi_reduction <add>, %0, %acc0 [1, 3] :
-      vector<4x8x16x32xf32> into vector<4x16xf32>
+      vector<4x8x16x32xf32> to vector<4x16xf32>
     %2 = vector.multi_reduction <add>, %1, %acc1 [0, 1] :
-      vector<4x16xf32> into f32
+      vector<4x16xf32> to f32
     ```
   }];
   let builders = [

From 9d03275550d33636a066f84ee3aab81ad1339637 Mon Sep 17 00:00:00 2001
From: Tacet <4922191+AdvenamTacet@users.noreply.github.com>
Date: Fri, 19 Jul 2024 23:24:10 +0200
Subject: [PATCH 237/486] [ASan][libc++] Turn off SSO annotations for Apple
 platforms (#96269)

This commit disables short string AddressSanitizer annotations on Apple
platforms as a temporary solution to the problem reported in #96099.

For more information on Apple's block implementation, please refer to
clang/docs/Block-ABI-Apple.rst [1]. The core issue lies in the fact
that blocks are unaware of their content, causing AddressSanitizer
errors when blocks are moved using `memmove`.

I believe - and I'm not alone - that the issue should ideally be
addressed within the block moving logic. However, this patch provides
a temporary fix until a proper resolution exists in the Blocks runtime.

[1]: https://github.com/llvm/llvm-project/blob/main/clang/docs/Block-ABI-Apple.rst
---
 libcxx/include/string | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libcxx/include/string b/libcxx/include/string
index 54e9d8990c220..2fd1b1e745908 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -1983,6 +1983,11 @@ private:
     (void)__old_mid;
     (void)__new_mid;
 #if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+  #if defined(__APPLE__)
+    // TODO: remove after addressing issue #96099 (https://github.com/llvm/llvm-project/issues/96099)
+    if(!__is_long())
+      return;
+  #endif
     std::__annotate_contiguous_container<_Allocator>(data(), data() + capacity() + 1, __old_mid, __new_mid);
 #endif
   }

From 837d606458f014ceed1b5d4504909f32b83362a8 Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Fri, 19 Jul 2024 17:24:40 -0400
Subject: [PATCH 238/486] [libc] Temporarily disable hypotf sNaN tests for
 NVPTX targets. (#99708)

https://lab.llvm.org/buildbot/#/builders/101/builds/2269
---
 libc/test/src/math/smoke/CMakeLists.txt | 2 ++
 libc/test/src/math/smoke/HypotTest.h    | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index 9384df8f5290d..a02648ea1018b 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -2738,6 +2738,7 @@ add_fp_unittest(
   DEPENDS
     libc.src.math.hypotf
     libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.macros.properties.architectures    
 )
 
 add_fp_unittest(
@@ -2751,6 +2752,7 @@ add_fp_unittest(
   DEPENDS
     libc.src.math.hypot
     libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.macros.properties.architectures
 )
 
 add_fp_unittest(
diff --git a/libc/test/src/math/smoke/HypotTest.h b/libc/test/src/math/smoke/HypotTest.h
index 7921d531a3196..d7c62dcbeb0ed 100644
--- a/libc/test/src/math/smoke/HypotTest.h
+++ b/libc/test/src/math/smoke/HypotTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_HYPOTTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_HYPOTTEST_H
 
+#include "src/__support/macros/properties/architectures.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
@@ -25,8 +26,13 @@ class HypotTestTemplate : public LIBC_NAMESPACE::testing::Test {
     // Pythagorean triples.
     constexpr T PYT[N][3] = {{3, 4, 5}, {5, 12, 13}, {8, 15, 17}, {7, 24, 25}};
 
+#ifndef LIBC_TARGET_ARCH_IS_NVPTX
+    // TODO: Investigate why sNaN tests are failing on nVidia.
+    // https://github.com/llvm/llvm-project/issues/99706.
     EXPECT_FP_EQ(func(inf, sNaN), aNaN);
     EXPECT_FP_EQ(func(sNaN, neg_inf), aNaN);
+#endif // !LIBC_TARGET_ARCH_IS_NVPTX
+
     EXPECT_FP_EQ(func(inf, aNaN), inf);
     EXPECT_FP_EQ(func(aNaN, neg_inf), inf);
     EXPECT_FP_EQ(func(aNaN, aNaN), aNaN);

From 9e74f6600c37131f90736124f7ac4c0901a1746a Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Fri, 19 Jul 2024 22:38:01 +0100
Subject: [PATCH 239/486] [compiler-rt] dump registers for FreeBSD/i386
 (#99702)

---
 .../lib/sanitizer_common/sanitizer_linux.cpp    | 12 ++++++++++++
 .../TestCases/FreeBSD/dump_registers_i386.cpp   | 17 +++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 compiler-rt/test/sanitizer_common/TestCases/FreeBSD/dump_registers_i386.cpp

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index 40c46a8949636..483a1042a6238 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -2262,6 +2262,18 @@ void SignalContext::DumpAllRegisters(void *context) {
   Printf("r14 = 0x%016llx  ", ucontext->uc_mcontext.mc_r14);
   Printf("r15 = 0x%016llx  ", ucontext->uc_mcontext.mc_r15);
   Printf("\n");
+#    elif defined(__i386__)
+  Report("Register values:\n");
+  Printf("eax = 0x%08x  ", ucontext->uc_mcontext.mc_eax);
+  Printf("ebx = 0x%08x  ", ucontext->uc_mcontext.mc_ebx);
+  Printf("ecx = 0x%08x  ", ucontext->uc_mcontext.mc_ecx);
+  Printf("edx = 0x%08x  ", ucontext->uc_mcontext.mc_edx);
+  Printf("\n");
+  Printf("edi = 0x%08x  ", ucontext->uc_mcontext.mc_edi);
+  Printf("esi = 0x%08x  ", ucontext->uc_mcontext.mc_esi);
+  Printf("ebp = 0x%08x  ", ucontext->uc_mcontext.mc_ebp);
+  Printf("esp = 0x%08x  ", ucontext->uc_mcontext.mc_esp);
+  Printf("\n");
 #    else
   (void)ucontext;
 #    endif
diff --git a/compiler-rt/test/sanitizer_common/TestCases/FreeBSD/dump_registers_i386.cpp b/compiler-rt/test/sanitizer_common/TestCases/FreeBSD/dump_registers_i386.cpp
new file mode 100644
index 0000000000000..74aea4d8b360a
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/FreeBSD/dump_registers_i386.cpp
@@ -0,0 +1,17 @@
+// Check that sanitizer prints registers dump_registers on dump_registers=1
+// RUN: %clangxx  %s -o %t
+// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-NODUMP --strict-whitespace
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-DUMP --strict-whitespace
+//
+// REQUIRES: i386-target-arch
+
+#include <signal.h>
+
+int main() {
+  raise(SIGSEGV);
+  // CHECK-DUMP: Register values
+  // CHECK-DUMP-NEXT: eax = {{0x[0-9a-f]+}}  ebx = {{0x[0-9a-f]+}}  ecx = {{0x[0-9a-f]+}}  edx = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: edi = {{0x[0-9a-f]+}}  esi = {{0x[0-9a-f]+}}  ebp = {{0x[0-9a-f]+}}  esp = {{0x[0-9a-f]+}}
+  // CHECK-NODUMP-NOT: Register values
+  return 0;
+}

From 84658fb82b67fc22ecba1560d0cddd09f9104178 Mon Sep 17 00:00:00 2001
From: Edd Dawson <edd.dawson@sony.com>
Date: Fri, 19 Jul 2024 22:38:16 +0100
Subject: [PATCH 240/486] =?UTF-8?q?Revert=20"[PS4/PS5][Driver][DWARF]=20Al?=
 =?UTF-8?q?ways=20emit=20.debug=5Faranges=20for=20SCE=20t=E2=80=A6=20(#997?=
 =?UTF-8?q?11)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…uning (#99629)"

This reverts commit 22eb290a9696e2a3fd042096c61e35eca2fcce0c.
---
 clang/lib/Driver/ToolChains/Clang.cpp         |  7 ++--
 clang/lib/Driver/ToolChains/PS4CPU.cpp        |  8 +++++
 clang/test/Driver/debug-options.c             | 21 ++++++------
 clang/test/Driver/lto-jobs.c                  |  2 +-
 clang/test/Driver/ps4-linker.c                |  8 ++---
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  5 +--
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h      |  3 --
 .../DebugInfo/X86/debug-aranges-sce-tuning.ll | 33 -------------------
 8 files changed, 30 insertions(+), 57 deletions(-)
 delete mode 100644 llvm/test/DebugInfo/X86/debug-aranges-sce-tuning.ll

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index abe517f0f5dea..f7b987bf810c1 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -4659,8 +4659,11 @@ renderDebugOptions(const ToolChain &TC, const Driver &D, const llvm::Triple &T,
 
   // -gdwarf-aranges turns on the emission of the aranges section in the
   // backend.
-  if (const Arg *A = Args.getLastArg(options::OPT_gdwarf_aranges);
-      A && checkDebugInfoOption(A, Args, D, TC)) {
+  // Always enabled for SCE tuning.
+  bool NeedAranges = DebuggerTuning == llvm::DebuggerKind::SCE;
+  if (const Arg *A = Args.getLastArg(options::OPT_gdwarf_aranges))
+    NeedAranges = checkDebugInfoOption(A, Args, D, TC) || NeedAranges;
+  if (NeedAranges) {
     CmdArgs.push_back("-mllvm");
     CmdArgs.push_back("-generate-arange-section");
   }
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.cpp b/clang/lib/Driver/ToolChains/PS4CPU.cpp
index d6af9388e54a6..974e486a0082b 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -162,6 +162,10 @@ void tools::PS4cpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   };
 
   if (UseLTO) {
+    // We default to creating the arange section, but LTO does not. Enable it
+    // here.
+    AddCodeGenFlag("-generate-arange-section");
+
     // This tells LTO to perform JustMyCode instrumentation.
     if (UseJMC)
       AddCodeGenFlag("-enable-jmc-instrument");
@@ -268,6 +272,10 @@ void tools::PS5cpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   };
 
   if (UseLTO) {
+    // We default to creating the arange section, but LTO does not. Enable it
+    // here.
+    AddCodeGenFlag("-generate-arange-section");
+
     // This tells LTO to perform JustMyCode instrumentation.
     if (UseJMC)
       AddCodeGenFlag("-enable-jmc-instrument");
diff --git a/clang/test/Driver/debug-options.c b/clang/test/Driver/debug-options.c
index 21785ba01cb41..0a665f7017d63 100644
--- a/clang/test/Driver/debug-options.c
+++ b/clang/test/Driver/debug-options.c
@@ -118,28 +118,27 @@
 // RUN: %clang_cl -### -c -Z7 -target x86_64-windows-msvc -- %s 2>&1 \
 // RUN:             | FileCheck -check-prefix=G_NOTUNING %s
 
-// On the PS4/PS5, -g defaults to -gno-column-info. We default to always
-// generating the arange section, but keyed off SCE DebuggerTuning being in
-// play during codegen, instead of -generate-arange-section.
+// On the PS4/PS5, -g defaults to -gno-column-info, and we always generate the
+// arange section.
 // RUN: %clang -### -c %s -target x86_64-scei-ps4 2>&1 \
 // RUN:             | FileCheck -check-prefix=NOG_PS %s
 // RUN: %clang -### -c %s -target x86_64-sie-ps5 2>&1 \
 // RUN:             | FileCheck -check-prefix=NOG_PS %s
 /// PS4 will stay on v4 even if the generic default version changes.
 // RUN: %clang -### -c %s -g -target x86_64-scei-ps4 2>&1 \
-// RUN:             | FileCheck -check-prefixes=G_DWARF4,G_SCE,NOCI,FWD_TMPL_PARAMS %s
+// RUN:             | FileCheck -check-prefixes=G_DWARF4,GARANGE,G_SCE,NOCI,FWD_TMPL_PARAMS %s
 // RUN: %clang -### -c %s -g -target x86_64-sie-ps5 2>&1 \
-// RUN:             | FileCheck -check-prefixes=G_DWARF5,G_SCE,NOCI,FWD_TMPL_PARAMS %s
+// RUN:             | FileCheck -check-prefixes=G_DWARF5,GARANGE,G_SCE,NOCI,FWD_TMPL_PARAMS %s
 // RUN: %clang -### -c %s -g -gcolumn-info -target x86_64-scei-ps4 2>&1 \
 // RUN:             | FileCheck -check-prefix=CI %s
 // RUN: %clang -### -c %s -gsce -target x86_64-unknown-linux 2>&1 \
 // RUN:             | FileCheck -check-prefix=NOCI %s
 // RUN: %clang -### %s -g -flto=thin -target x86_64-scei-ps4 2>&1 \
-// RUN:             | FileCheck -check-prefix=LDGARANGE %s
+// RUN:             | FileCheck -check-prefix=SNLDTLTOGARANGE %s
 // RUN: %clang -### %s -g -flto=full -target x86_64-scei-ps4 2>&1 \
-// RUN:             | FileCheck -check-prefix=LDGARANGE %s
+// RUN:             | FileCheck -check-prefix=SNLDFLTOGARANGE %s
 // RUN: %clang -### %s -g -flto -target x86_64-scei-ps5 2>&1 \
-// RUN:             | FileCheck -check-prefix=LDGARANGE %s
+// RUN:             | FileCheck -check-prefix=LLDGARANGE %s
 // RUN: %clang -### %s -g -target x86_64-scei-ps5 2>&1 \
 // RUN:             | FileCheck -check-prefix=LDGARANGE %s
 
@@ -322,7 +321,8 @@
 //
 // NOG_PS: "-cc1"
 // NOG_PS-NOT: "-dwarf-version=
-// NOG_PS-NOT: "-generate-arange-section"
+// NOG_PS: "-generate-arange-section"
+// NOG_PS-NOT: "-dwarf-version=
 //
 // G_ERR: error: unknown argument:
 //
@@ -402,7 +402,8 @@
 //
 
 // LDGARANGE: {{".*ld.*"}} {{.*}}
-// LDGARANGE-NOT: -generate-arange-section"
+// LDGARANGE-NOT: "-plugin-opt=-generate-arange-section"
+// LLDGARANGE: {{".*lld.*"}} {{.*}} "-plugin-opt=-generate-arange-section"
 // SNLDTLTOGARANGE: {{".*orbis-ld.*"}} {{.*}} "-lto-thin-debug-options= -generate-arange-section"
 // SNLDFLTOGARANGE: {{".*orbis-ld.*"}} {{.*}} "-lto-debug-options= -generate-arange-section"
 
diff --git a/clang/test/Driver/lto-jobs.c b/clang/test/Driver/lto-jobs.c
index b4f109e4c502c..43a478b0664d8 100644
--- a/clang/test/Driver/lto-jobs.c
+++ b/clang/test/Driver/lto-jobs.c
@@ -11,7 +11,7 @@
 // RUN: %clang --target=x86_64-scei-ps4 -### %s -flto=thin -flto-jobs=5 2> %t
 // RUN: FileCheck -check-prefix=CHECK-PS4-LINK-THIN-JOBS-ACTION < %t %s
 //
-// CHECK-PS4-LINK-THIN-JOBS-ACTION: "-lto-thin-debug-options= -threads=5"
+// CHECK-PS4-LINK-THIN-JOBS-ACTION: "-lto-thin-debug-options= -generate-arange-section -threads=5"
 
 // RUN: %clang --target=x86_64-apple-darwin13.3.0 -### %s -flto=thin -flto-jobs=5 2> %t
 // RUN: FileCheck -check-prefix=CHECK-LINK-THIN-JOBS2-ACTION < %t %s
diff --git a/clang/test/Driver/ps4-linker.c b/clang/test/Driver/ps4-linker.c
index be989cdd7d5b1..be0103bffe813 100644
--- a/clang/test/Driver/ps4-linker.c
+++ b/clang/test/Driver/ps4-linker.c
@@ -5,8 +5,8 @@
 // RUN: %clang --target=x86_64-scei-ps4 -flto=full -fjmc %s -### 2>&1 | FileCheck --check-prefixes=CHECK-FULL-LTO,CHECK-LIB %s
 
 // CHECK-NOT: -enable-jmc-instrument
-// CHECK-THIN-LTO: "-lto-thin-debug-options= -enable-jmc-instrument"
-// CHECK-FULL-LTO: "-lto-debug-options= -enable-jmc-instrument"
+// CHECK-THIN-LTO: "-lto-thin-debug-options= -generate-arange-section -enable-jmc-instrument"
+// CHECK-FULL-LTO: "-lto-debug-options= -generate-arange-section -enable-jmc-instrument"
 
 // Check the default library name.
 // CHECK-LIB: "--whole-archive" "-lSceDbgJmc" "--no-whole-archive"
@@ -16,5 +16,5 @@
 // RUN: %clang --target=x86_64-scei-ps4 -flto=thin -fcrash-diagnostics-dir=mydumps %s -### 2>&1 | FileCheck --check-prefixes=CHECK-DIAG-THIN-LTO %s
 // RUN: %clang --target=x86_64-scei-ps4 -flto=full -fcrash-diagnostics-dir=mydumps %s -### 2>&1 | FileCheck --check-prefixes=CHECK-DIAG-FULL-LTO %s
 
-// CHECK-DIAG-THIN-LTO: "-lto-thin-debug-options= -crash-diagnostics-dir=mydumps"
-// CHECK-DIAG-FULL-LTO: "-lto-debug-options= -crash-diagnostics-dir=mydumps"
+// CHECK-DIAG-THIN-LTO: "-lto-thin-debug-options= -generate-arange-section -crash-diagnostics-dir=mydumps"
+// CHECK-DIAG-FULL-LTO: "-lto-debug-options= -generate-arange-section -crash-diagnostics-dir=mydumps"
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 578a90d402c54..80cd5ec501f25 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -354,9 +354,6 @@ DwarfDebug::DwarfDebug(AsmPrinter *A)
 
   UseLocSection = !TT.isNVPTX();
 
-  // Always emit .debug_aranges for SCE tuning.
-  UseARangesSection = GenerateARangeSection || tuneForSCE();
-
   HasAppleExtensionAttributes = tuneForLLDB();
 
   // Handle split DWARF.
@@ -1453,7 +1450,7 @@ void DwarfDebug::endModule() {
   emitDebugInfo();
 
   // Emit info into a debug aranges section.
-  if (UseARangesSection)
+  if (GenerateARangeSection)
     emitDebugARanges();
 
   // Emit info into a debug ranges section.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 13f4c379e0027..452485b632c45 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -435,9 +435,6 @@ class DwarfDebug : public DebugHandlerBase {
   ///Allow emission of the .debug_loc section.
   bool UseLocSection = true;
 
-  /// Allow emission of .debug_aranges section
-  bool UseARangesSection = false;
-
   /// Generate DWARF v4 type units.
   bool GenerateTypeUnits;
 
diff --git a/llvm/test/DebugInfo/X86/debug-aranges-sce-tuning.ll b/llvm/test/DebugInfo/X86/debug-aranges-sce-tuning.ll
deleted file mode 100644
index ac0d61aef2b1c..0000000000000
--- a/llvm/test/DebugInfo/X86/debug-aranges-sce-tuning.ll
+++ /dev/null
@@ -1,33 +0,0 @@
-; This checks that .debug_aranges is always generated for the SCE debugger
-; tuning.
-
-; RUN: llc -mtriple=x86_64 -debugger-tune=sce -filetype=obj %s -o %t
-; RUN: llvm-dwarfdump -debug-aranges %t | FileCheck %s
-
-; CHECK:      .debug_aranges contents:
-; CHECK-NEXT: Address Range Header:
-; CHECK-SAME:   length = 0x0000002c,
-
-; IR generated and reduced from:
-; $ cat foo.c
-; int foo;
-; $ clang -g -S -emit-llvm foo.c -o foo.ll
-
-target triple = "x86_64-unknown-linux-gnu"
-
-@foo = dso_local global i32 0, align 4, !dbg !0
-
-!llvm.dbg.cu = !{!2}
-!llvm.module.flags = !{!6, !7, !8}
-!llvm.ident = !{!9}
-
-!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
-!1 = distinct !DIGlobalVariable(name: "foo", scope: !2, file: !3, line: 1, type: !5, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C11, file: !3, producer: "clang version 19.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None)
-!3 = !DIFile(filename: "foo.c", directory: "/tmp")
-!4 = !{!0}
-!5 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!6 = !{i32 7, !"Dwarf Version", i32 5}
-!7 = !{i32 2, !"Debug Info Version", i32 3}
-!8 = !{i32 1, !"wchar_size", i32 4}
-!9 = !{!"clang version 19.0.0"}

From a2f61ba08bebe0be50d3622f2535bc3a1d7e414f Mon Sep 17 00:00:00 2001
From: aaryanshukla <53713108+aaryanshukla@users.noreply.github.com>
Date: Fri, 19 Jul 2024 14:40:34 -0700
Subject: [PATCH 241/486] [libc][math]fadd implementation (#99694)

- **[libc] math fadd**
- **[libc][math] implemented fadd**
---
 libc/config/gpu/entrypoints.txt           |  1 +
 libc/config/linux/aarch64/entrypoints.txt |  1 +
 libc/config/linux/arm/entrypoints.txt     |  1 +
 libc/config/linux/riscv/entrypoints.txt   |  1 +
 libc/config/linux/x86_64/entrypoints.txt  |  1 +
 libc/config/windows/entrypoints.txt       |  1 +
 libc/docs/math/index.rst                  |  2 +-
 libc/spec/stdc.td                         |  1 +
 libc/src/math/CMakeLists.txt              |  1 +
 libc/src/math/fadd.h                      | 20 ++++++++++++++++++++
 libc/src/math/generic/CMakeLists.txt      | 12 ++++++++++++
 libc/src/math/generic/fadd.cpp            | 20 ++++++++++++++++++++
 libc/test/src/math/CMakeLists.txt         | 14 ++++++++++++++
 libc/test/src/math/fadd_test.cpp          | 13 +++++++++++++
 libc/test/src/math/smoke/CMakeLists.txt   | 14 ++++++++++++++
 libc/test/src/math/smoke/fadd_test.cpp    | 13 +++++++++++++
 16 files changed, 115 insertions(+), 1 deletion(-)
 create mode 100644 libc/src/math/fadd.h
 create mode 100644 libc/src/math/generic/fadd.cpp
 create mode 100644 libc/test/src/math/fadd_test.cpp
 create mode 100644 libc/test/src/math/smoke/fadd_test.cpp

diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt
index b8eb743cf587a..506c7d6d7b314 100644
--- a/libc/config/gpu/entrypoints.txt
+++ b/libc/config/gpu/entrypoints.txt
@@ -266,6 +266,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.expm1f
     libc.src.math.fabs
     libc.src.math.fabsf
+    libc.src.math.fadd
     libc.src.math.fdim
     libc.src.math.fdimf
     libc.src.math.floor
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 208889ba34a59..e2f6bd74bb694 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -370,6 +370,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fabs
     libc.src.math.fabsf
     libc.src.math.fabsl
+    libc.src.math.fadd 
     libc.src.math.fdim
     libc.src.math.fdimf
     libc.src.math.fdiml
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index a72f8668808a5..0d09e4c22953c 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -239,6 +239,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fabs
     libc.src.math.fabsf
     libc.src.math.fabsl
+    libc.src.math.fadd 
     libc.src.math.fdim
     libc.src.math.fdimf
     libc.src.math.fdiml
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 266c94d54a9df..33dd8d06173b2 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -371,6 +371,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fabs
     libc.src.math.fabsf
     libc.src.math.fabsl
+    libc.src.math.fadd 
     libc.src.math.fdim
     libc.src.math.fdimf
     libc.src.math.fdiml
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index cbdee084aa199..7309e95644c74 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -396,6 +396,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fabs
     libc.src.math.fabsf
     libc.src.math.fabsl
+    libc.src.math.fadd 
     libc.src.math.fdim
     libc.src.math.fdimf
     libc.src.math.fdiml
diff --git a/libc/config/windows/entrypoints.txt b/libc/config/windows/entrypoints.txt
index afc9ca87ff094..b6aced83c5815 100644
--- a/libc/config/windows/entrypoints.txt
+++ b/libc/config/windows/entrypoints.txt
@@ -144,6 +144,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fabs
     libc.src.math.fabsf
     libc.src.math.fabsl
+    libc.src.math.fadd
     libc.src.math.fdim
     libc.src.math.fdimf
     libc.src.math.fdiml
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index 5fab9ce4df949..cff7f69deb7f8 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -136,7 +136,7 @@ Basic Operations
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | fabs             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.7.3               | F.10.4.3                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fadd             | N/A              |                 |                        | N/A                  |                        | 7.12.14.1              | F.10.11                    |
+| fadd             | N/A              | |check|         | |check|                | N/A                  | |check|                | 7.12.14.1              | F.10.11                    |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | fdim             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.1              | F.10.9.1                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 18592e92d330a..285166ef0be78 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -402,6 +402,7 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"fabsl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
           GuardedFunctionSpec<"fabsf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fabsf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+          FunctionSpec<"fadd", RetValSpec<FloatType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
 
           FunctionSpec<"fdim", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fdimf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index c4e33130e9090..050bf0f1ebf22 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -141,6 +141,7 @@ add_math_entrypoint_object(fabsf)
 add_math_entrypoint_object(fabsl)
 add_math_entrypoint_object(fabsf16)
 add_math_entrypoint_object(fabsf128)
+add_math_entrypoint_object(fadd)
 
 add_math_entrypoint_object(fdim)
 add_math_entrypoint_object(fdimf)
diff --git a/libc/src/math/fadd.h b/libc/src/math/fadd.h
new file mode 100644
index 0000000000000..ec3ce18bb676a
--- /dev/null
+++ b/libc/src/math/fadd.h
@@ -0,0 +1,20 @@
+//===-- Implementation of fadd function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/macros/config.h"
+
+#ifndef LLVM_LIBC_SRC_MATH_FADD_H
+#define LLVM_LIBC_SRC_MATH_FADD_H
+
+namespace LIBC_NAMESPACE_DECL {
+
+float fadd(double x, double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FADD_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index d775026fabb3e..8e8ffb01adf60 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -452,6 +452,18 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  fadd
+  SRCS
+    fadd.cpp
+  HDRS
+    ../fadd.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   trunc
   SRCS
diff --git a/libc/src/math/generic/fadd.cpp b/libc/src/math/generic/fadd.cpp
new file mode 100644
index 0000000000000..66e5188cbcfd4
--- /dev/null
+++ b/libc/src/math/generic/fadd.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fadd function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fadd.h"
+#include "src/__support/FPUtil/generic/add_sub.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float, fadd, (double x, double y)) {
+  return fputil::generic::add<float>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index c28385f620cfd..f69b0d9b1b666 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -174,6 +174,20 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fadd_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    fadd_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.src.math.fadd
+    libc.src.__support.FPUtil.basic_operations
+)
+
 add_fp_unittest(
   trunc_test
   NEED_MPFR
diff --git a/libc/test/src/math/fadd_test.cpp b/libc/test/src/math/fadd_test.cpp
new file mode 100644
index 0000000000000..fe9ac8b252ed3
--- /dev/null
+++ b/libc/test/src/math/fadd_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fadd ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/math/fadd.h"
+
+LIST_ADD_TESTS(float, double, LIBC_NAMESPACE::fadd)
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index a02648ea1018b..69f53c5d27f15 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -141,6 +141,20 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fadd_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fadd_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.src.math.fadd
+    libc.src.__support.FPUtil.basic_operations
+
+)
+
 add_fp_unittest(
   trunc_test
   SUITE
diff --git a/libc/test/src/math/smoke/fadd_test.cpp b/libc/test/src/math/smoke/fadd_test.cpp
new file mode 100644
index 0000000000000..fe9ac8b252ed3
--- /dev/null
+++ b/libc/test/src/math/smoke/fadd_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fadd ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/math/fadd.h"
+
+LIST_ADD_TESTS(float, double, LIBC_NAMESPACE::fadd)

From 56ffbd97fda16008d02180a460211829354f1094 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar@redhat.com>
Date: Fri, 19 Jul 2024 14:47:35 -0700
Subject: [PATCH 242/486] [workflows] Avoid usage of access token in
 issue-write.yml (#94011)

This adds a new composite workflow that allows you to download artifacts
from other workflows without using an access token.

actions/download-artifact from GitHub requires an access token in order
to download artifacts from a different workflow, which is why we can't
use it here if we want to avoid using a token.

See
https://github.com/actions/download-artifact?tab=readme-ov-file#download-artifacts-from-other-workflow-runs-or-repositories
---
 .github/workflows/issue-write.yml             | 17 +++-
 .../unprivileged-download-artifact/action.yml | 81 +++++++++++++++++++
 2 files changed, 94 insertions(+), 4 deletions(-)
 create mode 100644 .github/workflows/unprivileged-download-artifact/action.yml

diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml
index d4814a2fe9014..5334157a7fd20 100644
--- a/.github/workflows/issue-write.yml
+++ b/.github/workflows/issue-write.yml
@@ -24,14 +24,21 @@ jobs:
         github.event.workflow_run.conclusion == 'failure'
       )
     steps:
+      - name: Fetch Sources
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/workflows/unprivileged-download-artifact/action.yml
+          sparse-checkout-cone-mode: false
       - name: 'Download artifact'
-        uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+        uses: ./.github/workflows/unprivileged-download-artifact
+        id: download-artifact
         with:
-          github-token: ${{ secrets.ISSUE_WRITE_DOWNLOAD_ARTIFACT }}
           run-id: ${{ github.event.workflow_run.id }}
-          name: workflow-args
+          artifact-name: workflow-args
 
       - name: 'Comment on PR'
+        if: steps.download-artifact.outputs.artifact-id != ''
         uses: actions/github-script@v3
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -144,5 +151,7 @@ jobs:
             });
 
       - name: Dump comments file
-        if: always()
+        if: >-
+          always() &&
+          steps.download-artifact.outputs.artifact-id != ''
         run: cat comments
diff --git a/.github/workflows/unprivileged-download-artifact/action.yml b/.github/workflows/unprivileged-download-artifact/action.yml
new file mode 100644
index 0000000000000..9d8fb59a67c0e
--- /dev/null
+++ b/.github/workflows/unprivileged-download-artifact/action.yml
@@ -0,0 +1,81 @@
+name: Unprivileged Download Artifact
+description: >-
+  Download artifacts from another workflow run without using an access token.
+inputs:
+  run-id:
+    description: >-
+      The run-id for the workflow run that you want to download the artifact
+      from.  If ommitted it will download the most recently created artifact
+      from the repo with the artifact-name.
+    required: false
+  artifact-name:
+    desciption: The name of the artifact to download.
+    required: true
+
+
+outputs:
+  filename:
+    description: >-
+      The filename of the downloaded artifact or the empty string if the
+      artifact was not found.
+    value: ${{ steps.download-artifact.outputs.filename }}
+  artifact-id:
+    description: "The id of the artifact being downloaded."
+    value: ${{ steps.artifact-url.outputs.id }}
+
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+      id: artifact-url
+      with:
+        script: |
+          var response;
+          if (!"${{ inputs.run-id }}") {
+            response = await github.rest.actions.listArtifactsForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: "${{ inputs.artifact-name }}"
+            })
+          } else {
+            response = await github.rest.actions.listWorkflowRunArtifacts({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: "${{ inputs.run-id }}",
+              name: "${{ inputs.artifact-name }}"
+            })
+          }
+
+          console.log(response)
+
+          for (artifact of response.data.artifacts) {
+            console.log(artifact);
+          }
+
+          if (response.data.artifacts.length == 0) {
+            console.log("Could not find artifact ${{ inputs.artifact-name }} for workflow run ${{ inputs.run-id }}")
+            return;
+          }
+
+          const url_response = await github.rest.actions.downloadArtifact({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            artifact_id: response.data.artifacts[0].id,
+            archive_format: "zip"
+          })
+
+          core.setOutput("url", url_response.url);
+          core.setOutput("id", response.data.artifacts[0].id);
+
+    - shell: bash
+      if: steps.artifact-url.outputs.url != ''
+      id: download-artifact
+      run: |
+        curl -L -o ${{ inputs.artifact-name }}.zip "${{ steps.artifact-url.outputs.url }}"
+        echo "filename=${{ inputs.artifact-name }}.zip" >> $GITHUB_OUTPUT
+
+    - shell: bash
+      if: steps.download-artifact.outputs.filename != ''
+      run: |
+        unzip ${{ steps.download-artifact.outputs.filename }}

From fada9227325b3eaa0bdc09a486f29a7f08b7b3fb Mon Sep 17 00:00:00 2001
From: Jacob Lalonde <jalalonde@fb.com>
Date: Fri, 19 Jul 2024 15:49:22 -0700
Subject: [PATCH 243/486] [LLDB][SBSaveCoreOptions] Fix TestProcessSaveCore
 (#99692)

In #98403 some of the tests were transitioned to the new
`SBProcess::SaveCore(SBSaveCoreOptions)` API, but were not detected in
testing. This patch addresses that.
---
 .../functionalities/process_save_core/TestProcessSaveCore.py  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/test/API/functionalities/process_save_core/TestProcessSaveCore.py b/lldb/test/API/functionalities/process_save_core/TestProcessSaveCore.py
index 07d06bdc116ec..8573d15733927 100644
--- a/lldb/test/API/functionalities/process_save_core/TestProcessSaveCore.py
+++ b/lldb/test/API/functionalities/process_save_core/TestProcessSaveCore.py
@@ -20,8 +20,8 @@ def test_cannot_save_core_unless_process_stopped(self):
         target = self.dbg.CreateTarget(exe)
         process = target.LaunchSimple(None, None, self.get_process_working_directory())
         self.assertNotEqual(process.GetState(), lldb.eStateStopped)
-        options = SBSaveCoreOptions()
-        options.SetOutputFile(SBFileSpec(core))
+        options = lldb.SBSaveCoreOptions()
+        options.SetOutputFile(lldb.SBFileSpec(core))
         error = process.SaveCore(core)
         self.assertTrue(error.Fail())
 

From b686600a57f37a938d5ede54b789d6b3543561b0 Mon Sep 17 00:00:00 2001
From: Daniel Hill <dhhillaz@gmail.com>
Date: Fri, 19 Jul 2024 16:52:01 -0700
Subject: [PATCH 244/486] [BOLT] Skip instruction shortening (#93032)

Add the ability to disable the instruction shortening pass through
--shorten-instructions=false
---
 bolt/lib/Rewrite/BinaryPassManager.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp
index aaa0e1ff4d46f..5dfef0b71cc79 100644
--- a/bolt/lib/Rewrite/BinaryPassManager.cpp
+++ b/bolt/lib/Rewrite/BinaryPassManager.cpp
@@ -263,6 +263,10 @@ static cl::opt<bool> CMOVConversionFlag("cmov-conversion",
                                         cl::ReallyHidden,
                                         cl::cat(BoltOptCategory));
 
+static cl::opt<bool> ShortenInstructions("shorten-instructions",
+                                         cl::desc("shorten instructions"),
+                                         cl::init(true),
+                                         cl::cat(BoltOptCategory));
 } // namespace opts
 
 namespace llvm {
@@ -378,7 +382,8 @@ Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
   else if (opts::Hugify)
     Manager.registerPass(std::make_unique<HugePage>(NeverPrint));
 
-  Manager.registerPass(std::make_unique<ShortenInstructions>(NeverPrint));
+  Manager.registerPass(std::make_unique<ShortenInstructions>(NeverPrint),
+                       opts::ShortenInstructions);
 
   Manager.registerPass(std::make_unique<RemoveNops>(NeverPrint),
                        !opts::KeepNops);

From b172f4aabe7918362cbac54e394657c37627238c Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Sat, 20 Jul 2024 07:53:12 +0800
Subject: [PATCH 245/486] [SandboxIR] Fix -Wunused-variable in SandboxIR.cpp
 (NFC)

/llvm-project/llvm/lib/SandboxIR/SandboxIR.cpp:584:8:
error: unused variable 'Pair' [-Werror,-Wunused-variable]
  auto Pair = LLVMValueToValueMap.insert({VPtr->Val, std::move(VPtr)});
       ^
1 error generated.
---
 llvm/lib/SandboxIR/SandboxIR.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index f392704a6d27e..d49ec0c91e599 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -581,7 +581,8 @@ Value *Context::registerValue(std::unique_ptr<Value> &&VPtr) {
   assert(VPtr->getSubclassID() != Value::ClassID::User &&
          "Can't register a user!");
   Value *V = VPtr.get();
-  auto Pair = LLVMValueToValueMap.insert({VPtr->Val, std::move(VPtr)});
+  [[maybe_unused]] auto Pair =
+         LLVMValueToValueMap.insert({VPtr->Val, std::move(VPtr)});
   assert(Pair.second && "Already exists!");
   return V;
 }

From 7f563232d685f0070d2c6bc74a134e4019dcadc1 Mon Sep 17 00:00:00 2001
From: Itis-hard2name <chenwei@xfusion.com>
Date: Sat, 20 Jul 2024 07:55:21 +0800
Subject: [PATCH 246/486] [bolt][Docs] fix missing option in cmake of stage3 in
 OptimizingClang.md (#93684)

Fixes #93681
---
 bolt/docs/OptimizingClang.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bolt/docs/OptimizingClang.md b/bolt/docs/OptimizingClang.md
index ff7e71b6a76bc..685fcc2b738fa 100644
--- a/bolt/docs/OptimizingClang.md
+++ b/bolt/docs/OptimizingClang.md
@@ -49,6 +49,7 @@ $ cd ${TOPLEV}/stage3
 $ CPATH=${TOPLEV}/stage2-prof-use-lto/install/bin/
 $ cmake -G Ninja ${TOPLEV}/llvm -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_C_COMPILER=$CPATH/clang -DCMAKE_CXX_COMPILER=$CPATH/clang++ \
+    -DLLVM_ENABLE_PROJECTS="clang" \
     -DLLVM_USE_LINKER=lld -DCMAKE_INSTALL_PREFIX=${TOPLEV}/stage3/install
 $ perf record -e cycles:u -j any,u -- ninja clang
 ```

From 1ee8238f0eab75386522a002c2cbb6146284c0c1 Mon Sep 17 00:00:00 2001
From: klensy <klensy@users.noreply.github.com>
Date: Sat, 20 Jul 2024 02:57:14 +0300
Subject: [PATCH 247/486] [BOLT][test] Fix Filecheck typos (#93979)

Fixes few FileCheck typos in tests and add missing(?) filecheck call in
test.

Co-authored-by: klensy <nightouser@gmail.com>
---
 bolt/test/AArch64/update-debug-reloc.test                | 2 +-
 bolt/test/AArch64/veneer-gold.s                          | 4 ++--
 bolt/test/X86/dwarf5-df-types-modify-dwo-name-mixed.test | 2 +-
 bolt/test/X86/dwarf5-one-loclists-two-bases.test         | 2 +-
 bolt/test/X86/dwarf5-two-loclists.test                   | 2 +-
 bolt/test/X86/dwarf5-two-rnglists.test                   | 4 ++--
 6 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/bolt/test/AArch64/update-debug-reloc.test b/bolt/test/AArch64/update-debug-reloc.test
index d57f42a3852a5..dd83229ea7143 100644
--- a/bolt/test/AArch64/update-debug-reloc.test
+++ b/bolt/test/AArch64/update-debug-reloc.test
@@ -2,7 +2,7 @@
 # update-debug-sections option.
 
 RUN: %clang %cflags -g %p/../Inputs/asm_foo.s %p/../Inputs/asm_main.c -o %t.exe
-RUN: llvm-bolt %t.exe -o %t --update-debug-sections
+RUN: llvm-bolt %t.exe -o %t --update-debug-sections 2>&1 | FileCheck %s
 
 CHECK: BOLT-INFO: Target architecture: aarch64
 CHECK-NOT: Reloc num: 10
diff --git a/bolt/test/AArch64/veneer-gold.s b/bolt/test/AArch64/veneer-gold.s
index 3b3e34ecb1a9f..275febce2b372 100644
--- a/bolt/test/AArch64/veneer-gold.s
+++ b/bolt/test/AArch64/veneer-gold.s
@@ -29,7 +29,7 @@ dummy:
 .type foo, %function
 foo:
 # CHECK: <foo>:
-# CHECK-NEXT : {{.*}} bl {{.*}} <foo2>
+# CHECK-NEXT: {{.*}} bl {{.*}} <foo2>
   bl .L2
   ret
 .size foo, .-foo
@@ -38,7 +38,7 @@ foo:
 .type foo2, %function
 foo2:
 # CHECK: <foo2>:
-# CHECK-NEXT : {{.*}} bl {{.*}} <foo2>
+# CHECK-NEXT: {{.*}} bl {{.*}} <foo2>
   bl .L2
   ret
 .size foo2, .-foo2
diff --git a/bolt/test/X86/dwarf5-df-types-modify-dwo-name-mixed.test b/bolt/test/X86/dwarf5-df-types-modify-dwo-name-mixed.test
index a4f5ee77ab565..6c603ba4ee19d 100644
--- a/bolt/test/X86/dwarf5-df-types-modify-dwo-name-mixed.test
+++ b/bolt/test/X86/dwarf5-df-types-modify-dwo-name-mixed.test
@@ -90,7 +90,7 @@
 ; BOLT-DWP: DW_TAG_compile_unit
 ; BOLT-DWP: DW_AT_dwo_name  ("main.dwo.dwo")
 ; BOLT-DWP: DW_TAG_type_unit
-; BOLT-DW-NOT: DW_AT_dwo_name
+; BOLT-DWP-NOT: DW_AT_dwo_name
 ; BOLT-DWP:       Contribution size = 68, Format = DWARF32, Version = 5
 ; BOLT-DWP-NEXT: "main"
 ; BOLT-DWP-NEXT: "int"
diff --git a/bolt/test/X86/dwarf5-one-loclists-two-bases.test b/bolt/test/X86/dwarf5-one-loclists-two-bases.test
index 873512aad5e8d..f25f6c7a46858 100644
--- a/bolt/test/X86/dwarf5-one-loclists-two-bases.test
+++ b/bolt/test/X86/dwarf5-one-loclists-two-bases.test
@@ -34,7 +34,7 @@
 # POSTCHECK: version = 0x0005
 # POSTCHECK: DW_AT_loclists_base [DW_FORM_sec_offset]	(0x0000000c)
 # POSTCHECK: DW_AT_rnglists_base [DW_FORM_sec_offset]	(0x0000000c)
-# POSTCHECK-EMPTY
+# POSTCHECK-EMPTY:
 # POSTCHECK: DW_TAG_variable
 # POSTCHECK: DW_AT_location [DW_FORM_loclistx]
 # POSTCHECK-SAME: indexed (0x0)
diff --git a/bolt/test/X86/dwarf5-two-loclists.test b/bolt/test/X86/dwarf5-two-loclists.test
index 2ede02f3b76fb..a7c6351f9813c 100644
--- a/bolt/test/X86/dwarf5-two-loclists.test
+++ b/bolt/test/X86/dwarf5-two-loclists.test
@@ -45,7 +45,7 @@
 # POSTCHECK: version = 0x0005
 # POSTCHECK: DW_AT_loclists_base [DW_FORM_sec_offset]	(0x0000000c)
 # POSTCHECK: DW_AT_rnglists_base [DW_FORM_sec_offset]	(0x0000000c)
-# POSTCHECK-EMPTY
+# POSTCHECK-EMPTY:
 # POSTCHECK: DW_TAG_variable
 # POSTCHECK: DW_AT_location [DW_FORM_loclistx]
 # POSTCHECK-SAME: indexed (0x0)
diff --git a/bolt/test/X86/dwarf5-two-rnglists.test b/bolt/test/X86/dwarf5-two-rnglists.test
index 17cdc7643bae5..98f2e347d7673 100644
--- a/bolt/test/X86/dwarf5-two-rnglists.test
+++ b/bolt/test/X86/dwarf5-two-rnglists.test
@@ -52,7 +52,7 @@
 # POSTCHECK-NEXT: DW_AT_addr_base [DW_FORM_sec_offset]  (0x00000008)
 # POSTCHECK-NEXT: DW_AT_loclists_base [DW_FORM_sec_offset]	(0x0000000c)
 # POSTCHECK-NEXT: DW_AT_rnglists_base [DW_FORM_sec_offset]	(0x0000000c)
-# POSTCHECK-EMPTY
+# POSTCHECK-EMPTY:
 # POSTCHECK: DW_TAG_subprogram
 # POSTCHECK-NEXT: DW_AT_ranges [DW_FORM_rnglistx]
 # POSTCHECK-SAME: indexed (0x1)
@@ -75,7 +75,7 @@
 # POSTCHECK-NEXT: DW_AT_addr_base [DW_FORM_sec_offset]  (0x00000030)
 # POSTCHECK-NEXT: DW_AT_loclists_base [DW_FORM_sec_offset]	(0x00000045)
 # POSTCHECK-NEXT: DW_AT_rnglists_base [DW_FORM_sec_offset]	(0x00000035)
-# POSTCHECK-EMPTY
+# POSTCHECK-EMPTY:
 
 # POSTCHECK: DW_TAG_subprogram
 # POSTCHECK-NEXT: DW_AT_ranges [DW_FORM_rnglistx]

From 8bc02bf5c6e94489a79c8d924e5d9866fcc18417 Mon Sep 17 00:00:00 2001
From: Eisuke Kawashima <e.kawaschima+github@gmail.com>
Date: Sat, 20 Jul 2024 08:59:56 +0900
Subject: [PATCH 248/486] fix(bolt/**.py): fix comparison to None (#94012)

from PEP8
(https://peps.python.org/pep-0008/#programming-recommendations):

> Comparisons to singletons like None should always be done with is or
is not, never the equality operators.

Co-authored-by: Eisuke Kawashima <e-kwsm@users.noreply.github.com>
---
 bolt/docs/generate_doc.py         | 4 ++--
 bolt/test/perf2bolt/lit.local.cfg | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bolt/docs/generate_doc.py b/bolt/docs/generate_doc.py
index d8829daf677b4..763dc00b44ca3 100644
--- a/bolt/docs/generate_doc.py
+++ b/bolt/docs/generate_doc.py
@@ -45,7 +45,7 @@ def parse_bolt_options(output):
         cleaned_line = line.strip()
 
         if cleaned_line.casefold() in map(str.casefold, section_headers):
-            if prev_section != None:  # Save last option from prev section
+            if prev_section is not None:  # Save last option from prev section
                 add_info(sections, current_section, option, description)
                 option, description = None, []
 
@@ -76,7 +76,7 @@ def parse_bolt_options(output):
                 description = [descr]
                 if option.startswith("--print") or option.startswith("--time"):
                     current_section = "BOLT printing options:"
-                elif prev_section != None:
+                elif prev_section is not None:
                     current_section = prev_section
             continue
 
diff --git a/bolt/test/perf2bolt/lit.local.cfg b/bolt/test/perf2bolt/lit.local.cfg
index 05f41ff333b0e..4ee9ad08cc78a 100644
--- a/bolt/test/perf2bolt/lit.local.cfg
+++ b/bolt/test/perf2bolt/lit.local.cfg
@@ -1,4 +1,4 @@
 import shutil
 
-if shutil.which("perf") != None:
+if shutil.which("perf") is not None:
     config.available_features.add("perf")
\ No newline at end of file

From be7f1827ff180df7d3f585432048e784eaa932ee Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 19 Jul 2024 17:13:20 -0700
Subject: [PATCH 249/486] [LV] Use llvm::all_of in
 LoopVectorizationCostModel::getMaximizedVFForTarget. NFC (#99585)

---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ff60bd894cd40..6b050c138552b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4594,15 +4594,12 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
 
     // Select the largest VF which doesn't require more registers than existing
     // ones.
-    for (int i = RUs.size() - 1; i >= 0; --i) {
-      bool Selected = true;
-      for (auto &pair : RUs[i].MaxLocalUsers) {
-        unsigned TargetNumRegisters = TTI.getNumberOfRegisters(pair.first);
-        if (pair.second > TargetNumRegisters)
-          Selected = false;
-      }
-      if (Selected) {
-        MaxVF = VFs[i];
+    for (int I = RUs.size() - 1; I >= 0; --I) {
+      const auto &MLU = RUs[I].MaxLocalUsers;
+      if (all_of(MLU, [&](decltype(MLU.front()) &LU) {
+            return LU.second <= TTI.getNumberOfRegisters(LU.first);
+          })) {
+        MaxVF = VFs[I];
         break;
       }
     }

From 05f0e86cc895181b3d2210458c78938f83353002 Mon Sep 17 00:00:00 2001
From: Jason Molenda <jmolenda@apple.com>
Date: Fri, 19 Jul 2024 17:26:13 -0700
Subject: [PATCH 250/486] [lldb] Change lldb's breakpoint handling behavior
 (#96260)

lldb today has two rules: When a thread stops at a BreakpointSite, we
set the thread's StopReason to be "breakpoint hit" (regardless if we've
actually hit the breakpoint, or if we've merely stopped *at* the
breakpoint instruction/point and haven't tripped it yet). And second,
when resuming a process, any thread sitting at a BreakpointSite is
silently stepped over the BreakpointSite -- because we've already
flagged the breakpoint hit when we stopped there originally.

In this patch, I change lldb to only set a thread's stop reason to
breakpoint-hit when we've actually executed the instruction/triggered
the breakpoint. When we resume, we only silently step past a
BreakpointSite that we've registered as hit. We preserve this state
across inferior function calls that the user may do while stopped, etc.

Also, when a user adds a new breakpoint at $pc while stopped, or changes
$pc to be the address of a BreakpointSite, we will silently step past
that breakpoint when the process resumes. This is purely a UX call, I
don't think there's any person who wants to set a breakpoint at $pc and
then hit it immediately on resuming.

One non-intuitive UX from this change, but I'm convinced it is
necessary: If you're stopped at a BreakpointSite that has not yet
executed, you `stepi`, you will hit the breakpoint and the pc will not
yet advance. This thread has not completed its stepi, and the thread
plan is still on the stack. If you then `continue` the thread, lldb will
now stop and say, "instruction step completed", one instruction past the
BreakpointSite. You can continue a second time to resume execution. I
discussed this with Jim, and trying to paper over this behavior will
lead to more complicated scenarios behaving non-intuitively. And mostly
it's the testsuite that was trying to instruction step past a breakpoint
and getting thrown off -- and I changed those tests to expect the new
behavior.

The bugs driving this change are all from lldb dropping the real stop
reason for a thread and setting it to breakpoint-hit when that was not
the case. Jim hit one where we have an aarch64 watchpoint that triggers
one instruction before a BreakpointSite. On this arch we are notified of
the watchpoint hit after the instruction has been unrolled -- we disable
the watchpoint, instruction step, re-enable the watchpoint and collect
the new value. But now we're on a BreakpointSite so the watchpoint-hit
stop reason is lost.

Another was reported by ZequanWu in
https://discourse.llvm.org/t/lldb-unable-to-break-at-start/78282 we
attach to/launch a process with the pc at a BreakpointSite and
misbehave. Caroline Tice mentioned it is also a problem they've had with
putting a breakpoint on _dl_debug_state.

The change to each Process plugin that does execution control is that

1. If we've stopped at a BreakpointSite that has not been executed yet,
we will call Thread::SetThreadStoppedAtUnexecutedBP(pc) to record
that.  When the thread resumes, if the pc is still at the same site, we
will continue, hit the breakpoint, and stop again.

2. When we've actually hit a breakpoint (enabled for this thread or not),
the Process plugin should call Thread::SetThreadHitBreakpointSite().
When we go to resume the thread, we will push a step-over-breakpoint
ThreadPlan before resuming.

The biggest set of changes is to StopInfoMachException where we
translate a Mach Exception into a stop reason. The Mach exception codes
differ in a few places depending on the target (unambiguously), and I
didn't want to duplicate the new code for each target so I've tested
what mach exceptions we get for each action on each target, and
reorganized StopInfoMachException::CreateStopReasonWithMachException to
document these possible values, and handle them without specializing
based on the target arch.

rdar://123942164
---
 lldb/include/lldb/Target/Thread.h             |  24 ++
 .../Process/Utility/StopInfoMachException.cpp | 322 ++++++++----------
 .../Process/Windows/Common/ProcessWindows.cpp |  32 +-
 .../Process/gdb-remote/ProcessGDBRemote.cpp   |  93 ++---
 .../Process/scripted/ScriptedThread.cpp       |  11 +
 lldb/source/Target/StopInfo.cpp               |   2 +
 lldb/source/Target/Thread.cpp                 |  15 +-
 .../TestConsecutiveBreakpoints.py             |  26 +-
 .../TestStepOverBreakpoint.py                 |   6 +-
 9 files changed, 266 insertions(+), 265 deletions(-)

diff --git a/lldb/include/lldb/Target/Thread.h b/lldb/include/lldb/Target/Thread.h
index 2ff1f50d497e2..d3e429e9256df 100644
--- a/lldb/include/lldb/Target/Thread.h
+++ b/lldb/include/lldb/Target/Thread.h
@@ -129,6 +129,7 @@ class Thread : public std::enable_shared_from_this<Thread>,
         register_backup_sp; // You need to restore the registers, of course...
     uint32_t current_inlined_depth;
     lldb::addr_t current_inlined_pc;
+    lldb::addr_t stopped_at_unexecuted_bp;
   };
 
   /// Constructor
@@ -378,6 +379,26 @@ class Thread : public std::enable_shared_from_this<Thread>,
 
   virtual void SetQueueLibdispatchQueueAddress(lldb::addr_t dispatch_queue_t) {}
 
+  /// When a thread stops at an enabled BreakpointSite that has not executed,
+  /// the Process plugin should call SetThreadStoppedAtUnexecutedBP(pc).
+  /// If that BreakpointSite was actually triggered (the instruction was
+  /// executed, for a software breakpoint), regardless of whether the
+  /// breakpoint is valid for this thread, SetThreadHitBreakpointSite()
+  /// should be called to record that fact.
+  ///
+  /// Depending on the structure of the Process plugin, it may be easiest
+  /// to call SetThreadStoppedAtUnexecutedBP(pc) unconditionally when at
+  /// a BreakpointSite, and later when it is known that it was triggered,
+  /// SetThreadHitBreakpointSite() can be called.  These two methods
+  /// overwrite the same piece of state in the Thread, the last one
+  /// called on a Thread wins.
+  void SetThreadStoppedAtUnexecutedBP(lldb::addr_t pc) {
+    m_stopped_at_unexecuted_bp = pc;
+  }
+  void SetThreadHitBreakpointSite() {
+    m_stopped_at_unexecuted_bp = LLDB_INVALID_ADDRESS;
+  }
+
   /// Whether this Thread already has all the Queue information cached or not
   ///
   /// A Thread may be associated with a libdispatch work Queue at a given
@@ -1312,6 +1333,9 @@ class Thread : public std::enable_shared_from_this<Thread>,
   bool m_should_run_before_public_stop;  // If this thread has "stop others" 
                                          // private work to do, then it will
                                          // set this.
+  lldb::addr_t m_stopped_at_unexecuted_bp; // Set to the address of a breakpoint
+                                           // instruction that we have not yet
+                                           // hit, but will hit when we resume.
   const uint32_t m_index_id; ///< A unique 1 based index assigned to each thread
                              /// for easy UI/command line access.
   lldb::RegisterContextSP m_reg_context_sp; ///< The register context for this
diff --git a/lldb/source/Plugins/Process/Utility/StopInfoMachException.cpp b/lldb/source/Plugins/Process/Utility/StopInfoMachException.cpp
index 25cee369d7ee3..f1853be12638e 100644
--- a/lldb/source/Plugins/Process/Utility/StopInfoMachException.cpp
+++ b/lldb/source/Plugins/Process/Utility/StopInfoMachException.cpp
@@ -488,38 +488,6 @@ const char *StopInfoMachException::GetDescription() {
   return m_description.c_str();
 }
 
-static StopInfoSP GetStopInfoForHardwareBP(Thread &thread, Target *target,
-                                           uint32_t exc_data_count,
-                                           uint64_t exc_sub_code,
-                                           uint64_t exc_sub_sub_code) {
-  // Try hardware watchpoint.
-  if (target) {
-    // The exc_sub_code indicates the data break address.
-    WatchpointResourceSP wp_rsrc_sp =
-        target->GetProcessSP()->GetWatchpointResourceList().FindByAddress(
-            (addr_t)exc_sub_code);
-    if (wp_rsrc_sp && wp_rsrc_sp->GetNumberOfConstituents() > 0) {
-      return StopInfo::CreateStopReasonWithWatchpointID(
-          thread, wp_rsrc_sp->GetConstituentAtIndex(0)->GetID());
-    }
-  }
-
-  // Try hardware breakpoint.
-  ProcessSP process_sp(thread.GetProcess());
-  if (process_sp) {
-    // The exc_sub_code indicates the data break address.
-    lldb::BreakpointSiteSP bp_sp =
-        process_sp->GetBreakpointSiteList().FindByAddress(
-            (lldb::addr_t)exc_sub_code);
-    if (bp_sp && bp_sp->IsEnabled()) {
-      return StopInfo::CreateStopReasonWithBreakpointSiteID(thread,
-                                                            bp_sp->GetID());
-    }
-  }
-
-  return nullptr;
-}
-
 #if defined(__APPLE__)
 const char *
 StopInfoMachException::MachException::Name(exception_type_t exc_type) {
@@ -607,6 +575,25 @@ StopInfoSP StopInfoMachException::CreateStopReasonWithMachException(
       target ? target->GetArchitecture().GetMachine()
              : llvm::Triple::UnknownArch;
 
+  ProcessSP process_sp(thread.GetProcess());
+  RegisterContextSP reg_ctx_sp(thread.GetRegisterContext());
+  // Caveat: with x86 KDP if we've hit a breakpoint, the pc we
+  // receive is past the breakpoint instruction.
+  // If we have a breakpoints at 0x100 and 0x101, we hit the
+  // 0x100 breakpoint and the pc is reported at 0x101.
+  // We will initially mark this thread as being stopped at an
+  // unexecuted breakpoint at 0x101. Later when we see that
+  // we stopped for a Breakpoint reason, we will decrement the
+  // pc, and update the thread to record that we hit the
+  // breakpoint at 0x100.
+  // The fact that the pc may be off by one at this point
+  // (for an x86 KDP breakpoint hit) is not a problem.
+  addr_t pc = reg_ctx_sp->GetPC();
+  BreakpointSiteSP bp_site_sp =
+      process_sp->GetBreakpointSiteList().FindByAddress(pc);
+  if (bp_site_sp && bp_site_sp->IsEnabled())
+    thread.SetThreadStoppedAtUnexecutedBP(pc);
+
   switch (exc_type) {
   case 1: // EXC_BAD_ACCESS
   case 2: // EXC_BAD_INSTRUCTION
@@ -633,171 +620,158 @@ StopInfoSP StopInfoMachException::CreateStopReasonWithMachException(
     }
     break;
 
+    // A mach exception comes with 2-4 pieces of data.
+    // The sub-codes are only provided for certain types
+    // of mach exceptions.
+    // [exc_type, exc_code, exc_sub_code, exc_sub_sub_code]
+    //
+    // Here are all of the EXC_BREAKPOINT, exc_type==6,
+    // exceptions we can receive.
+    //
+    // Instruction step:
+    //   [6, 1, 0]
+    //   Intel KDP [6, 3, ??]
+    //   armv7 [6, 0x102, <stop-pc>]  Same as software breakpoint!
+    //
+    // Software breakpoint:
+    //   x86 [6, 2, 0]
+    //   Intel KDP [6, 2, <bp-addr + 1>]
+    //   arm64 [6, 1, <bp-addr>]
+    //   armv7 [6, 0x102, <bp-addr>]  Same as instruction step!
+    //
+    // Hardware breakpoint:
+    //   x86 [6, 1, <bp-addr>, 0]
+    //   x86/Rosetta not implemented, see software breakpoint
+    //   arm64 [6, 1, <bp-addr>]
+    //   armv7 not implemented, see software breakpoint
+    //
+    // Hardware watchpoint:
+    //   x86 [6, 1, <accessed-addr>, 0] (both Intel hw and Rosetta)
+    //   arm64 [6, 0x102, <accessed-addr>, 0]
+    //   armv7 [6, 0x102, <accessed-addr>, 0]
+    //
+    // arm64 BRK instruction (imm arg not reflected in the ME)
+    //   [ 6, 1, <addr-of-BRK-insn>]
+    //
+    // In order of codes mach exceptions:
+    //   [6, 1, 0] - instruction step
+    //   [6, 1, <bp-addr>] - hardware breakpoint or watchpoint
+    //
+    //   [6, 2, 0] - software breakpoint
+    //   [6, 2, <bp-addr + 1>] - software breakpoint
+    //
+    //   [6, 3] - instruction step
+    //
+    //   [6, 0x102, <stop-pc>] armv7 instruction step
+    //   [6, 0x102, <bp-addr>] armv7 software breakpoint
+    //   [6, 0x102, <accessed-addr>, 0] arm64/armv7 watchpoint
+
   case 6: // EXC_BREAKPOINT
   {
-    bool is_actual_breakpoint = false;
-    bool is_trace_if_actual_breakpoint_missing = false;
-    switch (cpu) {
-    case llvm::Triple::x86:
-    case llvm::Triple::x86_64:
-      if (exc_code == 1) // EXC_I386_SGL
-      {
-        if (!exc_sub_code) {
-          // This looks like a plain trap.
-          // Have to check if there is a breakpoint here as well.  When you
-          // single-step onto a trap, the single step stops you not to trap.
-          // Since we also do that check below, let's just use that logic.
-          is_actual_breakpoint = true;
-          is_trace_if_actual_breakpoint_missing = true;
-        } else {
-          if (StopInfoSP stop_info =
-                  GetStopInfoForHardwareBP(thread, target, exc_data_count,
-                                           exc_sub_code, exc_sub_sub_code))
-            return stop_info;
-        }
-      } else if (exc_code == 2 || // EXC_I386_BPT
-                 exc_code == 3)   // EXC_I386_BPTFLT
-      {
-        // KDP returns EXC_I386_BPTFLT for trace breakpoints
-        if (exc_code == 3)
-          is_trace_if_actual_breakpoint_missing = true;
-
-        is_actual_breakpoint = true;
+    bool stopped_by_hitting_breakpoint = false;
+    bool stopped_by_completing_stepi = false;
+    bool stopped_watchpoint = false;
+    std::optional<addr_t> address;
+
+    // exc_code 1
+    if (exc_code == 1) {
+      if (exc_sub_code == 0) {
+        stopped_by_completing_stepi = true;
+      } else {
+        // Ambiguous: could be signalling a
+        // breakpoint or watchpoint hit.
+        stopped_by_hitting_breakpoint = true;
+        stopped_watchpoint = true;
+        address = exc_sub_code;
+      }
+    }
+
+    // exc_code 2
+    if (exc_code == 2) {
+      if (exc_sub_code == 0)
+        stopped_by_hitting_breakpoint = true;
+      else {
+        stopped_by_hitting_breakpoint = true;
+        // Intel KDP software breakpoint
         if (!pc_already_adjusted)
           pc_decrement = 1;
       }
-      break;
+    }
 
-    case llvm::Triple::arm:
-    case llvm::Triple::thumb:
-      if (exc_code == 0x102) // EXC_ARM_DA_DEBUG
-      {
-        // LWP_TODO: We need to find the WatchpointResource that matches
-        // the address, and evaluate its Watchpoints.
-
-        // It's a watchpoint, then, if the exc_sub_code indicates a
-        // known/enabled data break address from our watchpoint list.
-        lldb::WatchpointSP wp_sp;
-        if (target)
-          wp_sp = target->GetWatchpointList().FindByAddress(
-              (lldb::addr_t)exc_sub_code);
-        if (wp_sp && wp_sp->IsEnabled()) {
-          return StopInfo::CreateStopReasonWithWatchpointID(thread,
-                                                            wp_sp->GetID());
-        } else {
-          is_actual_breakpoint = true;
-          is_trace_if_actual_breakpoint_missing = true;
-        }
-      } else if (exc_code == 1) // EXC_ARM_BREAKPOINT
-      {
-        is_actual_breakpoint = true;
-        is_trace_if_actual_breakpoint_missing = true;
-      } else if (exc_code == 0) // FIXME not EXC_ARM_BREAKPOINT but a kernel
-                                // is currently returning this so accept it
-                                // as indicating a breakpoint until the
-                                // kernel is fixed
-      {
-        is_actual_breakpoint = true;
-        is_trace_if_actual_breakpoint_missing = true;
-      }
-      break;
+    // exc_code 3
+    if (exc_code == 3)
+      stopped_by_completing_stepi = true;
 
-    case llvm::Triple::aarch64_32:
-    case llvm::Triple::aarch64: {
-      // xnu describes three things with type EXC_BREAKPOINT:
-      //
-      //   exc_code 0x102 [EXC_ARM_DA_DEBUG], exc_sub_code addr-of-insn
-      //      Watchpoint access.  exc_sub_code is the address of the
-      //      instruction which trigged the watchpoint trap.
-      //      debugserver may add the watchpoint number that was triggered
-      //      in exc_sub_sub_code.
-      //
-      //   exc_code 1 [EXC_ARM_BREAKPOINT], exc_sub_code 0
-      //      Instruction step has completed.
-      //
-      //   exc_code 1 [EXC_ARM_BREAKPOINT], exc_sub_code address-of-instruction
-      //      Software breakpoint instruction executed.
-
-      if (exc_code == 1 && exc_sub_code == 0) // EXC_ARM_BREAKPOINT
-      {
-        // This is hit when we single instruction step aka MDSCR_EL1 SS bit 0
-        // is set
-        is_actual_breakpoint = true;
-        is_trace_if_actual_breakpoint_missing = true;
-        if (thread.GetTemporaryResumeState() != eStateStepping)
-          not_stepping_but_got_singlestep_exception = true;
-      }
-      if (exc_code == 0x102) // EXC_ARM_DA_DEBUG
-      {
-        // LWP_TODO: We need to find the WatchpointResource that matches
-        // the address, and evaluate its Watchpoints.
-
-        // It's a watchpoint, then, if the exc_sub_code indicates a
-        // known/enabled data break address from our watchpoint list.
-        lldb::WatchpointSP wp_sp;
-        if (target)
-          wp_sp = target->GetWatchpointList().FindByAddress(
-              (lldb::addr_t)exc_sub_code);
-        if (wp_sp && wp_sp->IsEnabled()) {
-          return StopInfo::CreateStopReasonWithWatchpointID(thread,
-                                                            wp_sp->GetID());
-        }
-        // EXC_ARM_DA_DEBUG seems to be reused for EXC_BREAKPOINT as well as
-        // EXC_BAD_ACCESS
-        if (thread.GetTemporaryResumeState() == eStateStepping)
-          return StopInfo::CreateStopReasonToTrace(thread);
+    // exc_code 0x102
+    if (exc_code == 0x102 && exc_sub_code != 0) {
+      if (cpu == llvm::Triple::arm || cpu == llvm::Triple::thumb) {
+        stopped_by_hitting_breakpoint = true;
+        stopped_by_completing_stepi = true;
       }
-      // It looks like exc_sub_code has the 4 bytes of the instruction that
-      // triggered the exception, i.e. our breakpoint opcode
-      is_actual_breakpoint = exc_code == 1;
-      break;
+      stopped_watchpoint = true;
+      address = exc_sub_code;
     }
 
-    default:
-      break;
-    }
+    // The Mach Exception may have been ambiguous --
+    // e.g. we stopped either because of a breakpoint
+    // or a watchpoint.  We'll disambiguate which it
+    // really was.
 
-    if (is_actual_breakpoint) {
-      RegisterContextSP reg_ctx_sp(thread.GetRegisterContext());
+    if (stopped_by_hitting_breakpoint) {
       addr_t pc = reg_ctx_sp->GetPC() - pc_decrement;
 
-      ProcessSP process_sp(thread.CalculateProcess());
-
-      lldb::BreakpointSiteSP bp_site_sp;
-      if (process_sp)
+      if (address)
+        bp_site_sp =
+            process_sp->GetBreakpointSiteList().FindByAddress(*address);
+      if (!bp_site_sp && reg_ctx_sp) {
         bp_site_sp = process_sp->GetBreakpointSiteList().FindByAddress(pc);
+      }
       if (bp_site_sp && bp_site_sp->IsEnabled()) {
-        // Update the PC if we were asked to do so, but only do so if we find
-        // a breakpoint that we know about cause this could be a trap
-        // instruction in the code
-        if (pc_decrement > 0 && adjust_pc_if_needed)
-          reg_ctx_sp->SetPC(pc);
-
-        // If the breakpoint is for this thread, then we'll report the hit,
-        // but if it is for another thread, we can just report no reason.  We
-        // don't need to worry about stepping over the breakpoint here, that
-        // will be taken care of when the thread resumes and notices that
-        // there's a breakpoint under the pc. If we have an operating system
-        // plug-in, we might have set a thread specific breakpoint using the
-        // operating system thread ID, so we can't make any assumptions about
-        // the thread ID so we must always report the breakpoint regardless
-        // of the thread.
+        // We've hit this breakpoint, whether it was intended for this thread
+        // or not.  Clear this in the Tread object so we step past it on resume.
+        thread.SetThreadHitBreakpointSite();
+
+        // If we have an operating system plug-in, we might have set a thread
+        // specific breakpoint using the operating system thread ID, so we
+        // can't make any assumptions about the thread ID so we must always
+        // report the breakpoint regardless of the thread.
         if (bp_site_sp->ValidForThisThread(thread) ||
-            thread.GetProcess()->GetOperatingSystem() != nullptr)
+            thread.GetProcess()->GetOperatingSystem() != nullptr) {
+          // Update the PC if we were asked to do so, but only do so if we find
+          // a breakpoint that we know about cause this could be a trap
+          // instruction in the code
+          if (pc_decrement > 0 && adjust_pc_if_needed && reg_ctx_sp)
+            reg_ctx_sp->SetPC(pc);
           return StopInfo::CreateStopReasonWithBreakpointSiteID(
               thread, bp_site_sp->GetID());
-        else if (is_trace_if_actual_breakpoint_missing)
-          return StopInfo::CreateStopReasonToTrace(thread);
-        else
+        } else {
           return StopInfoSP();
+        }
       }
+    }
 
-      // Don't call this a trace if we weren't single stepping this thread.
-      if (is_trace_if_actual_breakpoint_missing &&
-          thread.GetTemporaryResumeState() == eStateStepping) {
-        return StopInfo::CreateStopReasonToTrace(thread);
+    // Breakpoint-hit events are handled.
+    // Now handle watchpoints.
+
+    if (stopped_watchpoint && address) {
+      WatchpointResourceSP wp_rsrc_sp =
+          target->GetProcessSP()->GetWatchpointResourceList().FindByAddress(
+              *address);
+      if (wp_rsrc_sp && wp_rsrc_sp->GetNumberOfConstituents() > 0) {
+        return StopInfo::CreateStopReasonWithWatchpointID(
+            thread, wp_rsrc_sp->GetConstituentAtIndex(0)->GetID());
       }
     }
+
+    // Finally, handle instruction step.
+
+    if (stopped_by_completing_stepi) {
+      if (thread.GetTemporaryResumeState() != eStateStepping)
+        not_stepping_but_got_singlestep_exception = true;
+      else
+        return StopInfo::CreateStopReasonToTrace(thread);
+    }
+
   } break;
 
   case 7:  // EXC_SYSCALL
diff --git a/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
index f383b3d40a4f3..8b12b87eacbc6 100644
--- a/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
@@ -377,24 +377,17 @@ void ProcessWindows::RefreshStateAfterStop() {
   if (!stop_thread)
     return;
 
-  switch (active_exception->GetExceptionCode()) {
-  case EXCEPTION_SINGLE_STEP: {
-    RegisterContextSP register_context = stop_thread->GetRegisterContext();
-    const uint64_t pc = register_context->GetPC();
-    BreakpointSiteSP site(GetBreakpointSiteList().FindByAddress(pc));
-    if (site && site->ValidForThisThread(*stop_thread)) {
-      LLDB_LOG(log,
-               "Single-stepped onto a breakpoint in process {0} at "
-               "address {1:x} with breakpoint site {2}",
-               m_session_data->m_debugger->GetProcess().GetProcessId(), pc,
-               site->GetID());
-      stop_info = StopInfo::CreateStopReasonWithBreakpointSiteID(*stop_thread,
-                                                                 site->GetID());
-      stop_thread->SetStopInfo(stop_info);
+  RegisterContextSP register_context = stop_thread->GetRegisterContext();
+  uint64_t pc = register_context->GetPC();
 
-      return;
-    }
+  // If we're at a BreakpointSite, mark this as an Unexecuted Breakpoint.
+  // We'll clear that state if we've actually executed the breakpoint.
+  BreakpointSiteSP site(GetBreakpointSiteList().FindByAddress(pc));
+  if (site && site->IsEnabled())
+    stop_thread->SetThreadStoppedAtUnexecutedBP(pc);
 
+  switch (active_exception->GetExceptionCode()) {
+  case EXCEPTION_SINGLE_STEP: {
     auto *reg_ctx = static_cast<RegisterContextWindows *>(
         stop_thread->GetRegisterContext().get());
     uint32_t slot_id = reg_ctx->GetTriggeredHardwareBreakpointSlotId();
@@ -419,8 +412,6 @@ void ProcessWindows::RefreshStateAfterStop() {
   }
 
   case EXCEPTION_BREAKPOINT: {
-    RegisterContextSP register_context = stop_thread->GetRegisterContext();
-
     int breakpoint_size = 1;
     switch (GetTarget().GetArchitecture().GetMachine()) {
     case llvm::Triple::aarch64:
@@ -443,9 +434,9 @@ void ProcessWindows::RefreshStateAfterStop() {
     }
 
     // The current PC is AFTER the BP opcode, on all architectures.
-    uint64_t pc = register_context->GetPC() - breakpoint_size;
+    pc = register_context->GetPC() - breakpoint_size;
 
-    BreakpointSiteSP site(GetBreakpointSiteList().FindByAddress(pc));
+    site = GetBreakpointSiteList().FindByAddress(pc);
     if (site) {
       LLDB_LOG(log,
                "detected breakpoint in process {0} at address {1:x} with "
@@ -453,6 +444,7 @@ void ProcessWindows::RefreshStateAfterStop() {
                m_session_data->m_debugger->GetProcess().GetProcessId(), pc,
                site->GetID());
 
+      stop_thread->SetThreadHitBreakpointSite();
       if (site->ValidForThisThread(*stop_thread)) {
         LLDB_LOG(log,
                  "Breakpoint site {0} is valid for this thread ({1:x}), "
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 604c92369e9a2..9ab03c1fb8ff3 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -1598,29 +1598,8 @@ bool ProcessGDBRemote::CalculateThreadStopInfo(ThreadGDBRemote *thread) {
     // that have stop reasons, and if there is no entry for a thread, then it
     // has no stop reason.
     thread->GetRegisterContext()->InvalidateIfNeeded(true);
-    if (!GetThreadStopInfoFromJSON(thread, m_jstopinfo_sp)) {
-      // If a thread is stopped at a breakpoint site, set that as the stop
-      // reason even if it hasn't executed the breakpoint instruction yet.
-      // We will silently step over the breakpoint when we resume execution
-      // and miss the fact that this thread hit the breakpoint.
-      const size_t num_thread_ids = m_thread_ids.size();
-      for (size_t i = 0; i < num_thread_ids; i++) {
-        if (m_thread_ids[i] == thread->GetID() && m_thread_pcs.size() > i) {
-          addr_t pc = m_thread_pcs[i];
-          lldb::BreakpointSiteSP bp_site_sp =
-              thread->GetProcess()->GetBreakpointSiteList().FindByAddress(pc);
-          if (bp_site_sp) {
-            if (bp_site_sp->ValidForThisThread(*thread)) {
-              thread->SetStopInfo(
-                  StopInfo::CreateStopReasonWithBreakpointSiteID(
-                      *thread, bp_site_sp->GetID()));
-              return true;
-            }
-          }
-        }
-      }
+    if (!GetThreadStopInfoFromJSON(thread, m_jstopinfo_sp))
       thread->SetStopInfo(StopInfoSP());
-    }
     return true;
   }
 
@@ -1729,6 +1708,12 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo(
     if (ThreadSP memory_thread_sp = m_thread_list.GetBackingThread(thread_sp))
       thread_sp = memory_thread_sp;
 
+    addr_t pc = thread_sp->GetRegisterContext()->GetPC();
+    BreakpointSiteSP bp_site_sp =
+        thread_sp->GetProcess()->GetBreakpointSiteList().FindByAddress(pc);
+    if (bp_site_sp && bp_site_sp->IsEnabled())
+      thread_sp->SetThreadStoppedAtUnexecutedBP(pc);
+
     if (exc_type != 0) {
       const size_t exc_data_size = exc_data.size();
 
@@ -1745,26 +1730,10 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo(
       // to no reason.
       if (!reason.empty() && reason != "none") {
         if (reason == "trace") {
-          addr_t pc = thread_sp->GetRegisterContext()->GetPC();
-          lldb::BreakpointSiteSP bp_site_sp =
-              thread_sp->GetProcess()->GetBreakpointSiteList().FindByAddress(
-                  pc);
-
-          // If the current pc is a breakpoint site then the StopInfo should be
-          // set to Breakpoint Otherwise, it will be set to Trace.
-          if (bp_site_sp && bp_site_sp->ValidForThisThread(*thread_sp)) {
-            thread_sp->SetStopInfo(
-                StopInfo::CreateStopReasonWithBreakpointSiteID(
-                    *thread_sp, bp_site_sp->GetID()));
-          } else
-            thread_sp->SetStopInfo(
-                StopInfo::CreateStopReasonToTrace(*thread_sp));
+          thread_sp->SetStopInfo(StopInfo::CreateStopReasonToTrace(*thread_sp));
           handled = true;
         } else if (reason == "breakpoint") {
-          addr_t pc = thread_sp->GetRegisterContext()->GetPC();
-          lldb::BreakpointSiteSP bp_site_sp =
-              thread_sp->GetProcess()->GetBreakpointSiteList().FindByAddress(
-                  pc);
+          thread_sp->SetThreadHitBreakpointSite();
           if (bp_site_sp) {
             // If the breakpoint is for this thread, then we'll report the hit,
             // but if it is for another thread, we can just report no reason.
@@ -1880,39 +1849,41 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo(
               StopInfo::CreateStopReasonVForkDone(*thread_sp));
           handled = true;
         }
-      } else if (!signo) {
-        addr_t pc = thread_sp->GetRegisterContext()->GetPC();
-        lldb::BreakpointSiteSP bp_site_sp =
-            thread_sp->GetProcess()->GetBreakpointSiteList().FindByAddress(pc);
-
-        // If a thread is stopped at a breakpoint site, set that as the stop
-        // reason even if it hasn't executed the breakpoint instruction yet.
-        // We will silently step over the breakpoint when we resume execution
-        // and miss the fact that this thread hit the breakpoint.
-        if (bp_site_sp && bp_site_sp->ValidForThisThread(*thread_sp)) {
-          thread_sp->SetStopInfo(StopInfo::CreateStopReasonWithBreakpointSiteID(
-              *thread_sp, bp_site_sp->GetID()));
-          handled = true;
-        }
       }
 
       if (!handled && signo && !did_exec) {
         if (signo == SIGTRAP) {
           // Currently we are going to assume SIGTRAP means we are either
           // hitting a breakpoint or hardware single stepping.
+
+          // We can't disambiguate between stepping-to-a-breakpointsite and
+          // hitting-a-breakpointsite.
+          //
+          // A user can instruction-step, and be stopped at a BreakpointSite.
+          // Or a user can be sitting at a BreakpointSite,
+          // instruction-step which hits the breakpoint and the pc does not
+          // advance.
+          //
+          // In both cases, we're at a BreakpointSite when stopped, and
+          // the resume state was eStateStepping.
+
+          // Assume if we're at a BreakpointSite, we hit it.
           handled = true;
           addr_t pc =
               thread_sp->GetRegisterContext()->GetPC() + m_breakpoint_pc_offset;
-          lldb::BreakpointSiteSP bp_site_sp =
+          BreakpointSiteSP bp_site_sp =
               thread_sp->GetProcess()->GetBreakpointSiteList().FindByAddress(
                   pc);
 
+          // We can't know if we hit it or not. So if we are stopped at
+          // a BreakpointSite, assume we hit it, and should step past the
+          // breakpoint when we resume. This is contrary to how we handle
+          // BreakpointSites in any other location, but we can't know for
+          // sure what happened so it's a reasonable default.
           if (bp_site_sp) {
-            // If the breakpoint is for this thread, then we'll report the hit,
-            // but if it is for another thread, we can just report no reason.
-            // We don't need to worry about stepping over the breakpoint here,
-            // that will be taken care of when the thread resumes and notices
-            // that there's a breakpoint under the pc.
+            if (bp_site_sp->IsEnabled())
+              thread_sp->SetThreadHitBreakpointSite();
+
             if (bp_site_sp->ValidForThisThread(*thread_sp)) {
               if (m_breakpoint_pc_offset != 0)
                 thread_sp->GetRegisterContext()->SetPC(pc);
@@ -1926,8 +1897,6 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo(
           } else {
             // If we were stepping then assume the stop was the result of the
             // trace.  If we were not stepping then report the SIGTRAP.
-            // FIXME: We are still missing the case where we single step over a
-            // trap instruction.
             if (thread_sp->GetTemporaryResumeState() == eStateStepping)
               thread_sp->SetStopInfo(
                   StopInfo::CreateStopReasonToTrace(*thread_sp));
diff --git a/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp b/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
index 88a4ca3b0389f..d0d1508e85172 100644
--- a/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
+++ b/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
@@ -229,6 +229,17 @@ bool ScriptedThread::CalculateStopInfo() {
         LLVM_PRETTY_FUNCTION, "Failed to get scripted thread stop info.", error,
         LLDBLog::Thread);
 
+  // If we're at a BreakpointSite, mark that we stopped there and
+  // need to hit the breakpoint when we resume.  This will be cleared
+  // if we CreateStopReasonWithBreakpointSiteID.
+  if (RegisterContextSP reg_ctx_sp = GetRegisterContext()) {
+    addr_t pc = reg_ctx_sp->GetPC();
+    if (BreakpointSiteSP bp_site_sp =
+            GetProcess()->GetBreakpointSiteList().FindByAddress(pc))
+      if (bp_site_sp->IsEnabled())
+        SetThreadStoppedAtUnexecutedBP(pc);
+  }
+
   lldb::StopInfoSP stop_info_sp;
   lldb::StopReason stop_reason_type;
 
diff --git a/lldb/source/Target/StopInfo.cpp b/lldb/source/Target/StopInfo.cpp
index 95f78056b1644..895306b5ef0d4 100644
--- a/lldb/source/Target/StopInfo.cpp
+++ b/lldb/source/Target/StopInfo.cpp
@@ -1365,6 +1365,8 @@ class StopInfoVForkDone : public StopInfo {
 
 StopInfoSP StopInfo::CreateStopReasonWithBreakpointSiteID(Thread &thread,
                                                           break_id_t break_id) {
+  thread.SetThreadHitBreakpointSite();
+
   return StopInfoSP(new StopInfoBreakpoint(thread, break_id));
 }
 
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index e75f5a356cec2..2105efe9305fe 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -217,6 +217,7 @@ Thread::Thread(Process &process, lldb::tid_t tid, bool use_invalid_index_id)
       m_process_wp(process.shared_from_this()), m_stop_info_sp(),
       m_stop_info_stop_id(0), m_stop_info_override_stop_id(0),
       m_should_run_before_public_stop(false),
+      m_stopped_at_unexecuted_bp(LLDB_INVALID_ADDRESS),
       m_index_id(use_invalid_index_id ? LLDB_INVALID_INDEX32
                                       : process.GetNextThreadIndexID(tid)),
       m_reg_context_sp(), m_state(eStateUnloaded), m_state_mutex(),
@@ -519,6 +520,7 @@ bool Thread::CheckpointThreadState(ThreadStateCheckpoint &saved_state) {
   saved_state.current_inlined_depth = GetCurrentInlinedDepth();
   saved_state.m_completed_plan_checkpoint =
       GetPlans().CheckpointCompletedPlans();
+  saved_state.stopped_at_unexecuted_bp = m_stopped_at_unexecuted_bp;
 
   return true;
 }
@@ -554,6 +556,7 @@ void Thread::RestoreThreadStateFromCheckpoint(
       saved_state.current_inlined_depth);
   GetPlans().RestoreCompletedPlanCheckpoint(
       saved_state.m_completed_plan_checkpoint);
+  m_stopped_at_unexecuted_bp = saved_state.stopped_at_unexecuted_bp;
 }
 
 StateType Thread::GetState() const {
@@ -622,7 +625,15 @@ void Thread::SetupForResume() {
       const addr_t thread_pc = reg_ctx_sp->GetPC();
       BreakpointSiteSP bp_site_sp =
           GetProcess()->GetBreakpointSiteList().FindByAddress(thread_pc);
-      if (bp_site_sp) {
+      // If we're at a BreakpointSite which we have either
+      //   1. already triggered/hit, or
+      //   2. the Breakpoint was added while stopped, or the pc was moved
+      //      to this BreakpointSite
+      // Step past the breakpoint before resuming.
+      // If we stopped at a breakpoint instruction/BreakpointSite location
+      // without hitting it, and we're still at that same address on
+      // resuming, then we want to hit the BreakpointSite when we resume.
+      if (bp_site_sp && m_stopped_at_unexecuted_bp != thread_pc) {
         // Note, don't assume there's a ThreadPlanStepOverBreakpoint, the
         // target may not require anything special to step over a breakpoint.
 
@@ -711,6 +722,7 @@ bool Thread::ShouldResume(StateType resume_state) {
 
   if (need_to_resume) {
     ClearStackFrames();
+    m_stopped_at_unexecuted_bp = LLDB_INVALID_ADDRESS;
     // Let Thread subclasses do any special work they need to prior to resuming
     WillResume(resume_state);
   }
@@ -1894,6 +1906,7 @@ Unwind &Thread::GetUnwinder() {
 void Thread::Flush() {
   ClearStackFrames();
   m_reg_context_sp.reset();
+  m_stopped_at_unexecuted_bp = LLDB_INVALID_ADDRESS;
 }
 
 bool Thread::IsStillAtLastBreakpointHit() {
diff --git a/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py b/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
index 73de4a294388b..ecea28c6e1f6d 100644
--- a/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
+++ b/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
@@ -2,7 +2,6 @@
 Test that we handle breakpoints on consecutive instructions correctly.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -64,20 +63,30 @@ def test_single_step(self):
         """Test that single step stops at the second breakpoint."""
         self.prepare_test()
 
+        # Instruction step to the next instruction
+        # We haven't executed breakpoint2 yet, we're sitting at it now.
+        step_over = False
+        self.thread.StepInstruction(step_over)
+
         step_over = False
         self.thread.StepInstruction(step_over)
 
+        # We've now hit the breakpoint and this StepInstruction has
+        # been interrupted, it is still sitting on the thread plan stack.
+
         self.assertState(self.process.GetState(), lldb.eStateStopped)
         self.assertEqual(
             self.thread.GetFrameAtIndex(0).GetPCAddress().GetLoadAddress(self.target),
             self.bkpt_address.GetLoadAddress(self.target),
         )
-        self.thread = lldbutil.get_one_thread_stopped_at_breakpoint(
-            self.process, self.breakpoint2
-        )
-        self.assertIsNotNone(
-            self.thread, "Expected one thread to be stopped at breakpoint 2"
-        )
+
+        # One more instruction to complete the Step that was interrupted
+        # earlier.
+        self.thread.StepInstruction(step_over)
+        strm = lldb.SBStream()
+        self.thread.GetDescription(strm)
+        self.assertIn("instruction step into", strm.GetData())
+        self.assertIsNotNone(self.thread, "Expected to see that step-in had completed")
 
         self.finish_test()
 
@@ -106,4 +115,7 @@ def test_single_step_thread_specific(self):
             "Stop reason should be 'plan complete'",
         )
 
+        # Hit our second breakpoint
+        self.process.Continue()
+
         self.finish_test()
diff --git a/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py b/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
index 3a7440a31677a..1315288b35c55 100644
--- a/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
+++ b/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
@@ -5,7 +5,6 @@
 and eStopReasonPlanComplete when breakpoint's condition fails.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -90,6 +89,11 @@ def test_step_instruction(self):
                 self.assertGreaterEqual(step_count, steps_expected)
                 break
 
+        # We did a `stepi` when we hit our last breakpoint, and the stepi was not
+        # completed yet, so when we resume it will complete (running process.Continue()
+        # would have the same result - we step one instruction and stop again when
+        # our interrupted stepi completes).
+        self.thread.StepInstruction(True)
         # Run the process until termination
         self.process.Continue()
         self.assertState(self.process.GetState(), lldb.eStateExited)

From 6747f12931b770f2c0d48ba4b01a55c36ab35e7c Mon Sep 17 00:00:00 2001
From: Sayhaan Siddiqui <49014204+sayhaan@users.noreply.github.com>
Date: Fri, 19 Jul 2024 17:52:49 -0700
Subject: [PATCH 251/486] [BOLT][DWARF][NFC] Split processUnitDIE into two
 lambdas (#99225)

Split processUnitDIE into two lambdas to separate the processing of DWO
CUs and CUs in the main binary.
---
 bolt/lib/Rewrite/DWARFRewriter.cpp | 144 +++++++++++++++--------------
 1 file changed, 77 insertions(+), 67 deletions(-)

diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp
index 1ec216b39e95c..ccb45f40c5c7a 100644
--- a/bolt/lib/Rewrite/DWARFRewriter.cpp
+++ b/bolt/lib/Rewrite/DWARFRewriter.cpp
@@ -620,9 +620,10 @@ void DWARFRewriter::updateDebugInfo() {
   uint32_t CUIndex = 0;
   std::mutex AccessMutex;
   // Needs to be invoked in the same order as CUs are processed.
-  auto createRangeLocListAddressWriters =
-      [&](DWARFUnit &CU) -> DebugLocWriter * {
+  llvm::DenseMap<uint64_t, uint64_t> LocListWritersIndexByCU;
+  auto createRangeLocListAddressWriters = [&](DWARFUnit &CU) {
     std::lock_guard<std::mutex> Lock(AccessMutex);
+
     const uint16_t DwarfVersion = CU.getVersion();
     if (DwarfVersion >= 5) {
       auto AddrW = std::make_unique<DebugAddrWriterDwarf5>(
@@ -641,7 +642,6 @@ void DWARFRewriter::updateDebugInfo() {
         RangeListsWritersByCU[*DWOId] = std::move(DWORangeListsSectionWriter);
       }
       AddressWritersByCU[CU.getOffset()] = std::move(AddrW);
-
     } else {
       auto AddrW =
           std::make_unique<DebugAddrWriter>(&BC, CU.getAddressByteSize());
@@ -657,7 +657,7 @@ void DWARFRewriter::updateDebugInfo() {
             std::move(LegacyRangesSectionWriterByCU);
       }
     }
-    return LocListWritersByCU[CUIndex++].get();
+    LocListWritersIndexByCU[CU.getOffset()] = CUIndex++;
   };
 
   DWARF5AcceleratorTable DebugNamesTable(opts::CreateDebugNames, BC,
@@ -666,74 +666,68 @@ void DWARFRewriter::updateDebugInfo() {
   DWPState State;
   if (opts::WriteDWP)
     initDWPState(State);
-  auto processUnitDIE = [&](DWARFUnit *Unit, DIEBuilder *DIEBlder) {
-    // Check if the unit is a skeleton and we need special updates for it and
-    // its matching split/DWO CU.
-    std::optional<DWARFUnit *> SplitCU;
+  auto processSplitCU = [&](DWARFUnit &Unit, DWARFUnit &SplitCU,
+                            DIEBuilder &DIEBlder,
+                            DebugRangesSectionWriter &TempRangesSectionWriter,
+                            DebugAddrWriter &AddressWriter) {
+    DIEBuilder DWODIEBuilder(BC, &(SplitCU).getContext(), DebugNamesTable,
+                             &Unit);
+    DWODIEBuilder.buildDWOUnit(SplitCU);
+    std::string DWOName = "";
+    std::optional<std::string> DwarfOutputPath =
+        opts::DwarfOutputPath.empty()
+            ? std::nullopt
+            : std::optional<std::string>(opts::DwarfOutputPath.c_str());
+    {
+      std::lock_guard<std::mutex> Lock(AccessMutex);
+      DWOName = DIEBlder.updateDWONameCompDir(
+          *StrOffstsWriter, *StrWriter, Unit, DwarfOutputPath, std::nullopt);
+    }
+    DebugStrOffsetsWriter DWOStrOffstsWriter(BC);
+    DebugStrWriter DWOStrWriter((SplitCU).getContext(), true);
+    DWODIEBuilder.updateDWONameCompDirForTypes(
+        DWOStrOffstsWriter, DWOStrWriter, SplitCU, DwarfOutputPath, DWOName);
+    DebugLoclistWriter DebugLocDWoWriter(Unit, Unit.getVersion(), true,
+                                         AddressWriter);
+
+    updateUnitDebugInfo(SplitCU, DWODIEBuilder, DebugLocDWoWriter,
+                        TempRangesSectionWriter, AddressWriter);
+    DebugLocDWoWriter.finalize(DWODIEBuilder,
+                               *DWODIEBuilder.getUnitDIEbyUnit(SplitCU));
+    if (Unit.getVersion() >= 5)
+      TempRangesSectionWriter.finalizeSection();
+
+    emitDWOBuilder(DWOName, DWODIEBuilder, *this, SplitCU, Unit, State,
+                   DebugLocDWoWriter, DWOStrOffstsWriter, DWOStrWriter,
+                   GDBIndexSection);
+  };
+  auto processMainBinaryCU = [&](DWARFUnit &Unit, DIEBuilder &DIEBlder) {
+    DebugAddrWriter &AddressWriter =
+        *AddressWritersByCU[Unit.getOffset()].get();
+    DebugRangesSectionWriter &RangesSectionWriter =
+        Unit.getVersion() >= 5 ? *RangeListsSectionWriter.get()
+                               : *LegacyRangesSectionWriter.get();
+    DebugLocWriter &DebugLocWriter =
+        *LocListWritersByCU[LocListWritersIndexByCU[Unit.getOffset()]].get();
     std::optional<uint64_t> RangesBase;
-    std::optional<uint64_t> DWOId = Unit->getDWOId();
+    std::optional<DWARFUnit *> SplitCU;
+    std::optional<uint64_t> DWOId = Unit.getDWOId();
     if (DWOId)
       SplitCU = BC.getDWOCU(*DWOId);
-    DebugLocWriter *DebugLocWriter = createRangeLocListAddressWriters(*Unit);
-    DebugRangesSectionWriter *RangesSectionWriter =
-        Unit->getVersion() >= 5 ? RangeListsSectionWriter.get()
-                                : LegacyRangesSectionWriter.get();
-    DebugAddrWriter *AddressWriter =
-        AddressWritersByCU[Unit->getOffset()].get();
-    // Skipping CUs that failed to load.
-    if (SplitCU) {
-      DIEBuilder DWODIEBuilder(BC, &(*SplitCU)->getContext(), DebugNamesTable,
-                               Unit);
-      DWODIEBuilder.buildDWOUnit(**SplitCU);
-      std::string DWOName = "";
-      std::optional<std::string> DwarfOutputPath =
-          opts::DwarfOutputPath.empty()
-              ? std::nullopt
-              : std::optional<std::string>(opts::DwarfOutputPath.c_str());
-      {
-        std::lock_guard<std::mutex> Lock(AccessMutex);
-        DWOName = DIEBlder->updateDWONameCompDir(
-            *StrOffstsWriter, *StrWriter, *Unit, DwarfOutputPath, std::nullopt);
-      }
-      DebugStrOffsetsWriter DWOStrOffstsWriter(BC);
-      DebugStrWriter DWOStrWriter((*SplitCU)->getContext(), true);
-      DWODIEBuilder.updateDWONameCompDirForTypes(DWOStrOffstsWriter,
-                                                 DWOStrWriter, **SplitCU,
-                                                 DwarfOutputPath, DWOName);
-      DebugLoclistWriter DebugLocDWoWriter(*Unit, Unit->getVersion(), true,
-                                           *AddressWriter);
-      DebugRangesSectionWriter *TempRangesSectionWriter = RangesSectionWriter;
-      if (Unit->getVersion() >= 5) {
-        TempRangesSectionWriter = RangeListsWritersByCU[*DWOId].get();
-      } else {
-        TempRangesSectionWriter = LegacyRangesWritersByCU[*DWOId].get();
-        RangesBase = RangesSectionWriter->getSectionOffset();
-      }
-
-      updateUnitDebugInfo(*(*SplitCU), DWODIEBuilder, DebugLocDWoWriter,
-                          *TempRangesSectionWriter, *AddressWriter);
-      DebugLocDWoWriter.finalize(DWODIEBuilder,
-                                 *DWODIEBuilder.getUnitDIEbyUnit(**SplitCU));
-      if (Unit->getVersion() >= 5)
-        TempRangesSectionWriter->finalizeSection();
-
-      emitDWOBuilder(DWOName, DWODIEBuilder, *this, **SplitCU, *Unit, State,
-                     DebugLocDWoWriter, DWOStrOffstsWriter, DWOStrWriter,
-                     GDBIndexSection);
-    }
-
-    if (Unit->getVersion() >= 5) {
-      RangesBase = RangesSectionWriter->getSectionOffset() +
+    if (Unit.getVersion() >= 5) {
+      RangesBase = RangesSectionWriter.getSectionOffset() +
                    getDWARF5RngListLocListHeaderSize();
-      RangesSectionWriter->initSection(*Unit);
-      StrOffstsWriter->finalizeSection(*Unit, *DIEBlder);
+      RangesSectionWriter.initSection(Unit);
+      StrOffstsWriter->finalizeSection(Unit, DIEBlder);
+    } else if (SplitCU) {
+      RangesBase = LegacyRangesSectionWriter.get()->getSectionOffset();
     }
 
-    updateUnitDebugInfo(*Unit, *DIEBlder, *DebugLocWriter, *RangesSectionWriter,
-                        *AddressWriter, RangesBase);
-    DebugLocWriter->finalize(*DIEBlder, *DIEBlder->getUnitDIEbyUnit(*Unit));
-    if (Unit->getVersion() >= 5)
-      RangesSectionWriter->finalizeSection();
+    updateUnitDebugInfo(Unit, DIEBlder, DebugLocWriter, RangesSectionWriter,
+                        AddressWriter, RangesBase);
+    DebugLocWriter.finalize(DIEBlder, *DIEBlder.getUnitDIEbyUnit(Unit));
+    if (Unit.getVersion() >= 5)
+      RangesSectionWriter.finalizeSection();
   };
 
   DIEBuilder DIEBlder(BC, BC.DwCtx.get(), DebugNamesTable);
@@ -751,8 +745,24 @@ void DWARFRewriter::updateDebugInfo() {
   CUPartitionVector PartVec = partitionCUs(*BC.DwCtx);
   for (std::vector<DWARFUnit *> &Vec : PartVec) {
     DIEBlder.buildCompileUnits(Vec);
+    for (DWARFUnit *CU : DIEBlder.getProcessedCUs()) {
+      createRangeLocListAddressWriters(*CU);
+      std::optional<DWARFUnit *> SplitCU;
+      std::optional<uint64_t> DWOId = CU->getDWOId();
+      if (DWOId)
+        SplitCU = BC.getDWOCU(*DWOId);
+      if (!SplitCU)
+        continue;
+      DebugAddrWriter &AddressWriter =
+          *AddressWritersByCU[CU->getOffset()].get();
+      DebugRangesSectionWriter *TempRangesSectionWriter =
+          CU->getVersion() >= 5 ? RangeListsWritersByCU[*DWOId].get()
+                                : LegacyRangesWritersByCU[*DWOId].get();
+      processSplitCU(*CU, **SplitCU, DIEBlder, *TempRangesSectionWriter,
+                     AddressWriter);
+    }
     for (DWARFUnit *CU : DIEBlder.getProcessedCUs())
-      processUnitDIE(CU, &DIEBlder);
+      processMainBinaryCU(*CU, DIEBlder);
     finalizeCompileUnits(DIEBlder, *Streamer, OffsetMap,
                          DIEBlder.getProcessedCUs(), *FinalAddrWriter);
   }

From 4a739fb53cacdb850d27b164dd6f173d21d5f083 Mon Sep 17 00:00:00 2001
From: Max Winkler <max.enrico.winkler@gmail.com>
Date: Fri, 19 Jul 2024 18:04:39 -0700
Subject: [PATCH 252/486] [Clang] [docs] [MSVC] Add sections on `__forceinline`
 and intrinsic behaviour differences between Clang and MSVC (#99426)

We have had quite a few issues created around how Clang treats
intrinsics vs how MSVC treats intrinsics.

While I was writing this I also added some sections on behaviour changes
that caught me while porting my MSVC codebase to clang-cl.

Hopefully we can point issues around intrinsics to this doc and
hopefully it is useful to others who run into similar behaviour
differences.
The behaviour differences highlighted here are differences, as far as I
am aware, that we do not intend to change or fix for MSVC.
---
 clang/docs/MSVCCompatibility.rst | 130 +++++++++++++++++++++++++++++++
 1 file changed, 130 insertions(+)

diff --git a/clang/docs/MSVCCompatibility.rst b/clang/docs/MSVCCompatibility.rst
index b2486052abf9a..0b6fea597f8d3 100644
--- a/clang/docs/MSVCCompatibility.rst
+++ b/clang/docs/MSVCCompatibility.rst
@@ -154,3 +154,133 @@ a hint suggesting how to fix the problem.
 As of this writing, Clang is able to compile a simple ATL hello world
 application.  There are still issues parsing WRL headers for modern Windows 8
 apps, but they should be addressed soon.
+
+__forceinline behavior
+======================
+
+``__forceinline`` behaves like ``[[clang::always_inline]]``.
+Inlining is always attempted regardless of optimization level.
+
+This differs from MSVC where ``__forceinline`` is only respected once inline expansion is enabled
+which allows any function marked implicitly or explicitly ``inline`` or ``__forceinline`` to be expanded.
+Therefore functions marked ``__forceinline`` will be expanded when the optimization level is ``/Od`` unlike
+MSVC where ``__forceinline`` will not be expanded under ``/Od``.
+
+SIMD and instruction set intrinsic behavior
+===========================================
+
+Clang follows the GCC model for intrinsics and not the MSVC model.
+There are currently no plans to support the MSVC model.
+
+MSVC intrinsics always emit the machine instruction the intrinsic models regardless of the compile time options specified.
+For example ``__popcnt`` always emits the x86 popcnt instruction even if the compiler does not have the option enabled to emit popcnt on its own volition.
+
+There are two common cases where code that compiles with MSVC will need reworking to build on clang.
+Assume the examples are only built with `-msse2` so we do not have the intrinsics at compile time.
+
+.. code-block:: c++
+
+  unsigned PopCnt(unsigned v) {
+    if (HavePopCnt)
+      return __popcnt(v);
+    else
+      return GenericPopCnt(v);
+  }
+
+.. code-block:: c++
+
+  __m128 dot4_sse3(__m128 v0, __m128 v1) {
+    __m128 r = _mm_mul_ps(v0, v1);
+    r = _mm_hadd_ps(r, r);
+    r = _mm_hadd_ps(r, r);
+    return r;
+  }
+
+Clang expects that either you have compile time support for the target features, `-msse3` and `-mpopcnt`, you mark the function with the expected target feature or use runtime detection with an indirect call.
+
+.. code-block:: c++
+
+  __attribute__((__target__("sse3"))) __m128 dot4_sse3(__m128 v0, __m128 v1) {
+    __m128 r = _mm_mul_ps(v0, v1);
+    r = _mm_hadd_ps(r, r);
+    r = _mm_hadd_ps(r, r);
+    return r;
+  }
+
+The SSE3 dot product can be easily fixed by either building the translation unit with SSE3 support or using `__target__` to compile that specific function with SSE3 support.
+
+.. code-block:: c++
+
+  unsigned PopCnt(unsigned v) {
+    if (HavePopCnt)
+      return __popcnt(v);
+    else
+      return GenericPopCnt(v);
+  }
+
+The above ``PopCnt`` example must be changed to work with clang. If we mark the function with `__target__("popcnt")` then the compiler is free to emit popcnt at will which we do not want. While this isn't a concern in our small example it is a concern in larger functions with surrounding code around the intrinsics. Similar reasoning for compiling the translation unit with `-mpopcnt`.
+We must split each branch into its own function that can be called indirectly instead of using the intrinsic directly.
+
+.. code-block:: c++
+
+  __attribute__((__target__("popcnt"))) unsigned hwPopCnt(unsigned v) { return __popcnt(v); }
+  unsigned (*PopCnt)(unsigned) = HavePopCnt ? hwPopCnt : GenericPopCnt;
+
+.. code-block:: c++
+
+  __attribute__((__target__("popcnt"))) unsigned hwPopCnt(unsigned v) { return __popcnt(v); }
+  unsigned PopCnt(unsigned v) {
+    if (HavePopCnt)
+      return hwPopCnt(v);
+    else
+      return GenericPopCnt(v);
+  }
+
+In the above example ``hwPopCnt`` will not be inlined into ``PopCnt`` since ``PopCnt`` doesn't have the popcnt target feature.
+With a larger function that does real work the function call overhead is negligible. However in our popcnt example there is the function call
+overhead. There is no analog for this specific MSVC behavior in clang.
+
+For clang we effectively have to create the dispatch function ourselves to each specfic implementation.
+
+SIMD vector types
+=================
+
+Clang's simd vector types are builtin types and not user defined types as in MSVC. This does have some observable behavior changes.
+We will look at the x86 `__m128` type for the examples below but the statements apply to all vector types including ARM's `float32x4_t`.
+
+There are no members that can be accessed on the vector types. Vector types are not structs in clang.
+You cannot use ``__m128.m128_f32[0]`` to access the first element of the `__m128`.
+This also means struct initialization like ``__m128{ { 0.0f, 0.0f, 0.0f, 0.0f } }`` will not compile with clang.
+
+Since vector types are builtin types, clang implements operators on them natively.
+
+.. code-block:: c++
+
+  #ifdef _MSC_VER
+  __m128 operator+(__m128 a, __m128 b) { return _mm_add_ps(a, b); }
+  #endif
+
+The above code will fail to compile since overloaded 'operator+' must have at least one parameter of class or enumeration type.
+You will need to fix such code to have the check ``#if defined(_MSC_VER) && !defined(__clang__)``.
+
+Since `__m128` is not a class type in clang any overloads after a template definition will not be considered.
+
+.. code-block:: c++
+
+  template<class T>
+  void foo(T) {}
+
+  template<class T>
+  void bar(T t) {
+    foo(t);
+  }
+
+  void foo(__m128) {}
+
+  int main() {
+    bar(_mm_setzero_ps());
+  }
+
+With MSVC ``foo(__m128)`` will be selected but with clang ``foo<__m128>()`` will be selected since on clang `__m128` is a builtin type.
+
+In general the takeaway is `__m128` is a builtin type on clang while a class type on MSVC.

From a3ebb669d1027700a2f12bd5e075ad3fde3d4f70 Mon Sep 17 00:00:00 2001
From: aaryanshukla <53713108+aaryanshukla@users.noreply.github.com>
Date: Fri, 19 Jul 2024 18:43:27 -0700
Subject: [PATCH 253/486] [libc][math]: updated math docs for newhdrgen
 (#99715)

---
 libc/src/math/docs/add_math_function.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/src/math/docs/add_math_function.md b/libc/src/math/docs/add_math_function.md
index 9c23b8ca789bc..e9a6aadc6c44f 100644
--- a/libc/src/math/docs/add_math_function.md
+++ b/libc/src/math/docs/add_math_function.md
@@ -18,7 +18,7 @@ together with its specifications:
 ```
 - Add function specs to the file:
 ```
-  libc/spec/stdc.td
+  libc/newhdrgen/yaml/math.yaml
 ```
 
 ## Implementation

From 52c08d7ffd380f4abd819c20bec76252272f6337 Mon Sep 17 00:00:00 2001
From: Jason Molenda <jmolenda@apple.com>
Date: Fri, 19 Jul 2024 18:42:17 -0700
Subject: [PATCH 254/486] Revert "[lldb] Change lldb's breakpoint handling
 behavior (#96260)"

This reverts commit 05f0e86cc895181b3d2210458c78938f83353002.

The debuginfo dexter tests are failing, probably because the way
stepping over breakpoints has changed with my patches.  And there
are two API tests fails on the ubuntu-arm (32-bit) bot. I'll need
to investigate both of these, neither has an obvious failure reason.
---
 lldb/include/lldb/Target/Thread.h             |  24 --
 .../Process/Utility/StopInfoMachException.cpp | 322 ++++++++++--------
 .../Process/Windows/Common/ProcessWindows.cpp |  32 +-
 .../Process/gdb-remote/ProcessGDBRemote.cpp   |  93 +++--
 .../Process/scripted/ScriptedThread.cpp       |  11 -
 lldb/source/Target/StopInfo.cpp               |   2 -
 lldb/source/Target/Thread.cpp                 |  15 +-
 .../TestConsecutiveBreakpoints.py             |  26 +-
 .../TestStepOverBreakpoint.py                 |   6 +-
 9 files changed, 265 insertions(+), 266 deletions(-)

diff --git a/lldb/include/lldb/Target/Thread.h b/lldb/include/lldb/Target/Thread.h
index d3e429e9256df..2ff1f50d497e2 100644
--- a/lldb/include/lldb/Target/Thread.h
+++ b/lldb/include/lldb/Target/Thread.h
@@ -129,7 +129,6 @@ class Thread : public std::enable_shared_from_this<Thread>,
         register_backup_sp; // You need to restore the registers, of course...
     uint32_t current_inlined_depth;
     lldb::addr_t current_inlined_pc;
-    lldb::addr_t stopped_at_unexecuted_bp;
   };
 
   /// Constructor
@@ -379,26 +378,6 @@ class Thread : public std::enable_shared_from_this<Thread>,
 
   virtual void SetQueueLibdispatchQueueAddress(lldb::addr_t dispatch_queue_t) {}
 
-  /// When a thread stops at an enabled BreakpointSite that has not executed,
-  /// the Process plugin should call SetThreadStoppedAtUnexecutedBP(pc).
-  /// If that BreakpointSite was actually triggered (the instruction was
-  /// executed, for a software breakpoint), regardless of whether the
-  /// breakpoint is valid for this thread, SetThreadHitBreakpointSite()
-  /// should be called to record that fact.
-  ///
-  /// Depending on the structure of the Process plugin, it may be easiest
-  /// to call SetThreadStoppedAtUnexecutedBP(pc) unconditionally when at
-  /// a BreakpointSite, and later when it is known that it was triggered,
-  /// SetThreadHitBreakpointSite() can be called.  These two methods
-  /// overwrite the same piece of state in the Thread, the last one
-  /// called on a Thread wins.
-  void SetThreadStoppedAtUnexecutedBP(lldb::addr_t pc) {
-    m_stopped_at_unexecuted_bp = pc;
-  }
-  void SetThreadHitBreakpointSite() {
-    m_stopped_at_unexecuted_bp = LLDB_INVALID_ADDRESS;
-  }
-
   /// Whether this Thread already has all the Queue information cached or not
   ///
   /// A Thread may be associated with a libdispatch work Queue at a given
@@ -1333,9 +1312,6 @@ class Thread : public std::enable_shared_from_this<Thread>,
   bool m_should_run_before_public_stop;  // If this thread has "stop others" 
                                          // private work to do, then it will
                                          // set this.
-  lldb::addr_t m_stopped_at_unexecuted_bp; // Set to the address of a breakpoint
-                                           // instruction that we have not yet
-                                           // hit, but will hit when we resume.
   const uint32_t m_index_id; ///< A unique 1 based index assigned to each thread
                              /// for easy UI/command line access.
   lldb::RegisterContextSP m_reg_context_sp; ///< The register context for this
diff --git a/lldb/source/Plugins/Process/Utility/StopInfoMachException.cpp b/lldb/source/Plugins/Process/Utility/StopInfoMachException.cpp
index f1853be12638e..25cee369d7ee3 100644
--- a/lldb/source/Plugins/Process/Utility/StopInfoMachException.cpp
+++ b/lldb/source/Plugins/Process/Utility/StopInfoMachException.cpp
@@ -488,6 +488,38 @@ const char *StopInfoMachException::GetDescription() {
   return m_description.c_str();
 }
 
+static StopInfoSP GetStopInfoForHardwareBP(Thread &thread, Target *target,
+                                           uint32_t exc_data_count,
+                                           uint64_t exc_sub_code,
+                                           uint64_t exc_sub_sub_code) {
+  // Try hardware watchpoint.
+  if (target) {
+    // The exc_sub_code indicates the data break address.
+    WatchpointResourceSP wp_rsrc_sp =
+        target->GetProcessSP()->GetWatchpointResourceList().FindByAddress(
+            (addr_t)exc_sub_code);
+    if (wp_rsrc_sp && wp_rsrc_sp->GetNumberOfConstituents() > 0) {
+      return StopInfo::CreateStopReasonWithWatchpointID(
+          thread, wp_rsrc_sp->GetConstituentAtIndex(0)->GetID());
+    }
+  }
+
+  // Try hardware breakpoint.
+  ProcessSP process_sp(thread.GetProcess());
+  if (process_sp) {
+    // The exc_sub_code indicates the data break address.
+    lldb::BreakpointSiteSP bp_sp =
+        process_sp->GetBreakpointSiteList().FindByAddress(
+            (lldb::addr_t)exc_sub_code);
+    if (bp_sp && bp_sp->IsEnabled()) {
+      return StopInfo::CreateStopReasonWithBreakpointSiteID(thread,
+                                                            bp_sp->GetID());
+    }
+  }
+
+  return nullptr;
+}
+
 #if defined(__APPLE__)
 const char *
 StopInfoMachException::MachException::Name(exception_type_t exc_type) {
@@ -575,25 +607,6 @@ StopInfoSP StopInfoMachException::CreateStopReasonWithMachException(
       target ? target->GetArchitecture().GetMachine()
              : llvm::Triple::UnknownArch;
 
-  ProcessSP process_sp(thread.GetProcess());
-  RegisterContextSP reg_ctx_sp(thread.GetRegisterContext());
-  // Caveat: with x86 KDP if we've hit a breakpoint, the pc we
-  // receive is past the breakpoint instruction.
-  // If we have a breakpoints at 0x100 and 0x101, we hit the
-  // 0x100 breakpoint and the pc is reported at 0x101.
-  // We will initially mark this thread as being stopped at an
-  // unexecuted breakpoint at 0x101. Later when we see that
-  // we stopped for a Breakpoint reason, we will decrement the
-  // pc, and update the thread to record that we hit the
-  // breakpoint at 0x100.
-  // The fact that the pc may be off by one at this point
-  // (for an x86 KDP breakpoint hit) is not a problem.
-  addr_t pc = reg_ctx_sp->GetPC();
-  BreakpointSiteSP bp_site_sp =
-      process_sp->GetBreakpointSiteList().FindByAddress(pc);
-  if (bp_site_sp && bp_site_sp->IsEnabled())
-    thread.SetThreadStoppedAtUnexecutedBP(pc);
-
   switch (exc_type) {
   case 1: // EXC_BAD_ACCESS
   case 2: // EXC_BAD_INSTRUCTION
@@ -620,158 +633,171 @@ StopInfoSP StopInfoMachException::CreateStopReasonWithMachException(
     }
     break;
 
-    // A mach exception comes with 2-4 pieces of data.
-    // The sub-codes are only provided for certain types
-    // of mach exceptions.
-    // [exc_type, exc_code, exc_sub_code, exc_sub_sub_code]
-    //
-    // Here are all of the EXC_BREAKPOINT, exc_type==6,
-    // exceptions we can receive.
-    //
-    // Instruction step:
-    //   [6, 1, 0]
-    //   Intel KDP [6, 3, ??]
-    //   armv7 [6, 0x102, <stop-pc>]  Same as software breakpoint!
-    //
-    // Software breakpoint:
-    //   x86 [6, 2, 0]
-    //   Intel KDP [6, 2, <bp-addr + 1>]
-    //   arm64 [6, 1, <bp-addr>]
-    //   armv7 [6, 0x102, <bp-addr>]  Same as instruction step!
-    //
-    // Hardware breakpoint:
-    //   x86 [6, 1, <bp-addr>, 0]
-    //   x86/Rosetta not implemented, see software breakpoint
-    //   arm64 [6, 1, <bp-addr>]
-    //   armv7 not implemented, see software breakpoint
-    //
-    // Hardware watchpoint:
-    //   x86 [6, 1, <accessed-addr>, 0] (both Intel hw and Rosetta)
-    //   arm64 [6, 0x102, <accessed-addr>, 0]
-    //   armv7 [6, 0x102, <accessed-addr>, 0]
-    //
-    // arm64 BRK instruction (imm arg not reflected in the ME)
-    //   [ 6, 1, <addr-of-BRK-insn>]
-    //
-    // In order of codes mach exceptions:
-    //   [6, 1, 0] - instruction step
-    //   [6, 1, <bp-addr>] - hardware breakpoint or watchpoint
-    //
-    //   [6, 2, 0] - software breakpoint
-    //   [6, 2, <bp-addr + 1>] - software breakpoint
-    //
-    //   [6, 3] - instruction step
-    //
-    //   [6, 0x102, <stop-pc>] armv7 instruction step
-    //   [6, 0x102, <bp-addr>] armv7 software breakpoint
-    //   [6, 0x102, <accessed-addr>, 0] arm64/armv7 watchpoint
-
   case 6: // EXC_BREAKPOINT
   {
-    bool stopped_by_hitting_breakpoint = false;
-    bool stopped_by_completing_stepi = false;
-    bool stopped_watchpoint = false;
-    std::optional<addr_t> address;
-
-    // exc_code 1
-    if (exc_code == 1) {
-      if (exc_sub_code == 0) {
-        stopped_by_completing_stepi = true;
-      } else {
-        // Ambiguous: could be signalling a
-        // breakpoint or watchpoint hit.
-        stopped_by_hitting_breakpoint = true;
-        stopped_watchpoint = true;
-        address = exc_sub_code;
-      }
-    }
-
-    // exc_code 2
-    if (exc_code == 2) {
-      if (exc_sub_code == 0)
-        stopped_by_hitting_breakpoint = true;
-      else {
-        stopped_by_hitting_breakpoint = true;
-        // Intel KDP software breakpoint
+    bool is_actual_breakpoint = false;
+    bool is_trace_if_actual_breakpoint_missing = false;
+    switch (cpu) {
+    case llvm::Triple::x86:
+    case llvm::Triple::x86_64:
+      if (exc_code == 1) // EXC_I386_SGL
+      {
+        if (!exc_sub_code) {
+          // This looks like a plain trap.
+          // Have to check if there is a breakpoint here as well.  When you
+          // single-step onto a trap, the single step stops you not to trap.
+          // Since we also do that check below, let's just use that logic.
+          is_actual_breakpoint = true;
+          is_trace_if_actual_breakpoint_missing = true;
+        } else {
+          if (StopInfoSP stop_info =
+                  GetStopInfoForHardwareBP(thread, target, exc_data_count,
+                                           exc_sub_code, exc_sub_sub_code))
+            return stop_info;
+        }
+      } else if (exc_code == 2 || // EXC_I386_BPT
+                 exc_code == 3)   // EXC_I386_BPTFLT
+      {
+        // KDP returns EXC_I386_BPTFLT for trace breakpoints
+        if (exc_code == 3)
+          is_trace_if_actual_breakpoint_missing = true;
+
+        is_actual_breakpoint = true;
         if (!pc_already_adjusted)
           pc_decrement = 1;
       }
-    }
+      break;
 
-    // exc_code 3
-    if (exc_code == 3)
-      stopped_by_completing_stepi = true;
+    case llvm::Triple::arm:
+    case llvm::Triple::thumb:
+      if (exc_code == 0x102) // EXC_ARM_DA_DEBUG
+      {
+        // LWP_TODO: We need to find the WatchpointResource that matches
+        // the address, and evaluate its Watchpoints.
+
+        // It's a watchpoint, then, if the exc_sub_code indicates a
+        // known/enabled data break address from our watchpoint list.
+        lldb::WatchpointSP wp_sp;
+        if (target)
+          wp_sp = target->GetWatchpointList().FindByAddress(
+              (lldb::addr_t)exc_sub_code);
+        if (wp_sp && wp_sp->IsEnabled()) {
+          return StopInfo::CreateStopReasonWithWatchpointID(thread,
+                                                            wp_sp->GetID());
+        } else {
+          is_actual_breakpoint = true;
+          is_trace_if_actual_breakpoint_missing = true;
+        }
+      } else if (exc_code == 1) // EXC_ARM_BREAKPOINT
+      {
+        is_actual_breakpoint = true;
+        is_trace_if_actual_breakpoint_missing = true;
+      } else if (exc_code == 0) // FIXME not EXC_ARM_BREAKPOINT but a kernel
+                                // is currently returning this so accept it
+                                // as indicating a breakpoint until the
+                                // kernel is fixed
+      {
+        is_actual_breakpoint = true;
+        is_trace_if_actual_breakpoint_missing = true;
+      }
+      break;
 
-    // exc_code 0x102
-    if (exc_code == 0x102 && exc_sub_code != 0) {
-      if (cpu == llvm::Triple::arm || cpu == llvm::Triple::thumb) {
-        stopped_by_hitting_breakpoint = true;
-        stopped_by_completing_stepi = true;
+    case llvm::Triple::aarch64_32:
+    case llvm::Triple::aarch64: {
+      // xnu describes three things with type EXC_BREAKPOINT:
+      //
+      //   exc_code 0x102 [EXC_ARM_DA_DEBUG], exc_sub_code addr-of-insn
+      //      Watchpoint access.  exc_sub_code is the address of the
+      //      instruction which trigged the watchpoint trap.
+      //      debugserver may add the watchpoint number that was triggered
+      //      in exc_sub_sub_code.
+      //
+      //   exc_code 1 [EXC_ARM_BREAKPOINT], exc_sub_code 0
+      //      Instruction step has completed.
+      //
+      //   exc_code 1 [EXC_ARM_BREAKPOINT], exc_sub_code address-of-instruction
+      //      Software breakpoint instruction executed.
+
+      if (exc_code == 1 && exc_sub_code == 0) // EXC_ARM_BREAKPOINT
+      {
+        // This is hit when we single instruction step aka MDSCR_EL1 SS bit 0
+        // is set
+        is_actual_breakpoint = true;
+        is_trace_if_actual_breakpoint_missing = true;
+        if (thread.GetTemporaryResumeState() != eStateStepping)
+          not_stepping_but_got_singlestep_exception = true;
+      }
+      if (exc_code == 0x102) // EXC_ARM_DA_DEBUG
+      {
+        // LWP_TODO: We need to find the WatchpointResource that matches
+        // the address, and evaluate its Watchpoints.
+
+        // It's a watchpoint, then, if the exc_sub_code indicates a
+        // known/enabled data break address from our watchpoint list.
+        lldb::WatchpointSP wp_sp;
+        if (target)
+          wp_sp = target->GetWatchpointList().FindByAddress(
+              (lldb::addr_t)exc_sub_code);
+        if (wp_sp && wp_sp->IsEnabled()) {
+          return StopInfo::CreateStopReasonWithWatchpointID(thread,
+                                                            wp_sp->GetID());
+        }
+        // EXC_ARM_DA_DEBUG seems to be reused for EXC_BREAKPOINT as well as
+        // EXC_BAD_ACCESS
+        if (thread.GetTemporaryResumeState() == eStateStepping)
+          return StopInfo::CreateStopReasonToTrace(thread);
       }
-      stopped_watchpoint = true;
-      address = exc_sub_code;
+      // It looks like exc_sub_code has the 4 bytes of the instruction that
+      // triggered the exception, i.e. our breakpoint opcode
+      is_actual_breakpoint = exc_code == 1;
+      break;
     }
 
-    // The Mach Exception may have been ambiguous --
-    // e.g. we stopped either because of a breakpoint
-    // or a watchpoint.  We'll disambiguate which it
-    // really was.
+    default:
+      break;
+    }
 
-    if (stopped_by_hitting_breakpoint) {
+    if (is_actual_breakpoint) {
+      RegisterContextSP reg_ctx_sp(thread.GetRegisterContext());
       addr_t pc = reg_ctx_sp->GetPC() - pc_decrement;
 
-      if (address)
-        bp_site_sp =
-            process_sp->GetBreakpointSiteList().FindByAddress(*address);
-      if (!bp_site_sp && reg_ctx_sp) {
+      ProcessSP process_sp(thread.CalculateProcess());
+
+      lldb::BreakpointSiteSP bp_site_sp;
+      if (process_sp)
         bp_site_sp = process_sp->GetBreakpointSiteList().FindByAddress(pc);
-      }
       if (bp_site_sp && bp_site_sp->IsEnabled()) {
-        // We've hit this breakpoint, whether it was intended for this thread
-        // or not.  Clear this in the Tread object so we step past it on resume.
-        thread.SetThreadHitBreakpointSite();
-
-        // If we have an operating system plug-in, we might have set a thread
-        // specific breakpoint using the operating system thread ID, so we
-        // can't make any assumptions about the thread ID so we must always
-        // report the breakpoint regardless of the thread.
+        // Update the PC if we were asked to do so, but only do so if we find
+        // a breakpoint that we know about cause this could be a trap
+        // instruction in the code
+        if (pc_decrement > 0 && adjust_pc_if_needed)
+          reg_ctx_sp->SetPC(pc);
+
+        // If the breakpoint is for this thread, then we'll report the hit,
+        // but if it is for another thread, we can just report no reason.  We
+        // don't need to worry about stepping over the breakpoint here, that
+        // will be taken care of when the thread resumes and notices that
+        // there's a breakpoint under the pc. If we have an operating system
+        // plug-in, we might have set a thread specific breakpoint using the
+        // operating system thread ID, so we can't make any assumptions about
+        // the thread ID so we must always report the breakpoint regardless
+        // of the thread.
         if (bp_site_sp->ValidForThisThread(thread) ||
-            thread.GetProcess()->GetOperatingSystem() != nullptr) {
-          // Update the PC if we were asked to do so, but only do so if we find
-          // a breakpoint that we know about cause this could be a trap
-          // instruction in the code
-          if (pc_decrement > 0 && adjust_pc_if_needed && reg_ctx_sp)
-            reg_ctx_sp->SetPC(pc);
+            thread.GetProcess()->GetOperatingSystem() != nullptr)
           return StopInfo::CreateStopReasonWithBreakpointSiteID(
               thread, bp_site_sp->GetID());
-        } else {
+        else if (is_trace_if_actual_breakpoint_missing)
+          return StopInfo::CreateStopReasonToTrace(thread);
+        else
           return StopInfoSP();
-        }
       }
-    }
-
-    // Breakpoint-hit events are handled.
-    // Now handle watchpoints.
 
-    if (stopped_watchpoint && address) {
-      WatchpointResourceSP wp_rsrc_sp =
-          target->GetProcessSP()->GetWatchpointResourceList().FindByAddress(
-              *address);
-      if (wp_rsrc_sp && wp_rsrc_sp->GetNumberOfConstituents() > 0) {
-        return StopInfo::CreateStopReasonWithWatchpointID(
-            thread, wp_rsrc_sp->GetConstituentAtIndex(0)->GetID());
-      }
-    }
-
-    // Finally, handle instruction step.
-
-    if (stopped_by_completing_stepi) {
-      if (thread.GetTemporaryResumeState() != eStateStepping)
-        not_stepping_but_got_singlestep_exception = true;
-      else
+      // Don't call this a trace if we weren't single stepping this thread.
+      if (is_trace_if_actual_breakpoint_missing &&
+          thread.GetTemporaryResumeState() == eStateStepping) {
         return StopInfo::CreateStopReasonToTrace(thread);
+      }
     }
-
   } break;
 
   case 7:  // EXC_SYSCALL
diff --git a/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
index 8b12b87eacbc6..f383b3d40a4f3 100644
--- a/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
@@ -377,17 +377,24 @@ void ProcessWindows::RefreshStateAfterStop() {
   if (!stop_thread)
     return;
 
-  RegisterContextSP register_context = stop_thread->GetRegisterContext();
-  uint64_t pc = register_context->GetPC();
-
-  // If we're at a BreakpointSite, mark this as an Unexecuted Breakpoint.
-  // We'll clear that state if we've actually executed the breakpoint.
-  BreakpointSiteSP site(GetBreakpointSiteList().FindByAddress(pc));
-  if (site && site->IsEnabled())
-    stop_thread->SetThreadStoppedAtUnexecutedBP(pc);
-
   switch (active_exception->GetExceptionCode()) {
   case EXCEPTION_SINGLE_STEP: {
+    RegisterContextSP register_context = stop_thread->GetRegisterContext();
+    const uint64_t pc = register_context->GetPC();
+    BreakpointSiteSP site(GetBreakpointSiteList().FindByAddress(pc));
+    if (site && site->ValidForThisThread(*stop_thread)) {
+      LLDB_LOG(log,
+               "Single-stepped onto a breakpoint in process {0} at "
+               "address {1:x} with breakpoint site {2}",
+               m_session_data->m_debugger->GetProcess().GetProcessId(), pc,
+               site->GetID());
+      stop_info = StopInfo::CreateStopReasonWithBreakpointSiteID(*stop_thread,
+                                                                 site->GetID());
+      stop_thread->SetStopInfo(stop_info);
+
+      return;
+    }
+
     auto *reg_ctx = static_cast<RegisterContextWindows *>(
         stop_thread->GetRegisterContext().get());
     uint32_t slot_id = reg_ctx->GetTriggeredHardwareBreakpointSlotId();
@@ -412,6 +419,8 @@ void ProcessWindows::RefreshStateAfterStop() {
   }
 
   case EXCEPTION_BREAKPOINT: {
+    RegisterContextSP register_context = stop_thread->GetRegisterContext();
+
     int breakpoint_size = 1;
     switch (GetTarget().GetArchitecture().GetMachine()) {
     case llvm::Triple::aarch64:
@@ -434,9 +443,9 @@ void ProcessWindows::RefreshStateAfterStop() {
     }
 
     // The current PC is AFTER the BP opcode, on all architectures.
-    pc = register_context->GetPC() - breakpoint_size;
+    uint64_t pc = register_context->GetPC() - breakpoint_size;
 
-    site = GetBreakpointSiteList().FindByAddress(pc);
+    BreakpointSiteSP site(GetBreakpointSiteList().FindByAddress(pc));
     if (site) {
       LLDB_LOG(log,
                "detected breakpoint in process {0} at address {1:x} with "
@@ -444,7 +453,6 @@ void ProcessWindows::RefreshStateAfterStop() {
                m_session_data->m_debugger->GetProcess().GetProcessId(), pc,
                site->GetID());
 
-      stop_thread->SetThreadHitBreakpointSite();
       if (site->ValidForThisThread(*stop_thread)) {
         LLDB_LOG(log,
                  "Breakpoint site {0} is valid for this thread ({1:x}), "
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 9ab03c1fb8ff3..604c92369e9a2 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -1598,8 +1598,29 @@ bool ProcessGDBRemote::CalculateThreadStopInfo(ThreadGDBRemote *thread) {
     // that have stop reasons, and if there is no entry for a thread, then it
     // has no stop reason.
     thread->GetRegisterContext()->InvalidateIfNeeded(true);
-    if (!GetThreadStopInfoFromJSON(thread, m_jstopinfo_sp))
+    if (!GetThreadStopInfoFromJSON(thread, m_jstopinfo_sp)) {
+      // If a thread is stopped at a breakpoint site, set that as the stop
+      // reason even if it hasn't executed the breakpoint instruction yet.
+      // We will silently step over the breakpoint when we resume execution
+      // and miss the fact that this thread hit the breakpoint.
+      const size_t num_thread_ids = m_thread_ids.size();
+      for (size_t i = 0; i < num_thread_ids; i++) {
+        if (m_thread_ids[i] == thread->GetID() && m_thread_pcs.size() > i) {
+          addr_t pc = m_thread_pcs[i];
+          lldb::BreakpointSiteSP bp_site_sp =
+              thread->GetProcess()->GetBreakpointSiteList().FindByAddress(pc);
+          if (bp_site_sp) {
+            if (bp_site_sp->ValidForThisThread(*thread)) {
+              thread->SetStopInfo(
+                  StopInfo::CreateStopReasonWithBreakpointSiteID(
+                      *thread, bp_site_sp->GetID()));
+              return true;
+            }
+          }
+        }
+      }
       thread->SetStopInfo(StopInfoSP());
+    }
     return true;
   }
 
@@ -1708,12 +1729,6 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo(
     if (ThreadSP memory_thread_sp = m_thread_list.GetBackingThread(thread_sp))
       thread_sp = memory_thread_sp;
 
-    addr_t pc = thread_sp->GetRegisterContext()->GetPC();
-    BreakpointSiteSP bp_site_sp =
-        thread_sp->GetProcess()->GetBreakpointSiteList().FindByAddress(pc);
-    if (bp_site_sp && bp_site_sp->IsEnabled())
-      thread_sp->SetThreadStoppedAtUnexecutedBP(pc);
-
     if (exc_type != 0) {
       const size_t exc_data_size = exc_data.size();
 
@@ -1730,10 +1745,26 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo(
       // to no reason.
       if (!reason.empty() && reason != "none") {
         if (reason == "trace") {
-          thread_sp->SetStopInfo(StopInfo::CreateStopReasonToTrace(*thread_sp));
+          addr_t pc = thread_sp->GetRegisterContext()->GetPC();
+          lldb::BreakpointSiteSP bp_site_sp =
+              thread_sp->GetProcess()->GetBreakpointSiteList().FindByAddress(
+                  pc);
+
+          // If the current pc is a breakpoint site then the StopInfo should be
+          // set to Breakpoint Otherwise, it will be set to Trace.
+          if (bp_site_sp && bp_site_sp->ValidForThisThread(*thread_sp)) {
+            thread_sp->SetStopInfo(
+                StopInfo::CreateStopReasonWithBreakpointSiteID(
+                    *thread_sp, bp_site_sp->GetID()));
+          } else
+            thread_sp->SetStopInfo(
+                StopInfo::CreateStopReasonToTrace(*thread_sp));
           handled = true;
         } else if (reason == "breakpoint") {
-          thread_sp->SetThreadHitBreakpointSite();
+          addr_t pc = thread_sp->GetRegisterContext()->GetPC();
+          lldb::BreakpointSiteSP bp_site_sp =
+              thread_sp->GetProcess()->GetBreakpointSiteList().FindByAddress(
+                  pc);
           if (bp_site_sp) {
             // If the breakpoint is for this thread, then we'll report the hit,
             // but if it is for another thread, we can just report no reason.
@@ -1849,41 +1880,39 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo(
               StopInfo::CreateStopReasonVForkDone(*thread_sp));
           handled = true;
         }
+      } else if (!signo) {
+        addr_t pc = thread_sp->GetRegisterContext()->GetPC();
+        lldb::BreakpointSiteSP bp_site_sp =
+            thread_sp->GetProcess()->GetBreakpointSiteList().FindByAddress(pc);
+
+        // If a thread is stopped at a breakpoint site, set that as the stop
+        // reason even if it hasn't executed the breakpoint instruction yet.
+        // We will silently step over the breakpoint when we resume execution
+        // and miss the fact that this thread hit the breakpoint.
+        if (bp_site_sp && bp_site_sp->ValidForThisThread(*thread_sp)) {
+          thread_sp->SetStopInfo(StopInfo::CreateStopReasonWithBreakpointSiteID(
+              *thread_sp, bp_site_sp->GetID()));
+          handled = true;
+        }
       }
 
       if (!handled && signo && !did_exec) {
         if (signo == SIGTRAP) {
           // Currently we are going to assume SIGTRAP means we are either
           // hitting a breakpoint or hardware single stepping.
-
-          // We can't disambiguate between stepping-to-a-breakpointsite and
-          // hitting-a-breakpointsite.
-          //
-          // A user can instruction-step, and be stopped at a BreakpointSite.
-          // Or a user can be sitting at a BreakpointSite,
-          // instruction-step which hits the breakpoint and the pc does not
-          // advance.
-          //
-          // In both cases, we're at a BreakpointSite when stopped, and
-          // the resume state was eStateStepping.
-
-          // Assume if we're at a BreakpointSite, we hit it.
           handled = true;
           addr_t pc =
               thread_sp->GetRegisterContext()->GetPC() + m_breakpoint_pc_offset;
-          BreakpointSiteSP bp_site_sp =
+          lldb::BreakpointSiteSP bp_site_sp =
               thread_sp->GetProcess()->GetBreakpointSiteList().FindByAddress(
                   pc);
 
-          // We can't know if we hit it or not. So if we are stopped at
-          // a BreakpointSite, assume we hit it, and should step past the
-          // breakpoint when we resume. This is contrary to how we handle
-          // BreakpointSites in any other location, but we can't know for
-          // sure what happened so it's a reasonable default.
           if (bp_site_sp) {
-            if (bp_site_sp->IsEnabled())
-              thread_sp->SetThreadHitBreakpointSite();
-
+            // If the breakpoint is for this thread, then we'll report the hit,
+            // but if it is for another thread, we can just report no reason.
+            // We don't need to worry about stepping over the breakpoint here,
+            // that will be taken care of when the thread resumes and notices
+            // that there's a breakpoint under the pc.
             if (bp_site_sp->ValidForThisThread(*thread_sp)) {
               if (m_breakpoint_pc_offset != 0)
                 thread_sp->GetRegisterContext()->SetPC(pc);
@@ -1897,6 +1926,8 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo(
           } else {
             // If we were stepping then assume the stop was the result of the
             // trace.  If we were not stepping then report the SIGTRAP.
+            // FIXME: We are still missing the case where we single step over a
+            // trap instruction.
             if (thread_sp->GetTemporaryResumeState() == eStateStepping)
               thread_sp->SetStopInfo(
                   StopInfo::CreateStopReasonToTrace(*thread_sp));
diff --git a/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp b/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
index d0d1508e85172..88a4ca3b0389f 100644
--- a/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
+++ b/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
@@ -229,17 +229,6 @@ bool ScriptedThread::CalculateStopInfo() {
         LLVM_PRETTY_FUNCTION, "Failed to get scripted thread stop info.", error,
         LLDBLog::Thread);
 
-  // If we're at a BreakpointSite, mark that we stopped there and
-  // need to hit the breakpoint when we resume.  This will be cleared
-  // if we CreateStopReasonWithBreakpointSiteID.
-  if (RegisterContextSP reg_ctx_sp = GetRegisterContext()) {
-    addr_t pc = reg_ctx_sp->GetPC();
-    if (BreakpointSiteSP bp_site_sp =
-            GetProcess()->GetBreakpointSiteList().FindByAddress(pc))
-      if (bp_site_sp->IsEnabled())
-        SetThreadStoppedAtUnexecutedBP(pc);
-  }
-
   lldb::StopInfoSP stop_info_sp;
   lldb::StopReason stop_reason_type;
 
diff --git a/lldb/source/Target/StopInfo.cpp b/lldb/source/Target/StopInfo.cpp
index 895306b5ef0d4..95f78056b1644 100644
--- a/lldb/source/Target/StopInfo.cpp
+++ b/lldb/source/Target/StopInfo.cpp
@@ -1365,8 +1365,6 @@ class StopInfoVForkDone : public StopInfo {
 
 StopInfoSP StopInfo::CreateStopReasonWithBreakpointSiteID(Thread &thread,
                                                           break_id_t break_id) {
-  thread.SetThreadHitBreakpointSite();
-
   return StopInfoSP(new StopInfoBreakpoint(thread, break_id));
 }
 
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 2105efe9305fe..e75f5a356cec2 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -217,7 +217,6 @@ Thread::Thread(Process &process, lldb::tid_t tid, bool use_invalid_index_id)
       m_process_wp(process.shared_from_this()), m_stop_info_sp(),
       m_stop_info_stop_id(0), m_stop_info_override_stop_id(0),
       m_should_run_before_public_stop(false),
-      m_stopped_at_unexecuted_bp(LLDB_INVALID_ADDRESS),
       m_index_id(use_invalid_index_id ? LLDB_INVALID_INDEX32
                                       : process.GetNextThreadIndexID(tid)),
       m_reg_context_sp(), m_state(eStateUnloaded), m_state_mutex(),
@@ -520,7 +519,6 @@ bool Thread::CheckpointThreadState(ThreadStateCheckpoint &saved_state) {
   saved_state.current_inlined_depth = GetCurrentInlinedDepth();
   saved_state.m_completed_plan_checkpoint =
       GetPlans().CheckpointCompletedPlans();
-  saved_state.stopped_at_unexecuted_bp = m_stopped_at_unexecuted_bp;
 
   return true;
 }
@@ -556,7 +554,6 @@ void Thread::RestoreThreadStateFromCheckpoint(
       saved_state.current_inlined_depth);
   GetPlans().RestoreCompletedPlanCheckpoint(
       saved_state.m_completed_plan_checkpoint);
-  m_stopped_at_unexecuted_bp = saved_state.stopped_at_unexecuted_bp;
 }
 
 StateType Thread::GetState() const {
@@ -625,15 +622,7 @@ void Thread::SetupForResume() {
       const addr_t thread_pc = reg_ctx_sp->GetPC();
       BreakpointSiteSP bp_site_sp =
           GetProcess()->GetBreakpointSiteList().FindByAddress(thread_pc);
-      // If we're at a BreakpointSite which we have either
-      //   1. already triggered/hit, or
-      //   2. the Breakpoint was added while stopped, or the pc was moved
-      //      to this BreakpointSite
-      // Step past the breakpoint before resuming.
-      // If we stopped at a breakpoint instruction/BreakpointSite location
-      // without hitting it, and we're still at that same address on
-      // resuming, then we want to hit the BreakpointSite when we resume.
-      if (bp_site_sp && m_stopped_at_unexecuted_bp != thread_pc) {
+      if (bp_site_sp) {
         // Note, don't assume there's a ThreadPlanStepOverBreakpoint, the
         // target may not require anything special to step over a breakpoint.
 
@@ -722,7 +711,6 @@ bool Thread::ShouldResume(StateType resume_state) {
 
   if (need_to_resume) {
     ClearStackFrames();
-    m_stopped_at_unexecuted_bp = LLDB_INVALID_ADDRESS;
     // Let Thread subclasses do any special work they need to prior to resuming
     WillResume(resume_state);
   }
@@ -1906,7 +1894,6 @@ Unwind &Thread::GetUnwinder() {
 void Thread::Flush() {
   ClearStackFrames();
   m_reg_context_sp.reset();
-  m_stopped_at_unexecuted_bp = LLDB_INVALID_ADDRESS;
 }
 
 bool Thread::IsStillAtLastBreakpointHit() {
diff --git a/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py b/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
index ecea28c6e1f6d..73de4a294388b 100644
--- a/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
+++ b/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
@@ -2,6 +2,7 @@
 Test that we handle breakpoints on consecutive instructions correctly.
 """
 
+
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -63,30 +64,20 @@ def test_single_step(self):
         """Test that single step stops at the second breakpoint."""
         self.prepare_test()
 
-        # Instruction step to the next instruction
-        # We haven't executed breakpoint2 yet, we're sitting at it now.
-        step_over = False
-        self.thread.StepInstruction(step_over)
-
         step_over = False
         self.thread.StepInstruction(step_over)
 
-        # We've now hit the breakpoint and this StepInstruction has
-        # been interrupted, it is still sitting on the thread plan stack.
-
         self.assertState(self.process.GetState(), lldb.eStateStopped)
         self.assertEqual(
             self.thread.GetFrameAtIndex(0).GetPCAddress().GetLoadAddress(self.target),
             self.bkpt_address.GetLoadAddress(self.target),
         )
-
-        # One more instruction to complete the Step that was interrupted
-        # earlier.
-        self.thread.StepInstruction(step_over)
-        strm = lldb.SBStream()
-        self.thread.GetDescription(strm)
-        self.assertIn("instruction step into", strm.GetData())
-        self.assertIsNotNone(self.thread, "Expected to see that step-in had completed")
+        self.thread = lldbutil.get_one_thread_stopped_at_breakpoint(
+            self.process, self.breakpoint2
+        )
+        self.assertIsNotNone(
+            self.thread, "Expected one thread to be stopped at breakpoint 2"
+        )
 
         self.finish_test()
 
@@ -115,7 +106,4 @@ def test_single_step_thread_specific(self):
             "Stop reason should be 'plan complete'",
         )
 
-        # Hit our second breakpoint
-        self.process.Continue()
-
         self.finish_test()
diff --git a/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py b/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
index 1315288b35c55..3a7440a31677a 100644
--- a/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
+++ b/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
@@ -5,6 +5,7 @@
 and eStopReasonPlanComplete when breakpoint's condition fails.
 """
 
+
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -89,11 +90,6 @@ def test_step_instruction(self):
                 self.assertGreaterEqual(step_count, steps_expected)
                 break
 
-        # We did a `stepi` when we hit our last breakpoint, and the stepi was not
-        # completed yet, so when we resume it will complete (running process.Continue()
-        # would have the same result - we step one instruction and stop again when
-        # our interrupted stepi completes).
-        self.thread.StepInstruction(True)
         # Run the process until termination
         self.process.Continue()
         self.assertState(self.process.GetState(), lldb.eStateExited)

From cfc22605f61baf83035dba2e7c6db547a3fb7c62 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Sat, 20 Jul 2024 10:44:30 +0900
Subject: [PATCH 255/486] InstrProf: Mark BiasLI as invariant. (#95588)

Bias doesn't change after startup.

The test is enhanced for optimized sequences and atomic ops.
---
 .../Instrumentation/InstrProfiling.cpp        |  3 +++
 .../runtime-counter-relocation.ll             | 26 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index f994f8a62c320..09c56eb5fe6aa 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -945,6 +945,9 @@ Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
     IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
     auto *Bias = getOrCreateBiasVar(getInstrProfCounterBiasVarName());
     BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias, "profc_bias");
+    // Bias doesn't change after startup.
+    BiasLI->setMetadata(LLVMContext::MD_invariant_load,
+                        MDNode::get(M.getContext(), std::nullopt));
   }
   auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
   return Builder.CreateIntToPtr(Add, Addr->getType());
diff --git a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
index 53b1e4918e8d1..3f0dbef387d2f 100644
--- a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
@@ -1,9 +1,13 @@
 ; RUN: opt < %s -S -passes=instrprof | FileCheck %s
 ; RUN: opt < %s -S -passes=instrprof -runtime-counter-relocation | FileCheck -check-prefixes=RELOC %s
+; RUN: opt < %s -S -passes=instrprof,inline,gvn -runtime-counter-relocation | FileCheck -check-prefixes=RELOC,RELOCOPT %s
+; RUN: opt < %s -S -passes=instrprof            -runtime-counter-relocation -instrprof-atomic-counter-update-all | FileCheck -check-prefixes=ATOMIC %s
+; RUN: opt < %s -S -passes=instrprof,inline,gvn -runtime-counter-relocation -instrprof-atomic-counter-update-all | FileCheck -check-prefixes=ATOMIC,ATOMICOPT %s
 
 target triple = "x86_64-unknown-linux-gnu"
 
 @__profn_foo = private constant [3 x i8] c"foo"
+@__profn_bar = private constant [3 x i8] c"bar"
 ; RELOC: $__llvm_profile_counter_bias = comdat any
 ; RELOC: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0, comdat
 
@@ -12,14 +16,34 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK-NEXT: %[[PGOCOUNTINC:.+]] = add i64 %[[PGOCOUNT]], 1
 ; CHECK-NEXT: store i64 %[[PGOCOUNTINC]], ptr @__profc_foo
 ; RELOC-LABEL: define void @foo
-; RELOC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias
+; RELOC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align 8, !invariant.load !0
 ; RELOC-NEXT: %[[PROFC_BIAS:.+]] = add i64 ptrtoint (ptr @__profc_foo to i64), %[[BIAS]]
 ; RELOC-NEXT: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_BIAS]] to ptr
 ; RELOC-NEXT: %[[PGOCOUNT:.+]] = load i64, ptr %[[PROFC_ADDR]]
 ; RELOC-NEXT: %[[PGOCOUNTINC:.+]] = add i64 %[[PGOCOUNT]], 1
 ; RELOC-NEXT: store i64 %[[PGOCOUNTINC]], ptr %[[PROFC_ADDR]]
+; RELOCOPT-NEXT: %[[PROFC_BIAS1:.+]] = add i64 ptrtoint (ptr @__profc_bar to i64), %[[BIAS]]
+; RELOCOPT-NEXT: %[[PROFC_ADDR1:.+]] = inttoptr i64 %[[PROFC_BIAS1]] to ptr
+; RELOCOPT-NEXT: %[[PGOCOUNT1:.+]] = load i64, ptr %[[PROFC_ADDR1]]
+; RELOCOPT-NEXT: %[[PGOCOUNTINC1:.+]] = add i64 %[[PGOCOUNT1]], 1
+; RELOCOPT-NEXT: store i64 %[[PGOCOUNTINC1]], ptr %[[PROFC_ADDR1]]
+; ATOMIC-LABEL: define void @foo
+; ATOMIC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align 8, !invariant.load !0
+; ATOMIC-NEXT: %[[PROFC_BIAS:.+]] = add i64 ptrtoint (ptr @__profc_foo to i64), %[[BIAS]]
+; ATOMIC-NEXT: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_BIAS]] to ptr
+; ATOMIC-NEXT: %[[PGOCOUNTINC:.+]] = atomicrmw add ptr %[[PROFC_ADDR]], i64 1 monotonic
+; ATOMICOPT-NEXT: %[[PROFC_BIAS1:.+]] = add i64 ptrtoint (ptr @__profc_bar to i64), %[[BIAS]]
+; ATOMICOPT-NEXT: %[[PROFC_ADDR1:.+]] = inttoptr i64 %[[PROFC_BIAS1]] to ptr
+; ATOMICOPT-NEXT: %[[PGOCOUNTINC1:.+]] = atomicrmw add ptr %[[PROFC_ADDR1]], i64 1 monotonic
+
+define void @bar() {
+  call void @llvm.instrprof.increment(ptr @__profn_bar, i64 0, i32 1, i32 0)
+  ret void
+}
+
 define void @foo() {
   call void @llvm.instrprof.increment(ptr @__profn_foo, i64 0, i32 1, i32 0)
+  call void @bar()
   ret void
 }
 

From abaf13ad589d72b356a8788a5095f869d9d038d0 Mon Sep 17 00:00:00 2001
From: Jorge Gorbe Moya <jgorbe@google.com>
Date: Fri, 19 Jul 2024 18:58:25 -0700
Subject: [PATCH 256/486] Revert "Reapply "Add source file name for template
 instantiations in -ftime-trace"" (#99731)

Reverts llvm/llvm-project#99545

There were a couple of issues reported in the PR: a sanitizer warning
(https://lab.llvm.org/buildbot/#/builders/164/builds/1246/steps/14/logs/stdio)
and a tmp file accidentally included in the commit.
---
 a-abfdec1d.o.tmp                              |   0
 clang/docs/ReleaseNotes.rst                   |   3 -
 clang/include/clang/Driver/Options.td         |   4 -
 .../include/clang/Frontend/FrontendOptions.h  |   8 +-
 clang/lib/Driver/ToolChains/Clang.cpp         |   1 -
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  11 +-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |  11 +-
 clang/test/Driver/ftime-trace-sections.cpp    |   2 +-
 clang/test/Driver/ftime-trace.cpp             |  39 +++---
 clang/tools/driver/cc1_main.cpp               |   3 +-
 clang/unittests/Support/TimeProfilerTest.cpp  | 123 ++++--------------
 llvm/include/llvm/Support/TimeProfiler.h      |  23 +---
 llvm/lib/Support/TimeProfiler.cpp             |  61 ++-------
 13 files changed, 64 insertions(+), 225 deletions(-)
 delete mode 100644 a-abfdec1d.o.tmp

diff --git a/a-abfdec1d.o.tmp b/a-abfdec1d.o.tmp
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 7ac6ed934290d..aa91663260755 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -750,9 +750,6 @@ Improvements to Clang's time-trace
 - Clang now specifies that using ``auto`` in a lambda parameter is a C++14 extension when
   appropriate. (`#46059: <https://github.com/llvm/llvm-project/issues/46059>`_).
 
-- Clang now adds source file infomation for template instantiations as ``event["args"]["filename"]``. This
-  added behind an option ``-ftime-trace-verbose``. This is expected to increase the size of trace by 2-3 times.
-
 Improvements to Coverage Mapping
 --------------------------------
 
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9c6cebd77ff0a..8707e71f2a319 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3998,10 +3998,6 @@ def ftime_trace_granularity_EQ : Joined<["-"], "ftime-trace-granularity=">, Grou
   HelpText<"Minimum time granularity (in microseconds) traced by time profiler">,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
   MarshallingInfoInt<FrontendOpts<"TimeTraceGranularity">, "500u">;
-def ftime_trace_verbose : Joined<["-"], "ftime-trace-verbose">, Group<f_Group>,
-  HelpText<"Make time trace capture verbose event details (e.g. source filenames). This can increase the size of the output by 2-3 times">,
-  Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
-  MarshallingInfoFlag<FrontendOpts<"TimeTraceVerbose">>;
 def ftime_trace_EQ : Joined<["-"], "ftime-trace=">, Group<f_Group>,
   HelpText<"Similar to -ftime-trace. Specify the JSON file or a directory which will contain the JSON file">,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 8241925c98476..5e5034fe01eb5 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -580,11 +580,6 @@ class FrontendOptions {
   /// Minimum time granularity (in microseconds) traced by time profiler.
   unsigned TimeTraceGranularity;
 
-  /// Make time trace capture verbose event details (e.g. source filenames).
-  /// This can increase the size of the output by 2-3 times.
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned TimeTraceVerbose : 1;
-
   /// Path which stores the output files for -ftime-trace
   std::string TimeTracePath;
 
@@ -606,8 +601,7 @@ class FrontendOptions {
         EmitSymbolGraph(false), EmitExtensionSymbolGraphs(false),
         EmitSymbolGraphSymbolLabelsForTesting(false),
         EmitPrettySymbolGraphs(false), GenReducedBMI(false),
-        UseClangIRPipeline(false), TimeTraceGranularity(500),
-        TimeTraceVerbose(false) {}
+        UseClangIRPipeline(false), TimeTraceGranularity(500) {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file
   /// extension. For example, "c" would return Language::C.
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index f7b987bf810c1..c9f0fe0a3d60f 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6757,7 +6757,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (const char *Name = C.getTimeTraceFile(&JA)) {
     CmdArgs.push_back(Args.MakeArgString("-ftime-trace=" + Twine(Name)));
     Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_granularity_EQ);
-    Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_verbose);
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_ftrapv_handler_EQ)) {
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 725b62db5e80a..a7bc6749c5852 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -3426,16 +3426,11 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
     return true;
 
   llvm::TimeTraceScope TimeScope("InstantiateClass", [&]() {
-    llvm::TimeTraceMetadata M;
-    llvm::raw_string_ostream OS(M.Detail);
+    std::string Name;
+    llvm::raw_string_ostream OS(Name);
     Instantiation->getNameForDiagnostic(OS, getPrintingPolicy(),
                                         /*Qualified=*/true);
-    if (llvm::isTimeTraceVerbose()) {
-      auto Loc = SourceMgr.getExpansionLoc(Instantiation->getLocation());
-      M.File = SourceMgr.getFilename(Loc);
-      M.Line = SourceMgr.getExpansionLineNumber(Loc);
-    }
-    return M;
+    return Name;
   });
 
   Pattern = PatternDef;
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index a12d2eff1d2c8..97161febc15f7 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -4966,16 +4966,11 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   }
 
   llvm::TimeTraceScope TimeScope("InstantiateFunction", [&]() {
-    llvm::TimeTraceMetadata M;
-    llvm::raw_string_ostream OS(M.Detail);
+    std::string Name;
+    llvm::raw_string_ostream OS(Name);
     Function->getNameForDiagnostic(OS, getPrintingPolicy(),
                                    /*Qualified=*/true);
-    if (llvm::isTimeTraceVerbose()) {
-      auto Loc = SourceMgr.getExpansionLoc(Function->getLocation());
-      M.File = SourceMgr.getFilename(Loc);
-      M.Line = SourceMgr.getExpansionLineNumber(Loc);
-    }
-    return M;
+    return Name;
   });
 
   // If we're performing recursive template instantiation, create our own
diff --git a/clang/test/Driver/ftime-trace-sections.cpp b/clang/test/Driver/ftime-trace-sections.cpp
index da7109b9d81a6..0c16052bc0c3a 100644
--- a/clang/test/Driver/ftime-trace-sections.cpp
+++ b/clang/test/Driver/ftime-trace-sections.cpp
@@ -1,5 +1,5 @@
 // RUN: rm -rf %t && mkdir %t && cd %t
-// RUN: %clangxx -S -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
+// RUN: %clangxx -S -ftime-trace -ftime-trace-granularity=0 -o out %s
 // RUN: %python %S/ftime-trace-sections.py < out.json
 
 template <typename T>
diff --git a/clang/test/Driver/ftime-trace.cpp b/clang/test/Driver/ftime-trace.cpp
index 60c5885704b58..5fe63de915a71 100644
--- a/clang/test/Driver/ftime-trace.cpp
+++ b/clang/test/Driver/ftime-trace.cpp
@@ -1,18 +1,18 @@
 // RUN: rm -rf %t && mkdir -p %t && cd %t
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace -ftime-trace-granularity=0 -o out %s
 // RUN: cat out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=new-name.json -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=new-name.json -ftime-trace-granularity=0 -o out %s
 // RUN: cat new-name.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
 // RUN: mkdir dir1 dir2
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir1 -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir1 -ftime-trace-granularity=0 -o out %s
 // RUN: cat dir1/out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir2/ -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir2/ -ftime-trace-granularity=0 -o out %s
 // RUN: cat dir2/out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
@@ -34,33 +34,32 @@
 // RUN: mkdir d e f && cp %s d/a.cpp && touch d/b.c
 
 /// TODO: Support -fno-integrated-as.
-// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -fintegrated-as d/a.cpp -o e/a.o 2>&1 | FileCheck %s --check-prefix=COMPILE1
-// COMPILE1: -cc1{{.*}} "-ftime-trace=e/a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -fintegrated-as d/a.cpp -o e/a.o 2>&1 | FileCheck %s --check-prefix=COMPILE1
+// COMPILE1: -cc1{{.*}} "-ftime-trace=e/a.json" "-ftime-trace-granularity=0"
 
-// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=COMPILE2
-// COMPILE2: -cc1{{.*}} "-ftime-trace=f/a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
-// COMPILE2: -cc1{{.*}} "-ftime-trace=f/b.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=COMPILE2
+// COMPILE2: -cc1{{.*}} "-ftime-trace=f/a.json" "-ftime-trace-granularity=0"
+// COMPILE2: -cc1{{.*}} "-ftime-trace=f/b.json" "-ftime-trace-granularity=0"
 
 /// -o specifies the link output. Create ${output}-${basename}.json.
-// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o e/x 2>&1 | FileCheck %s --check-prefix=LINK1
-// LINK1: -cc1{{.*}} "-ftime-trace=e/x-a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
-// LINK1: -cc1{{.*}} "-ftime-trace=e/x-b.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -o e/x 2>&1 | FileCheck %s --check-prefix=LINK1
+// LINK1: -cc1{{.*}} "-ftime-trace=e/x-a.json" "-ftime-trace-granularity=0"
+// LINK1: -cc1{{.*}} "-ftime-trace=e/x-b.json" "-ftime-trace-granularity=0"
 
 /// -dumpdir is f/g, not ending with a path separator. We create f/g${basename}.json.
-// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o e/x -dumpdir f/g 2>&1 | FileCheck %s --check-prefix=LINK2
-// LINK2: -cc1{{.*}} "-ftime-trace=f/ga.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
-// LINK2: -cc1{{.*}} "-ftime-trace=f/gb.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -o e/x -dumpdir f/g 2>&1 | FileCheck %s --check-prefix=LINK2
+// LINK2: -cc1{{.*}} "-ftime-trace=f/ga.json" "-ftime-trace-granularity=0"
+// LINK2: -cc1{{.*}} "-ftime-trace=f/gb.json" "-ftime-trace-granularity=0"
 
-// RUN: %clang -### -ftime-trace=e -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o f/x -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=LINK3
-// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}a-{{[^.]*}}.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
-// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}b-{{[^.]*}}.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// RUN: %clang -### -ftime-trace=e -ftime-trace-granularity=0 d/a.cpp d/b.c -o f/x -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=LINK3
+// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}a-{{[^.]*}}.json" "-ftime-trace-granularity=0"
+// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}b-{{[^.]*}}.json" "-ftime-trace-granularity=0"
 
-// RUN: %clang -### -ftime-trace -ftime-trace=e -ftime-trace-granularity=1 -ftime-trace-verbose -xassembler d/a.cpp 2>&1 | \
+// RUN: %clang -### -ftime-trace -ftime-trace=e -ftime-trace-granularity=1 -xassembler d/a.cpp 2>&1 | \
 // RUN:   FileCheck %s --check-prefix=UNUSED
 // UNUSED:      warning: argument unused during compilation: '-ftime-trace'
 // UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace=e'
 // UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace-granularity=1'
-// UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace-verbose'
 // UNUSED-NOT:  warning:
 
 template <typename T>
diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp
index f5e5fad36573e..c2ccb47a15bc8 100644
--- a/clang/tools/driver/cc1_main.cpp
+++ b/clang/tools/driver/cc1_main.cpp
@@ -241,8 +241,7 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
 
   if (!Clang->getFrontendOpts().TimeTracePath.empty()) {
     llvm::timeTraceProfilerInitialize(
-        Clang->getFrontendOpts().TimeTraceGranularity, Argv0,
-        Clang->getFrontendOpts().TimeTraceVerbose);
+        Clang->getFrontendOpts().TimeTraceGranularity, Argv0);
   }
   // --print-supported-cpus takes priority over the actual compilation.
   if (Clang->getFrontendOpts().PrintSupportedCPUs)
diff --git a/clang/unittests/Support/TimeProfilerTest.cpp b/clang/unittests/Support/TimeProfilerTest.cpp
index 56d880cffde61..5f3950ff033f1 100644
--- a/clang/unittests/Support/TimeProfilerTest.cpp
+++ b/clang/unittests/Support/TimeProfilerTest.cpp
@@ -10,15 +10,11 @@
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Lex/PreprocessorOptions.h"
 
-#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/JSON.h"
-#include "llvm/Support/Path.h"
 #include "llvm/Support/TimeProfiler.h"
-#include "llvm/Support/VirtualFileSystem.h"
 #include <stack>
 
 #include "gtest/gtest.h"
-#include <tuple>
 
 using namespace clang;
 using namespace llvm;
@@ -27,8 +23,7 @@ namespace {
 
 // Should be called before testing.
 void setupProfiler() {
-  timeTraceProfilerInitialize(/*TimeTraceGranularity=*/0, "test",
-                              /*TimeTraceVerbose=*/true);
+  timeTraceProfilerInitialize(/*TimeTraceGranularity=*/0, "test");
 }
 
 // Should be called after `compileFromString()`.
@@ -43,24 +38,14 @@ std::string teardownProfiler() {
 
 // Returns true if code compiles successfully.
 // We only parse AST here. This is enough for constexpr evaluation.
-bool compileFromString(StringRef Code, StringRef Standard, StringRef File,
-                       llvm::StringMap<std::string> Headers = {}) {
+bool compileFromString(StringRef Code, StringRef Standard, StringRef FileName) {
   CompilerInstance Compiler;
   Compiler.createDiagnostics();
 
-  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> FS(
-      new llvm::vfs::InMemoryFileSystem());
-  FS->addFile(File, 0, MemoryBuffer::getMemBuffer(Code));
-  for (const auto &Header : Headers) {
-    FS->addFile(Header.getKey(), 0,
-                MemoryBuffer::getMemBuffer(Header.getValue()));
-  }
-  llvm::IntrusiveRefCntPtr<FileManager> Files(
-      new FileManager(FileSystemOptions(), FS));
-  Compiler.setFileManager(Files.get());
-
   auto Invocation = std::make_shared<CompilerInvocation>();
-  std::vector<const char *> Args = {Standard.data(), File.data()};
+  Invocation->getPreprocessorOpts().addRemappedFile(
+      FileName, MemoryBuffer::getMemBuffer(Code).release());
+  const char *Args[] = {Standard.data(), FileName.data()};
   CompilerInvocation::CreateFromArgs(*Invocation, Args,
                                      Compiler.getDiagnostics());
   Compiler.setInvocation(std::move(Invocation));
@@ -75,28 +60,13 @@ bool compileFromString(StringRef Code, StringRef Standard, StringRef File,
   return Compiler.ExecuteAction(Action);
 }
 
-std::string GetMetadata(json::Object *Event) {
-  std::string Metadata;
-  llvm::raw_string_ostream OS(Metadata);
-  if (json::Object *Args = Event->getObject("args")) {
-    if (auto Detail = Args->getString("detail"))
-      OS << Detail;
-    // Use only filename to not include os-specific path separators.
-    if (auto File = Args->getString("file"))
-      OS << ", " << llvm::sys::path::filename(*File);
-    if (auto Line = Args->getInteger("line"))
-      OS << ":" << *Line;
-  }
-  return Metadata;
-}
-
 // Returns pretty-printed trace graph.
 std::string buildTraceGraph(StringRef Json) {
   struct EventRecord {
     int64_t TimestampBegin;
     int64_t TimestampEnd;
-    std::string Name;
-    std::string Metadata;
+    StringRef Name;
+    StringRef Detail;
   };
   std::vector<EventRecord> Events;
 
@@ -111,13 +81,10 @@ std::string buildTraceGraph(StringRef Json) {
     int64_t TimestampBegin = TraceEventObj->getInteger("ts").value_or(0);
     int64_t TimestampEnd =
         TimestampBegin + TraceEventObj->getInteger("dur").value_or(0);
-    std::string Name = TraceEventObj->getString("name").value_or("").str();
-    std::string Metadata = GetMetadata(TraceEventObj);
-
-    // Source events are asynchronous events and may not perfectly nest the
-    // synchronous events. Skip testing them.
-    if (Name == "Source")
-      continue;
+    StringRef Name = TraceEventObj->getString("name").value_or("");
+    StringRef Detail = "";
+    if (json::Object *Args = TraceEventObj->getObject("args"))
+      Detail = Args->getString("detail").value_or("");
 
     // This is a "summary" event, like "Total PerformPendingInstantiations",
     // skip it
@@ -125,7 +92,7 @@ std::string buildTraceGraph(StringRef Json) {
       continue;
 
     Events.emplace_back(
-        EventRecord{TimestampBegin, TimestampEnd, Name, Metadata});
+        EventRecord{TimestampBegin, TimestampEnd, Name, Detail});
   }
 
   // There can be nested events that are very fast, for example:
@@ -165,9 +132,9 @@ std::string buildTraceGraph(StringRef Json) {
       Stream << "| ";
     }
     Stream.write(Event.Name.data(), Event.Name.size());
-    if (!Event.Metadata.empty()) {
+    if (!Event.Detail.empty()) {
       Stream << " (";
-      Stream.write(Event.Metadata.data(), Event.Metadata.size());
+      Stream.write(Event.Detail.data(), Event.Detail.size());
       Stream << ")";
     }
     Stream << "\n";
@@ -178,7 +145,7 @@ std::string buildTraceGraph(StringRef Json) {
 } // namespace
 
 TEST(TimeProfilerTest, ConstantEvaluationCxx20) {
-  std::string Code = R"(
+  constexpr StringRef Code = R"(
 void print(double value);
 
 namespace slow_namespace {
@@ -208,7 +175,8 @@ constexpr int slow_init_list[] = {1, 1, 2, 3, 5, 8, 13, 21}; // 25th line
   setupProfiler();
   ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc"));
   std::string Json = teardownProfiler();
-  ASSERT_EQ(R"(
+  std::string TraceGraph = buildTraceGraph(Json);
+  ASSERT_TRUE(TraceGraph == R"(
 Frontend
 | ParseDeclarationOrFunctionDefinition (test.cc:2:1)
 | ParseDeclarationOrFunctionDefinition (test.cc:6:1)
@@ -234,54 +202,14 @@ Frontend
 | ParseDeclarationOrFunctionDefinition (test.cc:25:1)
 | | EvaluateAsInitializer (slow_init_list)
 | PerformPendingInstantiations
-)",
-            buildTraceGraph(Json));
-}
-
-TEST(TimeProfilerTest, TemplateInstantiations) {
-  std::string B_H = R"(
-    template <typename T>
-    T fooB(T t) {
-      return T();
-    }
+)");
 
-    #define MacroTemp(x) template <typename T> void foo##x(T) { T(); }
-  )";
-
-  std::string A_H = R"(
-    #include "b.h"
-
-    MacroTemp(MTA)
-
-    template <typename T>
-    void fooA(T t) { fooB(t); fooMTA(t); }
-  )";
-  std::string Code = R"(
-    #include "a.h"
-    void user() { fooA(0); }
-  )";
-
-  setupProfiler();
-  ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc",
-                                /*Headers=*/{{"a.h", A_H}, {"b.h", B_H}}));
-  std::string Json = teardownProfiler();
-  ASSERT_EQ(R"(
-Frontend
-| ParseFunctionDefinition (fooB)
-| ParseFunctionDefinition (fooMTA)
-| ParseFunctionDefinition (fooA)
-| ParseDeclarationOrFunctionDefinition (test.cc:3:5)
-| | ParseFunctionDefinition (user)
-| PerformPendingInstantiations
-| | InstantiateFunction (fooA<int>, a.h:7)
-| | | InstantiateFunction (fooB<int>, b.h:3)
-| | | InstantiateFunction (fooMTA<int>, a.h:4)
-)",
-            buildTraceGraph(Json));
+  // NOTE: If this test is failing, run this test with
+  // `llvm::errs() << TraceGraph;` and change the assert above.
 }
 
 TEST(TimeProfilerTest, ConstantEvaluationC99) {
-  std::string Code = R"(
+  constexpr StringRef Code = R"(
 struct {
   short quantval[4]; // 3rd line
 } value;
@@ -290,12 +218,15 @@ struct {
   setupProfiler();
   ASSERT_TRUE(compileFromString(Code, "-std=c99", "test.c"));
   std::string Json = teardownProfiler();
-  ASSERT_EQ(R"(
+  std::string TraceGraph = buildTraceGraph(Json);
+  ASSERT_TRUE(TraceGraph == R"(
 Frontend
 | ParseDeclarationOrFunctionDefinition (test.c:2:1)
 | | isIntegerConstantExpr (<test.c:3:18>)
 | | EvaluateKnownConstIntCheckOverflow (<test.c:3:18>)
 | PerformPendingInstantiations
-)",
-            buildTraceGraph(Json));
+)");
+
+  // NOTE: If this test is failing, run this test with
+  // `llvm::errs() << TraceGraph;` and change the assert above.
 }
diff --git a/llvm/include/llvm/Support/TimeProfiler.h b/llvm/include/llvm/Support/TimeProfiler.h
index 6eb92930b36fd..31f7df10916db 100644
--- a/llvm/include/llvm/Support/TimeProfiler.h
+++ b/llvm/include/llvm/Support/TimeProfiler.h
@@ -83,28 +83,16 @@ namespace llvm {
 
 class raw_pwrite_stream;
 
-struct TimeTraceMetadata {
-  std::string Detail;
-  // Source file and line number information for the event.
-  std::string File;
-  int Line;
-
-  bool isEmpty() const { return Detail.empty() && File.empty(); }
-};
-
 struct TimeTraceProfiler;
 TimeTraceProfiler *getTimeTraceProfilerInstance();
 
-bool isTimeTraceVerbose();
-
 struct TimeTraceProfilerEntry;
 
 /// Initialize the time trace profiler.
 /// This sets up the global \p TimeTraceProfilerInstance
 /// variable to be the profiler instance.
 void timeTraceProfilerInitialize(unsigned TimeTraceGranularity,
-                                 StringRef ProcName,
-                                 bool TimeTraceVerbose = false);
+                                 StringRef ProcName);
 
 /// Cleanup the time trace profiler, if it was initialized.
 void timeTraceProfilerCleanup();
@@ -140,10 +128,6 @@ TimeTraceProfilerEntry *
 timeTraceProfilerBegin(StringRef Name,
                        llvm::function_ref<std::string()> Detail);
 
-TimeTraceProfilerEntry *
-timeTraceProfilerBegin(StringRef Name,
-                       llvm::function_ref<TimeTraceMetadata()> MetaData);
-
 /// Manually begin a time section, with the given \p Name and \p Detail.
 /// This starts Async Events having \p Name as a category which is shown
 /// separately from other traces. See
@@ -180,11 +164,6 @@ class TimeTraceScope {
     if (getTimeTraceProfilerInstance() != nullptr)
       Entry = timeTraceProfilerBegin(Name, Detail);
   }
-  TimeTraceScope(StringRef Name,
-                 llvm::function_ref<TimeTraceMetadata()> Metadata) {
-    if (getTimeTraceProfilerInstance() != nullptr)
-      Entry = timeTraceProfilerBegin(Name, Metadata);
-  }
   ~TimeTraceScope() {
     if (getTimeTraceProfilerInstance() != nullptr)
       timeTraceProfilerEnd(Entry);
diff --git a/llvm/lib/Support/TimeProfiler.cpp b/llvm/lib/Support/TimeProfiler.cpp
index c2014028ddadc..9612db7d30f98 100644
--- a/llvm/lib/Support/TimeProfiler.cpp
+++ b/llvm/lib/Support/TimeProfiler.cpp
@@ -73,20 +73,12 @@ struct llvm::TimeTraceProfilerEntry {
   const TimePointType Start;
   TimePointType End;
   const std::string Name;
-  TimeTraceMetadata Metadata;
-
+  const std::string Detail;
   const bool AsyncEvent = false;
   TimeTraceProfilerEntry(TimePointType &&S, TimePointType &&E, std::string &&N,
                          std::string &&Dt, bool Ae)
-      : Start(std::move(S)), End(std::move(E)), Name(std::move(N)), Metadata(),
-        AsyncEvent(Ae) {
-    Metadata.Detail = std::move(Dt);
-  }
-
-  TimeTraceProfilerEntry(TimePointType &&S, TimePointType &&E, std::string &&N,
-                         TimeTraceMetadata &&Mt, bool Ae)
       : Start(std::move(S)), End(std::move(E)), Name(std::move(N)),
-        Metadata(std::move(Mt)), AsyncEvent(Ae) {}
+        Detail(std::move(Dt)), AsyncEvent(Ae) {}
 
   // Calculate timings for FlameGraph. Cast time points to microsecond precision
   // rather than casting duration. This avoids truncation issues causing inner
@@ -105,12 +97,10 @@ struct llvm::TimeTraceProfilerEntry {
 };
 
 struct llvm::TimeTraceProfiler {
-  TimeTraceProfiler(unsigned TimeTraceGranularity = 0, StringRef ProcName = "",
-                    bool TimeTraceVerbose = false)
+  TimeTraceProfiler(unsigned TimeTraceGranularity = 0, StringRef ProcName = "")
       : BeginningOfTime(system_clock::now()), StartTime(ClockType::now()),
         ProcName(ProcName), Pid(sys::Process::getProcessId()),
-        Tid(llvm::get_threadid()), TimeTraceGranularity(TimeTraceGranularity),
-        TimeTraceVerbose(TimeTraceVerbose) {
+        Tid(llvm::get_threadid()), TimeTraceGranularity(TimeTraceGranularity) {
     llvm::get_thread_name(ThreadName);
   }
 
@@ -123,15 +113,6 @@ struct llvm::TimeTraceProfiler {
     return Stack.back().get();
   }
 
-  TimeTraceProfilerEntry *
-  begin(std::string Name, llvm::function_ref<TimeTraceMetadata()> Metadata,
-        bool AsyncEvent = false) {
-    Stack.emplace_back(std::make_unique<TimeTraceProfilerEntry>(
-        ClockType::now(), TimePointType(), std::move(Name), Metadata(),
-        AsyncEvent));
-    return Stack.back().get();
-  }
-
   void end() {
     assert(!Stack.empty() && "Must call begin() first");
     end(*Stack.back());
@@ -203,15 +184,8 @@ struct llvm::TimeTraceProfiler {
           J.attribute("dur", DurUs);
         }
         J.attribute("name", E.Name);
-        if (!E.Metadata.isEmpty()) {
-          J.attributeObject("args", [&] {
-            if (!E.Metadata.Detail.empty())
-              J.attribute("detail", E.Metadata.Detail);
-            if (!E.Metadata.File.empty())
-              J.attribute("file", E.Metadata.File);
-            if (E.Metadata.Line > 0)
-              J.attribute("line", E.Metadata.Line);
-          });
+        if (!E.Detail.empty()) {
+          J.attributeObject("args", [&] { J.attribute("detail", E.Detail); });
         }
       });
 
@@ -333,25 +307,14 @@ struct llvm::TimeTraceProfiler {
 
   // Minimum time granularity (in microseconds)
   const unsigned TimeTraceGranularity;
-
-  // Make time trace capture verbose event details (e.g. source filenames). This
-  // can increase the size of the output by 2-3 times.
-  const bool TimeTraceVerbose;
 };
 
-bool llvm::isTimeTraceVerbose() {
-  return getTimeTraceProfilerInstance() &&
-         getTimeTraceProfilerInstance()->TimeTraceVerbose;
-}
-
 void llvm::timeTraceProfilerInitialize(unsigned TimeTraceGranularity,
-                                       StringRef ProcName,
-                                       bool TimeTraceVerbose) {
+                                       StringRef ProcName) {
   assert(TimeTraceProfilerInstance == nullptr &&
          "Profiler should not be initialized");
   TimeTraceProfilerInstance = new TimeTraceProfiler(
-      TimeTraceGranularity, llvm::sys::path::filename(ProcName),
-      TimeTraceVerbose);
+      TimeTraceGranularity, llvm::sys::path::filename(ProcName));
 }
 
 // Removes all TimeTraceProfilerInstances.
@@ -418,14 +381,6 @@ llvm::timeTraceProfilerBegin(StringRef Name,
   return nullptr;
 }
 
-TimeTraceProfilerEntry *
-llvm::timeTraceProfilerBegin(StringRef Name,
-                             llvm::function_ref<TimeTraceMetadata()> Metadata) {
-  if (TimeTraceProfilerInstance != nullptr)
-    return TimeTraceProfilerInstance->begin(std::string(Name), Metadata, false);
-  return nullptr;
-}
-
 TimeTraceProfilerEntry *llvm::timeTraceAsyncProfilerBegin(StringRef Name,
                                                           StringRef Detail) {
   if (TimeTraceProfilerInstance != nullptr)

From d4dc8e4a85c4a88a9080ac7b4e478498cb5edd33 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Sat, 20 Jul 2024 11:49:03 +0900
Subject: [PATCH 257/486] Fixup for #95588, don't assume `align 8`.

This shall fix aarch64 builders.
---
 .../InstrProfiling/runtime-counter-relocation.ll              | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
index 3f0dbef387d2f..e1da23e7be31c 100644
--- a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
@@ -16,7 +16,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK-NEXT: %[[PGOCOUNTINC:.+]] = add i64 %[[PGOCOUNT]], 1
 ; CHECK-NEXT: store i64 %[[PGOCOUNTINC]], ptr @__profc_foo
 ; RELOC-LABEL: define void @foo
-; RELOC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align 8, !invariant.load !0
+; RELOC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align {{[0-9]+}}, !invariant.load !0
 ; RELOC-NEXT: %[[PROFC_BIAS:.+]] = add i64 ptrtoint (ptr @__profc_foo to i64), %[[BIAS]]
 ; RELOC-NEXT: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_BIAS]] to ptr
 ; RELOC-NEXT: %[[PGOCOUNT:.+]] = load i64, ptr %[[PROFC_ADDR]]
@@ -28,7 +28,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; RELOCOPT-NEXT: %[[PGOCOUNTINC1:.+]] = add i64 %[[PGOCOUNT1]], 1
 ; RELOCOPT-NEXT: store i64 %[[PGOCOUNTINC1]], ptr %[[PROFC_ADDR1]]
 ; ATOMIC-LABEL: define void @foo
-; ATOMIC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align 8, !invariant.load !0
+; ATOMIC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align {{[0-9]+}}, !invariant.load !0
 ; ATOMIC-NEXT: %[[PROFC_BIAS:.+]] = add i64 ptrtoint (ptr @__profc_foo to i64), %[[BIAS]]
 ; ATOMIC-NEXT: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_BIAS]] to ptr
 ; ATOMIC-NEXT: %[[PGOCOUNTINC:.+]] = atomicrmw add ptr %[[PROFC_ADDR]], i64 1 monotonic

From 2d756d9d4c89ac50404f942f2db2f4a402aa0e00 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 19 Jul 2024 20:12:13 -0700
Subject: [PATCH 258/486] [lld-macho,test] Adjust reproduce-thin-archive-objc.s

When `cd %t` is used, it's conventional to move it above and omit `-o /dev/null`.

We don't check the string before `warning:` since (a) the string is not
very useful and (b) downstream might customize `ctx->e.logName`
(argv[0]).

count 0 is better than `--allow-empty`. In addition, without `2>&1` the
previous test was effective.
---
 lld/test/MachO/reproduce-thin-archive-objc.s | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/lld/test/MachO/reproduce-thin-archive-objc.s b/lld/test/MachO/reproduce-thin-archive-objc.s
index c5fe42f130526..8159f03f0f740 100644
--- a/lld/test/MachO/reproduce-thin-archive-objc.s
+++ b/lld/test/MachO/reproduce-thin-archive-objc.s
@@ -4,20 +4,19 @@
 ## during linking. However, we need to iterate over all members for -ObjC, check that we don't
 ## crash when we encounter a missing member.
 
-# RUN: rm -rf %t; mkdir %t
-# RUN: sed s/SYM/_main/   %s | llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/main.o
-# RUN: sed s/SYM/_unused/ %s | llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/unused.o
+# RUN: rm -rf %t && mkdir %t && cd %t
+# RUN: sed s/SYM/_main/   %s | llvm-mc -filetype=obj -triple=x86_64-apple-macos -o main.o
+# RUN: sed s/SYM/_unused/ %s | llvm-mc -filetype=obj -triple=x86_64-apple-macos -o unused.o
 
-# RUN: cd %t; llvm-ar rcsT unused.a unused.o; rm unused.o
+# RUN: llvm-ar rcsT unused.a unused.o; rm unused.o
 ## FIXME: Absolute paths don't end up relativized in the repro file.
 
 # RUN: %no-fatal-warnings-lld %t/main.o %t/unused.a -ObjC -o /dev/null 2>&1 \
 # RUN:                      | FileCheck %s --check-prefix=WARN
 
-# RUN: %lld %t/main.o %t/unused.a -ObjC --no-warn-thin-archive-missing-members -o /dev/null \
-# RUN:    | FileCheck %s --implicit-check-not 'warning' --allow-empty
+# RUN: %lld main.o unused.a -ObjC --no-warn-thin-archive-missing-members 2>&1 | count 0
 
-# WARN: ld64.lld: warning: {{.*}}unused.a: -ObjC failed to open archive member: 'unused.o'
+# WARN: warning: {{.*}}unused.a: -ObjC failed to open archive member: 'unused.o'
 
 .text
 .globl SYM

From 740161a9b98c9920dedf1852b5f1c94d0a683af5 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Sat, 20 Jul 2024 12:02:48 +0900
Subject: [PATCH 259/486] Revert "[LLVM][LTO] Factor out RTLib calls and allow
 them to be dropped (#98512)"

This reverts commit c05126bdfc3b02daa37d11056fa43db1a6cdef69.
(llvmorg-19-init-17714-gc05126bdfc3b)
See #99610
---
 lld/COFF/Driver.cpp                           |   7 +-
 lld/ELF/Driver.cpp                            |   6 +-
 lld/wasm/Driver.cpp                           |   6 +-
 .../llvm/CodeGen/GlobalISel/LegalizerHelper.h |   2 +-
 .../include/llvm/CodeGen/RuntimeLibcallUtil.h |  96 -----
 llvm/include/llvm/CodeGen/RuntimeLibcalls.h   | 113 +++++
 llvm/include/llvm/CodeGen/TargetLowering.h    |  37 +-
 llvm/include/llvm/IR/RuntimeLibcalls.h        | 126 ------
 llvm/include/llvm/LTO/LTO.h                   |   2 +-
 llvm/lib/CodeGen/AtomicExpandPass.cpp         |   2 +-
 llvm/lib/CodeGen/DwarfEHPrepare.cpp           |   2 +-
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |   2 +-
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |   2 +-
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |   2 +-
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |   2 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   2 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      |   2 +-
 .../SelectionDAG/StatepointLowering.cpp       |   2 +-
 llvm/lib/CodeGen/TargetLoweringBase.cpp       | 387 +++++++++++++++++-
 llvm/lib/IR/CMakeLists.txt                    |   1 -
 llvm/lib/IR/RuntimeLibcalls.cpp               | 379 -----------------
 llvm/lib/LTO/LTO.cpp                          |  14 +-
 llvm/lib/Object/IRSymtab.cpp                  |  20 +-
 llvm/lib/Target/AArch64/AArch64FastISel.cpp   |   2 +-
 .../Target/AArch64/AArch64ISelLowering.cpp    |   2 +-
 llvm/lib/Target/ARM/ARMFastISel.cpp           |   2 +-
 llvm/lib/Target/ARM/ARMISelLowering.cpp       |   2 +-
 llvm/lib/Target/ARM/ARMLegalizerInfo.h        |   2 +-
 llvm/lib/Target/ARM/ARMSelectionDAGInfo.h     |   2 +-
 .../Target/Hexagon/HexagonISelLowering.cpp    |   4 +-
 llvm/lib/Target/Lanai/LanaiISelLowering.cpp   |   2 +-
 .../LoongArch/LoongArchISelLowering.cpp       |   2 +-
 llvm/lib/Target/Mips/MipsISelLowering.cpp     |   2 +-
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |   2 +-
 .../WebAssemblyRuntimeLibcallSignatures.cpp   |   2 +-
 .../WebAssemblyRuntimeLibcallSignatures.h     |   2 +-
 llvm/tools/lto/lto.cpp                        |   2 +-
 37 files changed, 568 insertions(+), 676 deletions(-)
 delete mode 100644 llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
 create mode 100644 llvm/include/llvm/CodeGen/RuntimeLibcalls.h
 delete mode 100644 llvm/include/llvm/IR/RuntimeLibcalls.h
 delete mode 100644 llvm/lib/IR/RuntimeLibcalls.cpp

diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 9e28b1c50be50..cef6271e4c8f8 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -2428,12 +2428,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       // file's symbol table. If any of those library functions are defined in a
       // bitcode file in an archive member, we need to arrange to use LTO to
       // compile those archive members by adding them to the link beforehand.
-      if (!ctx.bitcodeFileInstances.empty()) {
-        llvm::Triple TT(
-            ctx.bitcodeFileInstances.front()->obj->getTargetTriple());
-        for (auto *s : lto::LTO::getRuntimeLibcallSymbols(TT))
+      if (!ctx.bitcodeFileInstances.empty())
+        for (auto *s : lto::LTO::getRuntimeLibcallSymbols())
           ctx.symtab.addLibcall(s);
-      }
 
       // Windows specific -- if __load_config_used can be resolved, resolve it.
       if (ctx.symtab.findUnderscore("_load_config_used"))
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 40e095a133d95..3a3070aea94f5 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -2884,11 +2884,9 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // to, i.e. if the symbol's definition is in bitcode. Any other required
   // libcall symbols will be added to the link after LTO when we add the LTO
   // object file to the link.
-  if (!ctx.bitcodeFiles.empty()) {
-    llvm::Triple TT(ctx.bitcodeFiles.front()->obj->getTargetTriple());
-    for (auto *s : lto::LTO::getRuntimeLibcallSymbols(TT))
+  if (!ctx.bitcodeFiles.empty())
+    for (auto *s : lto::LTO::getRuntimeLibcallSymbols())
       handleLibcall(s);
-  }
 
   // Archive members defining __wrap symbols may be extracted.
   std::vector<WrappedSymbol> wrapped = addWrappedSymbols(args);
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index 8c83d17db02f5..b66b988005d58 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -1320,11 +1320,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   // We only need to add libcall symbols to the link before LTO if the symbol's
   // definition is in bitcode. Any other required libcall symbols will be added
   // to the link after LTO when we add the LTO object file to the link.
-  if (!ctx.bitcodeFiles.empty()) {
-    llvm::Triple TT(ctx.bitcodeFiles.front()->obj->getTargetTriple());
-    for (auto *s : lto::LTO::getRuntimeLibcallSymbols(TT))
+  if (!ctx.bitcodeFiles.empty())
+    for (auto *s : lto::LTO::getRuntimeLibcallSymbols())
       handleLibcall(s);
-  }
   if (errorCount())
     return;
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index b17bc9aa2a44e..b599b0368b9aa 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -22,7 +22,7 @@
 
 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h b/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
deleted file mode 100644
index ce63dcc405fd5..0000000000000
--- a/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
+++ /dev/null
@@ -1,96 +0,0 @@
-//===-- CodeGen/RuntimeLibcallUtil.h - Runtime Library Calls ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines some helper functions for runtime library calls.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_RUNTIMELIBCALLS_H
-#define LLVM_CODEGEN_RUNTIMELIBCALLS_H
-
-#include "llvm/CodeGen/ValueTypes.h"
-#include "llvm/IR/RuntimeLibcalls.h"
-#include "llvm/Support/AtomicOrdering.h"
-
-namespace llvm {
-namespace RTLIB {
-
-/// GetFPLibCall - Helper to return the right libcall for the given floating
-/// point type, or UNKNOWN_LIBCALL if there is none.
-Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64,
-                     Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128);
-
-/// getFPEXT - Return the FPEXT_*_* value for the given types, or
-/// UNKNOWN_LIBCALL if there is none.
-Libcall getFPEXT(EVT OpVT, EVT RetVT);
-
-/// getFPROUND - Return the FPROUND_*_* value for the given types, or
-/// UNKNOWN_LIBCALL if there is none.
-Libcall getFPROUND(EVT OpVT, EVT RetVT);
-
-/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
-/// UNKNOWN_LIBCALL if there is none.
-Libcall getFPTOSINT(EVT OpVT, EVT RetVT);
-
-/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
-/// UNKNOWN_LIBCALL if there is none.
-Libcall getFPTOUINT(EVT OpVT, EVT RetVT);
-
-/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
-/// UNKNOWN_LIBCALL if there is none.
-Libcall getSINTTOFP(EVT OpVT, EVT RetVT);
-
-/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
-/// UNKNOWN_LIBCALL if there is none.
-Libcall getUINTTOFP(EVT OpVT, EVT RetVT);
-
-/// getPOWI - Return the POWI_* value for the given types, or
-/// UNKNOWN_LIBCALL if there is none.
-Libcall getPOWI(EVT RetVT);
-
-/// getLDEXP - Return the LDEXP_* value for the given types, or
-/// UNKNOWN_LIBCALL if there is none.
-Libcall getLDEXP(EVT RetVT);
-
-/// getFREXP - Return the FREXP_* value for the given types, or
-/// UNKNOWN_LIBCALL if there is none.
-Libcall getFREXP(EVT RetVT);
-
-/// Return the SYNC_FETCH_AND_* value for the given opcode and type, or
-/// UNKNOWN_LIBCALL if there is none.
-Libcall getSYNC(unsigned Opc, MVT VT);
-
-/// Return the outline atomics value for the given atomic ordering, access
-/// size and set of libcalls for a given atomic, or UNKNOWN_LIBCALL if there
-/// is none.
-Libcall getOutlineAtomicHelper(const Libcall (&LC)[5][4], AtomicOrdering Order,
-                               uint64_t MemSize);
-
-/// Return the outline atomics value for the given opcode, atomic ordering
-/// and type, or UNKNOWN_LIBCALL if there is none.
-Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT);
-
-/// getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return
-/// MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
-/// UNKNOW_LIBCALL if there is none.
-Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
-
-/// getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return
-/// MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
-/// UNKNOW_LIBCALL if there is none.
-Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
-
-/// getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return
-/// MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
-/// UNKNOW_LIBCALL if there is none.
-Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
-
-} // namespace RTLIB
-} // namespace llvm
-
-#endif
diff --git a/llvm/include/llvm/CodeGen/RuntimeLibcalls.h b/llvm/include/llvm/CodeGen/RuntimeLibcalls.h
new file mode 100644
index 0000000000000..3a407c4a4d940
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/RuntimeLibcalls.h
@@ -0,0 +1,113 @@
+//===-- CodeGen/RuntimeLibcalls.h - Runtime Library Calls -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the enum representing the list of runtime library calls
+// the backend may emit during code generation, and also some helper functions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_RUNTIMELIBCALLS_H
+#define LLVM_CODEGEN_RUNTIMELIBCALLS_H
+
+#include "llvm/CodeGen/ValueTypes.h"
+#include "llvm/Support/AtomicOrdering.h"
+
+namespace llvm {
+namespace RTLIB {
+  /// RTLIB::Libcall enum - This enum defines all of the runtime library calls
+  /// the backend can emit.  The various long double types cannot be merged,
+  /// because 80-bit library functions use "xf" and 128-bit use "tf".
+  ///
+  /// When adding PPCF128 functions here, note that their names generally need
+  /// to be overridden for Darwin with the xxx$LDBL128 form.  See
+  /// PPCISelLowering.cpp.
+  ///
+  enum Libcall {
+#define HANDLE_LIBCALL(code, name) code,
+    #include "llvm/IR/RuntimeLibcalls.def"
+#undef HANDLE_LIBCALL
+  };
+
+  /// GetFPLibCall - Helper to return the right libcall for the given floating
+  /// point type, or UNKNOWN_LIBCALL if there is none.
+  Libcall getFPLibCall(EVT VT,
+                       Libcall Call_F32,
+                       Libcall Call_F64,
+                       Libcall Call_F80,
+                       Libcall Call_F128,
+                       Libcall Call_PPCF128);
+
+  /// getFPEXT - Return the FPEXT_*_* value for the given types, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getFPEXT(EVT OpVT, EVT RetVT);
+
+  /// getFPROUND - Return the FPROUND_*_* value for the given types, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getFPROUND(EVT OpVT, EVT RetVT);
+
+  /// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getFPTOSINT(EVT OpVT, EVT RetVT);
+
+  /// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getFPTOUINT(EVT OpVT, EVT RetVT);
+
+  /// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getSINTTOFP(EVT OpVT, EVT RetVT);
+
+  /// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getUINTTOFP(EVT OpVT, EVT RetVT);
+
+  /// getPOWI - Return the POWI_* value for the given types, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getPOWI(EVT RetVT);
+
+  /// getLDEXP - Return the LDEXP_* value for the given types, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getLDEXP(EVT RetVT);
+
+  /// getFREXP - Return the FREXP_* value for the given types, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getFREXP(EVT RetVT);
+
+  /// Return the SYNC_FETCH_AND_* value for the given opcode and type, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getSYNC(unsigned Opc, MVT VT);
+
+  /// Return the outline atomics value for the given atomic ordering, access
+  /// size and set of libcalls for a given atomic, or UNKNOWN_LIBCALL if there
+  /// is none.
+  Libcall getOutlineAtomicHelper(const Libcall (&LC)[5][4],
+                                 AtomicOrdering Order, uint64_t MemSize);
+
+  /// Return the outline atomics value for the given opcode, atomic ordering
+  /// and type, or UNKNOWN_LIBCALL if there is none.
+  Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT);
+
+  /// getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return
+  /// MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
+  /// UNKNOW_LIBCALL if there is none.
+  Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
+
+  /// getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return
+  /// MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
+  /// UNKNOW_LIBCALL if there is none.
+  Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
+
+  /// getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return
+  /// MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
+  /// UNKNOW_LIBCALL if there is none.
+  Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
+
+}
+}
+
+#endif
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index d4a2166bf768e..64c73820add64 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -31,7 +31,7 @@
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
@@ -45,7 +45,6 @@
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/RuntimeLibcalls.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/AtomicOrdering.h"
@@ -3415,40 +3414,44 @@ class TargetLoweringBase {
     return nullptr;
   }
 
+  //===--------------------------------------------------------------------===//
+  // Runtime Library hooks
+  //
+
   /// Rename the default libcall routine name for the specified libcall.
   void setLibcallName(RTLIB::Libcall Call, const char *Name) {
-    Libcalls.setLibcallName(Call, Name);
+    LibcallRoutineNames[Call] = Name;
   }
-
   void setLibcallName(ArrayRef<RTLIB::Libcall> Calls, const char *Name) {
-    Libcalls.setLibcallName(Calls, Name);
+    for (auto Call : Calls)
+      setLibcallName(Call, Name);
   }
 
   /// Get the libcall routine name for the specified libcall.
   const char *getLibcallName(RTLIB::Libcall Call) const {
-    return Libcalls.getLibcallName(Call);
+    return LibcallRoutineNames[Call];
   }
 
   /// Override the default CondCode to be used to test the result of the
   /// comparison libcall against zero.
   void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC) {
-    Libcalls.setCmpLibcallCC(Call, CC);
+    CmpLibcallCCs[Call] = CC;
   }
 
   /// Get the CondCode that's to be used to test the result of the comparison
   /// libcall against zero.
   ISD::CondCode getCmpLibcallCC(RTLIB::Libcall Call) const {
-    return Libcalls.getCmpLibcallCC(Call);
+    return CmpLibcallCCs[Call];
   }
 
   /// Set the CallingConv that should be used for the specified libcall.
   void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC) {
-    Libcalls.setLibcallCallingConv(Call, CC);
+    LibcallCallingConvs[Call] = CC;
   }
 
   /// Get the CallingConv that should be used for the specified libcall.
   CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const {
-    return Libcalls.getLibcallCallingConv(Call);
+    return LibcallCallingConvs[Call];
   }
 
   /// Execute target specific actions to finalize target lowering.
@@ -3627,8 +3630,18 @@ class TargetLoweringBase {
   std::map<std::pair<unsigned, MVT::SimpleValueType>, MVT::SimpleValueType>
     PromoteToType;
 
-  /// The list of libcalls that the target will use.
-  RTLIB::RuntimeLibcallsInfo Libcalls;
+  /// Stores the name each libcall.
+  const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1];
+
+  /// The ISD::CondCode that should be used to test the result of each of the
+  /// comparison libcall against zero.
+  ISD::CondCode CmpLibcallCCs[RTLIB::UNKNOWN_LIBCALL];
+
+  /// Stores the CallingConv that should be used for each libcall.
+  CallingConv::ID LibcallCallingConvs[RTLIB::UNKNOWN_LIBCALL];
+
+  /// Set default libcall names and calling conventions.
+  void InitLibcalls(const Triple &TT);
 
   /// The bits of IndexedModeActions used to store the legalisation actions
   /// We store the data as   | ML | MS |  L |  S | each taking 4 bits.
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.h b/llvm/include/llvm/IR/RuntimeLibcalls.h
deleted file mode 100644
index 3057bff397b2f..0000000000000
--- a/llvm/include/llvm/IR/RuntimeLibcalls.h
+++ /dev/null
@@ -1,126 +0,0 @@
-//===- RuntimeLibcalls.h - Interface for runtime libcalls -------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements a common interface to work with library calls into a
-// runtime that may be emitted by a given backend.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_IR_RUNTIME_LIBCALLS_H
-#define LLVM_IR_RUNTIME_LIBCALLS_H
-
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/CodeGen/ISDOpcodes.h"
-#include "llvm/IR/CallingConv.h"
-#include "llvm/Support/AtomicOrdering.h"
-#include "llvm/TargetParser/Triple.h"
-
-namespace llvm {
-namespace RTLIB {
-
-/// RTLIB::Libcall enum - This enum defines all of the runtime library calls
-/// the backend can emit.  The various long double types cannot be merged,
-/// because 80-bit library functions use "xf" and 128-bit use "tf".
-///
-/// When adding PPCF128 functions here, note that their names generally need
-/// to be overridden for Darwin with the xxx$LDBL128 form.  See
-/// PPCISelLowering.cpp.
-///
-enum Libcall {
-#define HANDLE_LIBCALL(code, name) code,
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-};
-
-/// A simple container for information about the supported runtime calls.
-struct RuntimeLibcallsInfo {
-  explicit RuntimeLibcallsInfo(const Triple &TT) {
-    initLibcalls(TT);
-    initCmpLibcallCCs();
-  }
-
-  /// Rename the default libcall routine name for the specified libcall.
-  void setLibcallName(RTLIB::Libcall Call, const char *Name) {
-    LibcallRoutineNames[Call] = Name;
-  }
-
-  void setLibcallName(ArrayRef<RTLIB::Libcall> Calls, const char *Name) {
-    for (auto Call : Calls)
-      setLibcallName(Call, Name);
-  }
-
-  /// Get the libcall routine name for the specified libcall.
-  const char *getLibcallName(RTLIB::Libcall Call) const {
-    return LibcallRoutineNames[Call];
-  }
-
-  /// Override the default CondCode to be used to test the result of the
-  /// comparison libcall against zero.
-  void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC) {
-    CmpLibcallCCs[Call] = CC;
-  }
-
-  /// Get the CondCode that's to be used to test the result of the comparison
-  /// libcall against zero.
-  ISD::CondCode getCmpLibcallCC(RTLIB::Libcall Call) const {
-    return CmpLibcallCCs[Call];
-  }
-
-  /// Set the CallingConv that should be used for the specified libcall.
-  void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC) {
-    LibcallCallingConvs[Call] = CC;
-  }
-
-  /// Get the CallingConv that should be used for the specified libcall.
-  CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const {
-    return LibcallCallingConvs[Call];
-  }
-
-  iterator_range<const char **> getLibcallNames() {
-    return llvm::make_range(LibcallRoutineNames,
-                            LibcallRoutineNames + RTLIB::UNKNOWN_LIBCALL);
-  }
-
-private:
-  /// Stores the name each libcall.
-  const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1];
-
-  /// The ISD::CondCode that should be used to test the result of each of the
-  /// comparison libcall against zero.
-  ISD::CondCode CmpLibcallCCs[RTLIB::UNKNOWN_LIBCALL];
-
-  /// Stores the CallingConv that should be used for each libcall.
-  CallingConv::ID LibcallCallingConvs[RTLIB::UNKNOWN_LIBCALL];
-
-  static bool darwinHasSinCos(const Triple &TT) {
-    assert(TT.isOSDarwin() && "should be called with darwin triple");
-    // Don't bother with 32 bit x86.
-    if (TT.getArch() == Triple::x86)
-      return false;
-    // Macos < 10.9 has no sincos_stret.
-    if (TT.isMacOSX())
-      return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit();
-    // iOS < 7.0 has no sincos_stret.
-    if (TT.isiOS())
-      return !TT.isOSVersionLT(7, 0);
-    // Any other darwin such as WatchOS/TvOS is new enough.
-    return true;
-  }
-
-  /// Sets default libcall calling conventions.
-  void initCmpLibcallCCs();
-
-  /// Set default libcall names. If a target wants to opt-out of a libcall it
-  /// should be placed here.
-  void initLibcalls(const Triple &TT);
-};
-
-} // namespace RTLIB
-} // namespace llvm
-
-#endif // LLVM_IR_RUNTIME_LIBCALLS_H
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index 30eda34cd7ba5..94996ae89e35d 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -301,7 +301,7 @@ class LTO {
 
   /// Static method that returns a list of libcall symbols that can be generated
   /// by LTO but might not be visible from bitcode symbol table.
-  static SmallVector<const char *> getRuntimeLibcallSymbols(const Triple &TT);
+  static ArrayRef<const char*> getRuntimeLibcallSymbols();
 
 private:
   Config Conf;
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index ebcf76175a36b..cea09bcb45386 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -21,7 +21,7 @@
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/AtomicExpand.h"
 #include "llvm/CodeGen/AtomicExpandUtils.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
diff --git a/llvm/lib/CodeGen/DwarfEHPrepare.cpp b/llvm/lib/CodeGen/DwarfEHPrepare.cpp
index 324329ce989e7..09e7cfb12bdba 100644
--- a/llvm/lib/CodeGen/DwarfEHPrepare.cpp
+++ b/llvm/lib/CodeGen/DwarfEHPrepare.cpp
@@ -18,7 +18,7 @@
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 68a8a273a1b47..b10d51adb69ae 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -38,7 +38,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/StackProtector.h"
 #include "llvm/CodeGen/SwitchLoweringUtils.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 640a425ffa735..b8653aff690ef 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -25,7 +25,7 @@
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 9b2153c68ccae..b584aa5923ed6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -37,7 +37,7 @@
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 7f5b46af01c62..9f515739ee048 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -25,7 +25,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 02d44cd36ae53..0ccae50286345 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -36,7 +36,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 98a795edb7a03..c179c97029c7a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -44,7 +44,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
 #include "llvm/CodeGen/StackMaps.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 4268da8670d50..671ec84fb9416 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -26,7 +26,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/StackMaps.h"
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 8040f1eeae810..f0fa6aa772e37 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -28,7 +28,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/StackMaps.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
@@ -98,6 +98,350 @@ static cl::opt<bool> DisableStrictNodeMutation("disable-strictnode-mutation",
        cl::desc("Don't mutate strict-float node to a legalize node"),
        cl::init(false), cl::Hidden);
 
+static bool darwinHasSinCos(const Triple &TT) {
+  assert(TT.isOSDarwin() && "should be called with darwin triple");
+  // Don't bother with 32 bit x86.
+  if (TT.getArch() == Triple::x86)
+    return false;
+  // Macos < 10.9 has no sincos_stret.
+  if (TT.isMacOSX())
+    return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit();
+  // iOS < 7.0 has no sincos_stret.
+  if (TT.isiOS())
+    return !TT.isOSVersionLT(7, 0);
+  // Any other darwin such as WatchOS/TvOS is new enough.
+  return true;
+}
+
+void TargetLoweringBase::InitLibcalls(const Triple &TT) {
+#define HANDLE_LIBCALL(code, name) \
+  setLibcallName(RTLIB::code, name);
+#include "llvm/IR/RuntimeLibcalls.def"
+#undef HANDLE_LIBCALL
+  // Initialize calling conventions to their default.
+  for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC)
+    setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C);
+
+  // Use the f128 variants of math functions on x86_64
+  if (TT.getArch() == Triple::ArchType::x86_64 && TT.isGNUEnvironment()) {
+    setLibcallName(RTLIB::REM_F128, "fmodf128");
+    setLibcallName(RTLIB::FMA_F128, "fmaf128");
+    setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
+    setLibcallName(RTLIB::CBRT_F128, "cbrtf128");
+    setLibcallName(RTLIB::LOG_F128, "logf128");
+    setLibcallName(RTLIB::LOG_FINITE_F128, "__logf128_finite");
+    setLibcallName(RTLIB::LOG2_F128, "log2f128");
+    setLibcallName(RTLIB::LOG2_FINITE_F128, "__log2f128_finite");
+    setLibcallName(RTLIB::LOG10_F128, "log10f128");
+    setLibcallName(RTLIB::LOG10_FINITE_F128, "__log10f128_finite");
+    setLibcallName(RTLIB::EXP_F128, "expf128");
+    setLibcallName(RTLIB::EXP_FINITE_F128, "__expf128_finite");
+    setLibcallName(RTLIB::EXP2_F128, "exp2f128");
+    setLibcallName(RTLIB::EXP2_FINITE_F128, "__exp2f128_finite");
+    setLibcallName(RTLIB::EXP10_F128, "exp10f128");
+    setLibcallName(RTLIB::SIN_F128, "sinf128");
+    setLibcallName(RTLIB::COS_F128, "cosf128");
+    setLibcallName(RTLIB::TAN_F128, "tanf128");
+    setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
+    setLibcallName(RTLIB::ASIN_F128, "asinf128");
+    setLibcallName(RTLIB::ACOS_F128, "acosf128");
+    setLibcallName(RTLIB::ATAN_F128, "atanf128");
+    setLibcallName(RTLIB::SINH_F128, "sinhf128");
+    setLibcallName(RTLIB::COSH_F128, "coshf128");
+    setLibcallName(RTLIB::TANH_F128, "tanhf128");
+    setLibcallName(RTLIB::POW_F128, "powf128");
+    setLibcallName(RTLIB::POW_FINITE_F128, "__powf128_finite");
+    setLibcallName(RTLIB::CEIL_F128, "ceilf128");
+    setLibcallName(RTLIB::TRUNC_F128, "truncf128");
+    setLibcallName(RTLIB::RINT_F128, "rintf128");
+    setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
+    setLibcallName(RTLIB::ROUND_F128, "roundf128");
+    setLibcallName(RTLIB::ROUNDEVEN_F128, "roundevenf128");
+    setLibcallName(RTLIB::FLOOR_F128, "floorf128");
+    setLibcallName(RTLIB::COPYSIGN_F128, "copysignf128");
+    setLibcallName(RTLIB::FMIN_F128, "fminf128");
+    setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
+    setLibcallName(RTLIB::LROUND_F128, "lroundf128");
+    setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
+    setLibcallName(RTLIB::LRINT_F128, "lrintf128");
+    setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
+    setLibcallName(RTLIB::LDEXP_F128, "ldexpf128");
+    setLibcallName(RTLIB::FREXP_F128, "frexpf128");
+  }
+
+  // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
+  if (TT.isPPC()) {
+    setLibcallName(RTLIB::ADD_F128, "__addkf3");
+    setLibcallName(RTLIB::SUB_F128, "__subkf3");
+    setLibcallName(RTLIB::MUL_F128, "__mulkf3");
+    setLibcallName(RTLIB::DIV_F128, "__divkf3");
+    setLibcallName(RTLIB::POWI_F128, "__powikf2");
+    setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
+    setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
+    setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
+    setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
+    setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
+    setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
+    setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti");
+    setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
+    setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
+    setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti");
+    setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
+    setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
+    setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf");
+    setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
+    setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
+    setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf");
+    setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
+    setLibcallName(RTLIB::UNE_F128, "__nekf2");
+    setLibcallName(RTLIB::OGE_F128, "__gekf2");
+    setLibcallName(RTLIB::OLT_F128, "__ltkf2");
+    setLibcallName(RTLIB::OLE_F128, "__lekf2");
+    setLibcallName(RTLIB::OGT_F128, "__gtkf2");
+    setLibcallName(RTLIB::UO_F128, "__unordkf2");
+  }
+
+  // A few names are different on particular architectures or environments.
+  if (TT.isOSDarwin()) {
+    // For f16/f32 conversions, Darwin uses the standard naming scheme, instead
+    // of the gnueabi-style __gnu_*_ieee.
+    // FIXME: What about other targets?
+    setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
+    setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
+
+    // Some darwins have an optimized __bzero/bzero function.
+    switch (TT.getArch()) {
+    case Triple::x86:
+    case Triple::x86_64:
+      if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6))
+        setLibcallName(RTLIB::BZERO, "__bzero");
+      break;
+    case Triple::aarch64:
+    case Triple::aarch64_32:
+      setLibcallName(RTLIB::BZERO, "bzero");
+      break;
+    default:
+      break;
+    }
+
+    if (darwinHasSinCos(TT)) {
+      setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret");
+      setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret");
+      if (TT.isWatchABI()) {
+        setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
+                              CallingConv::ARM_AAPCS_VFP);
+        setLibcallCallingConv(RTLIB::SINCOS_STRET_F64,
+                              CallingConv::ARM_AAPCS_VFP);
+      }
+    }
+
+    switch (TT.getOS()) {
+    case Triple::MacOSX:
+      if (TT.isMacOSXVersionLT(10, 9)) {
+        setLibcallName(RTLIB::EXP10_F32, nullptr);
+        setLibcallName(RTLIB::EXP10_F64, nullptr);
+      } else {
+        setLibcallName(RTLIB::EXP10_F32, "__exp10f");
+        setLibcallName(RTLIB::EXP10_F64, "__exp10");
+      }
+      break;
+    case Triple::IOS:
+      if (TT.isOSVersionLT(7, 0)) {
+        setLibcallName(RTLIB::EXP10_F32, nullptr);
+        setLibcallName(RTLIB::EXP10_F64, nullptr);
+        break;
+      }
+      [[fallthrough]];
+    case Triple::TvOS:
+    case Triple::WatchOS:
+    case Triple::XROS:
+      setLibcallName(RTLIB::EXP10_F32, "__exp10f");
+      setLibcallName(RTLIB::EXP10_F64, "__exp10");
+      break;
+    default:
+      break;
+    }
+  } else {
+    setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
+    setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
+  }
+
+  if (TT.isGNUEnvironment() || TT.isOSFuchsia() ||
+      (TT.isAndroid() && !TT.isAndroidVersionLT(9))) {
+    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
+    setLibcallName(RTLIB::SINCOS_F64, "sincos");
+    setLibcallName(RTLIB::SINCOS_F80, "sincosl");
+    setLibcallName(RTLIB::SINCOS_F128, "sincosl");
+    setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl");
+  }
+
+  if (TT.isPS()) {
+    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
+    setLibcallName(RTLIB::SINCOS_F64, "sincos");
+  }
+
+  if (TT.isOSOpenBSD()) {
+    setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr);
+  }
+
+  if (TT.isOSWindows() && !TT.isOSCygMing()) {
+    setLibcallName(RTLIB::LDEXP_F32, nullptr);
+    setLibcallName(RTLIB::LDEXP_F80, nullptr);
+    setLibcallName(RTLIB::LDEXP_F128, nullptr);
+    setLibcallName(RTLIB::LDEXP_PPCF128, nullptr);
+
+    setLibcallName(RTLIB::FREXP_F32, nullptr);
+    setLibcallName(RTLIB::FREXP_F80, nullptr);
+    setLibcallName(RTLIB::FREXP_F128, nullptr);
+    setLibcallName(RTLIB::FREXP_PPCF128, nullptr);
+  }
+
+  if (TT.isAArch64()) {
+    if (TT.isOSMSVCRT()) {
+      // MSVCRT doesn't have powi; fall back to pow
+      setLibcallName(RTLIB::POWI_F32, nullptr);
+      setLibcallName(RTLIB::POWI_F64, nullptr);
+    }
+  }
+
+  // Disable most libcalls on AMDGPU.
+  if (TT.isAMDGPU()) {
+    for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I) {
+      if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
+        setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
+    }
+  }
+
+  // Disable most libcalls on NVPTX.
+  if (TT.isNVPTX()) {
+    for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I)
+      if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
+        setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
+  }
+
+  if (TT.isARM() || TT.isThumb()) {
+    // These libcalls are not available in 32-bit.
+    setLibcallName(RTLIB::SHL_I128, nullptr);
+    setLibcallName(RTLIB::SRL_I128, nullptr);
+    setLibcallName(RTLIB::SRA_I128, nullptr);
+    setLibcallName(RTLIB::MUL_I128, nullptr);
+    setLibcallName(RTLIB::MULO_I64, nullptr);
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+
+    if (TT.isOSMSVCRT()) {
+      // MSVCRT doesn't have powi; fall back to pow
+      setLibcallName(RTLIB::POWI_F32, nullptr);
+      setLibcallName(RTLIB::POWI_F64, nullptr);
+    }
+  }
+
+  if (TT.getArch() == Triple::ArchType::avr) {
+    // Division rtlib functions (not supported), use divmod functions instead
+    setLibcallName(RTLIB::SDIV_I8, nullptr);
+    setLibcallName(RTLIB::SDIV_I16, nullptr);
+    setLibcallName(RTLIB::SDIV_I32, nullptr);
+    setLibcallName(RTLIB::UDIV_I8, nullptr);
+    setLibcallName(RTLIB::UDIV_I16, nullptr);
+    setLibcallName(RTLIB::UDIV_I32, nullptr);
+
+    // Modulus rtlib functions (not supported), use divmod functions instead
+    setLibcallName(RTLIB::SREM_I8, nullptr);
+    setLibcallName(RTLIB::SREM_I16, nullptr);
+    setLibcallName(RTLIB::SREM_I32, nullptr);
+    setLibcallName(RTLIB::UREM_I8, nullptr);
+    setLibcallName(RTLIB::UREM_I16, nullptr);
+    setLibcallName(RTLIB::UREM_I32, nullptr);
+  }
+
+  if (TT.getArch() == Triple::ArchType::hexagon) {
+    // These cause problems when the shift amount is non-constant.
+    setLibcallName(RTLIB::SHL_I128, nullptr);
+    setLibcallName(RTLIB::SRL_I128, nullptr);
+    setLibcallName(RTLIB::SRA_I128, nullptr);
+  }
+
+  if (TT.isLoongArch()) {
+    if (!TT.isLoongArch64()) {
+      // Set libcalls.
+      setLibcallName(RTLIB::MUL_I128, nullptr);
+      // The MULO libcall is not part of libgcc, only compiler-rt.
+      setLibcallName(RTLIB::MULO_I64, nullptr);
+    }
+    // The MULO libcall is not part of libgcc, only compiler-rt.
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+  }
+
+  if (TT.isMIPS32()) {
+    // These libcalls are not available in 32-bit.
+    setLibcallName(RTLIB::SHL_I128, nullptr);
+    setLibcallName(RTLIB::SRL_I128, nullptr);
+    setLibcallName(RTLIB::SRA_I128, nullptr);
+    setLibcallName(RTLIB::MUL_I128, nullptr);
+    setLibcallName(RTLIB::MULO_I64, nullptr);
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+  }
+
+  if (TT.isPPC()) {
+    if (!TT.isPPC64()) {
+      // These libcalls are not available in 32-bit.
+      setLibcallName(RTLIB::SHL_I128, nullptr);
+      setLibcallName(RTLIB::SRL_I128, nullptr);
+      setLibcallName(RTLIB::SRA_I128, nullptr);
+      setLibcallName(RTLIB::MUL_I128, nullptr);
+      setLibcallName(RTLIB::MULO_I64, nullptr);
+    }
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+  }
+
+  if (TT.isRISCV32()) {
+    // These libcalls are not available in 32-bit.
+    setLibcallName(RTLIB::SHL_I128, nullptr);
+    setLibcallName(RTLIB::SRL_I128, nullptr);
+    setLibcallName(RTLIB::SRA_I128, nullptr);
+    setLibcallName(RTLIB::MUL_I128, nullptr);
+    setLibcallName(RTLIB::MULO_I64, nullptr);
+  }
+
+  if (TT.isSPARC()) {
+    if (!TT.isSPARC64()) {
+      // These libcalls are not available in 32-bit.
+      setLibcallName(RTLIB::MULO_I64, nullptr);
+      setLibcallName(RTLIB::MUL_I128, nullptr);
+      setLibcallName(RTLIB::SHL_I128, nullptr);
+      setLibcallName(RTLIB::SRL_I128, nullptr);
+      setLibcallName(RTLIB::SRA_I128, nullptr);
+    }
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+  }
+
+  if (TT.isSystemZ()) {
+    setLibcallName(RTLIB::SRL_I128, nullptr);
+    setLibcallName(RTLIB::SHL_I128, nullptr);
+    setLibcallName(RTLIB::SRA_I128, nullptr);
+  }
+
+  if (TT.isX86()) {
+    if (TT.getArch() == Triple::ArchType::x86) {
+      // These libcalls are not available in 32-bit.
+      setLibcallName(RTLIB::SHL_I128, nullptr);
+      setLibcallName(RTLIB::SRL_I128, nullptr);
+      setLibcallName(RTLIB::SRA_I128, nullptr);
+      setLibcallName(RTLIB::MUL_I128, nullptr);
+      // The MULO libcall is not part of libgcc, only compiler-rt.
+      setLibcallName(RTLIB::MULO_I64, nullptr);
+    }
+
+    // The MULO libcall is not part of libgcc, only compiler-rt.
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+
+    if (TT.isOSMSVCRT()) {
+      // MSVCRT doesn't have powi; fall back to pow
+      setLibcallName(RTLIB::POWI_F32, nullptr);
+      setLibcallName(RTLIB::POWI_F64, nullptr);
+    }
+  }
+}
+
 /// GetFPLibCall - Helper to return the right libcall for the given floating
 /// point type, or UNKNOWN_LIBCALL if there is none.
 RTLIB::Libcall RTLIB::getFPLibCall(EVT VT,
@@ -574,9 +918,41 @@ RTLIB::Libcall RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
   }
 }
 
+/// InitCmpLibcallCCs - Set default comparison libcall CC.
+static void InitCmpLibcallCCs(ISD::CondCode *CCs) {
+  std::fill(CCs, CCs + RTLIB::UNKNOWN_LIBCALL, ISD::SETCC_INVALID);
+  CCs[RTLIB::OEQ_F32] = ISD::SETEQ;
+  CCs[RTLIB::OEQ_F64] = ISD::SETEQ;
+  CCs[RTLIB::OEQ_F128] = ISD::SETEQ;
+  CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
+  CCs[RTLIB::UNE_F32] = ISD::SETNE;
+  CCs[RTLIB::UNE_F64] = ISD::SETNE;
+  CCs[RTLIB::UNE_F128] = ISD::SETNE;
+  CCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
+  CCs[RTLIB::OGE_F32] = ISD::SETGE;
+  CCs[RTLIB::OGE_F64] = ISD::SETGE;
+  CCs[RTLIB::OGE_F128] = ISD::SETGE;
+  CCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
+  CCs[RTLIB::OLT_F32] = ISD::SETLT;
+  CCs[RTLIB::OLT_F64] = ISD::SETLT;
+  CCs[RTLIB::OLT_F128] = ISD::SETLT;
+  CCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
+  CCs[RTLIB::OLE_F32] = ISD::SETLE;
+  CCs[RTLIB::OLE_F64] = ISD::SETLE;
+  CCs[RTLIB::OLE_F128] = ISD::SETLE;
+  CCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
+  CCs[RTLIB::OGT_F32] = ISD::SETGT;
+  CCs[RTLIB::OGT_F64] = ISD::SETGT;
+  CCs[RTLIB::OGT_F128] = ISD::SETGT;
+  CCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
+  CCs[RTLIB::UO_F32] = ISD::SETNE;
+  CCs[RTLIB::UO_F64] = ISD::SETNE;
+  CCs[RTLIB::UO_F128] = ISD::SETNE;
+  CCs[RTLIB::UO_PPCF128] = ISD::SETNE;
+}
+
 /// NOTE: The TargetMachine owns TLOF.
-TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm)
-    : TM(tm), Libcalls(TM.getTargetTriple()) {
+TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
   initActions();
 
   // Perform these initializations only once.
@@ -608,6 +984,11 @@ TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm)
 
   MinCmpXchgSizeInBits = 0;
   SupportsUnalignedAtomics = false;
+
+  std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr);
+
+  InitLibcalls(TM.getTargetTriple());
+  InitCmpLibcallCCs(CmpLibcallCCs);
 }
 
 void TargetLoweringBase::initActions() {
diff --git a/llvm/lib/IR/CMakeLists.txt b/llvm/lib/IR/CMakeLists.txt
index 8bf199f0f44c9..4eb08f648760d 100644
--- a/llvm/lib/IR/CMakeLists.txt
+++ b/llvm/lib/IR/CMakeLists.txt
@@ -73,7 +73,6 @@ add_llvm_component_library(LLVMCore
   VectorBuilder.cpp
   Verifier.cpp
   VFABIDemangler.cpp
-  RuntimeLibcalls.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/IR
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
deleted file mode 100644
index de3db557d8b50..0000000000000
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ /dev/null
@@ -1,379 +0,0 @@
-//===- RuntimeLibcalls.cpp - Interface for runtime libcalls -----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/IR/RuntimeLibcalls.h"
-
-using namespace llvm;
-using namespace RTLIB;
-
-/// Set default libcall names. If a target wants to opt-out of a libcall it
-/// should be placed here.
-void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
-  std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames),
-            nullptr);
-
-#define HANDLE_LIBCALL(code, name) setLibcallName(RTLIB::code, name);
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-
-  // Initialize calling conventions to their default.
-  for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC)
-    setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C);
-
-  // Use the f128 variants of math functions on x86_64
-  if (TT.getArch() == Triple::ArchType::x86_64 && TT.isGNUEnvironment()) {
-    setLibcallName(RTLIB::REM_F128, "fmodf128");
-    setLibcallName(RTLIB::FMA_F128, "fmaf128");
-    setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
-    setLibcallName(RTLIB::CBRT_F128, "cbrtf128");
-    setLibcallName(RTLIB::LOG_F128, "logf128");
-    setLibcallName(RTLIB::LOG_FINITE_F128, "__logf128_finite");
-    setLibcallName(RTLIB::LOG2_F128, "log2f128");
-    setLibcallName(RTLIB::LOG2_FINITE_F128, "__log2f128_finite");
-    setLibcallName(RTLIB::LOG10_F128, "log10f128");
-    setLibcallName(RTLIB::LOG10_FINITE_F128, "__log10f128_finite");
-    setLibcallName(RTLIB::EXP_F128, "expf128");
-    setLibcallName(RTLIB::EXP_FINITE_F128, "__expf128_finite");
-    setLibcallName(RTLIB::EXP2_F128, "exp2f128");
-    setLibcallName(RTLIB::EXP2_FINITE_F128, "__exp2f128_finite");
-    setLibcallName(RTLIB::EXP10_F128, "exp10f128");
-    setLibcallName(RTLIB::SIN_F128, "sinf128");
-    setLibcallName(RTLIB::COS_F128, "cosf128");
-    setLibcallName(RTLIB::TAN_F128, "tanf128");
-    setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
-    setLibcallName(RTLIB::ASIN_F128, "asinf128");
-    setLibcallName(RTLIB::ACOS_F128, "acosf128");
-    setLibcallName(RTLIB::ATAN_F128, "atanf128");
-    setLibcallName(RTLIB::SINH_F128, "sinhf128");
-    setLibcallName(RTLIB::COSH_F128, "coshf128");
-    setLibcallName(RTLIB::TANH_F128, "tanhf128");
-    setLibcallName(RTLIB::POW_F128, "powf128");
-    setLibcallName(RTLIB::POW_FINITE_F128, "__powf128_finite");
-    setLibcallName(RTLIB::CEIL_F128, "ceilf128");
-    setLibcallName(RTLIB::TRUNC_F128, "truncf128");
-    setLibcallName(RTLIB::RINT_F128, "rintf128");
-    setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
-    setLibcallName(RTLIB::ROUND_F128, "roundf128");
-    setLibcallName(RTLIB::ROUNDEVEN_F128, "roundevenf128");
-    setLibcallName(RTLIB::FLOOR_F128, "floorf128");
-    setLibcallName(RTLIB::COPYSIGN_F128, "copysignf128");
-    setLibcallName(RTLIB::FMIN_F128, "fminf128");
-    setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
-    setLibcallName(RTLIB::LROUND_F128, "lroundf128");
-    setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
-    setLibcallName(RTLIB::LRINT_F128, "lrintf128");
-    setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
-    setLibcallName(RTLIB::LDEXP_F128, "ldexpf128");
-    setLibcallName(RTLIB::FREXP_F128, "frexpf128");
-  }
-
-  // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
-  if (TT.isPPC()) {
-    setLibcallName(RTLIB::ADD_F128, "__addkf3");
-    setLibcallName(RTLIB::SUB_F128, "__subkf3");
-    setLibcallName(RTLIB::MUL_F128, "__mulkf3");
-    setLibcallName(RTLIB::DIV_F128, "__divkf3");
-    setLibcallName(RTLIB::POWI_F128, "__powikf2");
-    setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
-    setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
-    setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
-    setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
-    setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
-    setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
-    setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti");
-    setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
-    setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
-    setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti");
-    setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
-    setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
-    setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf");
-    setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
-    setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
-    setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf");
-    setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
-    setLibcallName(RTLIB::UNE_F128, "__nekf2");
-    setLibcallName(RTLIB::OGE_F128, "__gekf2");
-    setLibcallName(RTLIB::OLT_F128, "__ltkf2");
-    setLibcallName(RTLIB::OLE_F128, "__lekf2");
-    setLibcallName(RTLIB::OGT_F128, "__gtkf2");
-    setLibcallName(RTLIB::UO_F128, "__unordkf2");
-  }
-
-  // A few names are different on particular architectures or environments.
-  if (TT.isOSDarwin()) {
-    // For f16/f32 conversions, Darwin uses the standard naming scheme,
-    // instead of the gnueabi-style __gnu_*_ieee.
-    // FIXME: What about other targets?
-    setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
-    setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
-
-    // Some darwins have an optimized __bzero/bzero function.
-    switch (TT.getArch()) {
-    case Triple::x86:
-    case Triple::x86_64:
-      if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6))
-        setLibcallName(RTLIB::BZERO, "__bzero");
-      break;
-    case Triple::aarch64:
-    case Triple::aarch64_32:
-      setLibcallName(RTLIB::BZERO, "bzero");
-      break;
-    default:
-      break;
-    }
-
-    if (darwinHasSinCos(TT)) {
-      setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret");
-      setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret");
-      if (TT.isWatchABI()) {
-        setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
-                              CallingConv::ARM_AAPCS_VFP);
-        setLibcallCallingConv(RTLIB::SINCOS_STRET_F64,
-                              CallingConv::ARM_AAPCS_VFP);
-      }
-    }
-
-    switch (TT.getOS()) {
-    case Triple::MacOSX:
-      if (TT.isMacOSXVersionLT(10, 9)) {
-        setLibcallName(RTLIB::EXP10_F32, nullptr);
-        setLibcallName(RTLIB::EXP10_F64, nullptr);
-      } else {
-        setLibcallName(RTLIB::EXP10_F32, "__exp10f");
-        setLibcallName(RTLIB::EXP10_F64, "__exp10");
-      }
-      break;
-    case Triple::IOS:
-      if (TT.isOSVersionLT(7, 0)) {
-        setLibcallName(RTLIB::EXP10_F32, nullptr);
-        setLibcallName(RTLIB::EXP10_F64, nullptr);
-        break;
-      }
-      [[fallthrough]];
-    case Triple::TvOS:
-    case Triple::WatchOS:
-    case Triple::XROS:
-      setLibcallName(RTLIB::EXP10_F32, "__exp10f");
-      setLibcallName(RTLIB::EXP10_F64, "__exp10");
-      break;
-    default:
-      break;
-    }
-  } else {
-    setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
-    setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
-  }
-
-  if (TT.isGNUEnvironment() || TT.isOSFuchsia() ||
-      (TT.isAndroid() && !TT.isAndroidVersionLT(9))) {
-    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
-    setLibcallName(RTLIB::SINCOS_F64, "sincos");
-    setLibcallName(RTLIB::SINCOS_F80, "sincosl");
-    setLibcallName(RTLIB::SINCOS_F128, "sincosl");
-    setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl");
-  }
-
-  if (TT.isPS()) {
-    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
-    setLibcallName(RTLIB::SINCOS_F64, "sincos");
-  }
-
-  if (TT.isOSOpenBSD()) {
-    setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr);
-  }
-
-  if (TT.isOSWindows() && !TT.isOSCygMing()) {
-    setLibcallName(RTLIB::LDEXP_F32, nullptr);
-    setLibcallName(RTLIB::LDEXP_F80, nullptr);
-    setLibcallName(RTLIB::LDEXP_F128, nullptr);
-    setLibcallName(RTLIB::LDEXP_PPCF128, nullptr);
-
-    setLibcallName(RTLIB::FREXP_F32, nullptr);
-    setLibcallName(RTLIB::FREXP_F80, nullptr);
-    setLibcallName(RTLIB::FREXP_F128, nullptr);
-    setLibcallName(RTLIB::FREXP_PPCF128, nullptr);
-  }
-
-  if (TT.isAArch64()) {
-    if (TT.isOSMSVCRT()) {
-      // MSVCRT doesn't have powi; fall back to pow
-      setLibcallName(RTLIB::POWI_F32, nullptr);
-      setLibcallName(RTLIB::POWI_F64, nullptr);
-    }
-  }
-
-  // Disable most libcalls on AMDGPU.
-  if (TT.isAMDGPU()) {
-    for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I) {
-      if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
-        setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
-    }
-  }
-
-  // Disable most libcalls on NVPTX.
-  if (TT.isNVPTX()) {
-    for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I)
-      if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
-        setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
-  }
-
-  if (TT.isARM() || TT.isThumb()) {
-    // These libcalls are not available in 32-bit.
-    setLibcallName(RTLIB::SHL_I128, nullptr);
-    setLibcallName(RTLIB::SRL_I128, nullptr);
-    setLibcallName(RTLIB::SRA_I128, nullptr);
-    setLibcallName(RTLIB::MUL_I128, nullptr);
-    setLibcallName(RTLIB::MULO_I64, nullptr);
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-
-    if (TT.isOSMSVCRT()) {
-      // MSVCRT doesn't have powi; fall back to pow
-      setLibcallName(RTLIB::POWI_F32, nullptr);
-      setLibcallName(RTLIB::POWI_F64, nullptr);
-    }
-  }
-
-  if (TT.getArch() == Triple::ArchType::avr) {
-    // Division rtlib functions (not supported), use divmod functions instead
-    setLibcallName(RTLIB::SDIV_I8, nullptr);
-    setLibcallName(RTLIB::SDIV_I16, nullptr);
-    setLibcallName(RTLIB::SDIV_I32, nullptr);
-    setLibcallName(RTLIB::UDIV_I8, nullptr);
-    setLibcallName(RTLIB::UDIV_I16, nullptr);
-    setLibcallName(RTLIB::UDIV_I32, nullptr);
-
-    // Modulus rtlib functions (not supported), use divmod functions instead
-    setLibcallName(RTLIB::SREM_I8, nullptr);
-    setLibcallName(RTLIB::SREM_I16, nullptr);
-    setLibcallName(RTLIB::SREM_I32, nullptr);
-    setLibcallName(RTLIB::UREM_I8, nullptr);
-    setLibcallName(RTLIB::UREM_I16, nullptr);
-    setLibcallName(RTLIB::UREM_I32, nullptr);
-  }
-
-  if (TT.getArch() == Triple::ArchType::hexagon) {
-    // These cause problems when the shift amount is non-constant.
-    setLibcallName(RTLIB::SHL_I128, nullptr);
-    setLibcallName(RTLIB::SRL_I128, nullptr);
-    setLibcallName(RTLIB::SRA_I128, nullptr);
-  }
-
-  if (TT.isLoongArch()) {
-    if (!TT.isLoongArch64()) {
-      // Set libcalls.
-      setLibcallName(RTLIB::MUL_I128, nullptr);
-      // The MULO libcall is not part of libgcc, only compiler-rt.
-      setLibcallName(RTLIB::MULO_I64, nullptr);
-    }
-    // The MULO libcall is not part of libgcc, only compiler-rt.
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-  }
-
-  if (TT.isMIPS32()) {
-    // These libcalls are not available in 32-bit.
-    setLibcallName(RTLIB::SHL_I128, nullptr);
-    setLibcallName(RTLIB::SRL_I128, nullptr);
-    setLibcallName(RTLIB::SRA_I128, nullptr);
-    setLibcallName(RTLIB::MUL_I128, nullptr);
-    setLibcallName(RTLIB::MULO_I64, nullptr);
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-  }
-
-  if (TT.isPPC()) {
-    if (!TT.isPPC64()) {
-      // These libcalls are not available in 32-bit.
-      setLibcallName(RTLIB::SHL_I128, nullptr);
-      setLibcallName(RTLIB::SRL_I128, nullptr);
-      setLibcallName(RTLIB::SRA_I128, nullptr);
-      setLibcallName(RTLIB::MUL_I128, nullptr);
-      setLibcallName(RTLIB::MULO_I64, nullptr);
-    }
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-  }
-
-  if (TT.isRISCV32()) {
-    // These libcalls are not available in 32-bit.
-    setLibcallName(RTLIB::SHL_I128, nullptr);
-    setLibcallName(RTLIB::SRL_I128, nullptr);
-    setLibcallName(RTLIB::SRA_I128, nullptr);
-    setLibcallName(RTLIB::MUL_I128, nullptr);
-    setLibcallName(RTLIB::MULO_I64, nullptr);
-  }
-
-  if (TT.isSPARC()) {
-    if (!TT.isSPARC64()) {
-      // These libcalls are not available in 32-bit.
-      setLibcallName(RTLIB::MULO_I64, nullptr);
-      setLibcallName(RTLIB::MUL_I128, nullptr);
-      setLibcallName(RTLIB::SHL_I128, nullptr);
-      setLibcallName(RTLIB::SRL_I128, nullptr);
-      setLibcallName(RTLIB::SRA_I128, nullptr);
-    }
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-  }
-
-  if (TT.isSystemZ()) {
-    setLibcallName(RTLIB::SRL_I128, nullptr);
-    setLibcallName(RTLIB::SHL_I128, nullptr);
-    setLibcallName(RTLIB::SRA_I128, nullptr);
-  }
-
-  if (TT.isX86()) {
-    if (TT.getArch() == Triple::ArchType::x86) {
-      // These libcalls are not available in 32-bit.
-      setLibcallName(RTLIB::SHL_I128, nullptr);
-      setLibcallName(RTLIB::SRL_I128, nullptr);
-      setLibcallName(RTLIB::SRA_I128, nullptr);
-      setLibcallName(RTLIB::MUL_I128, nullptr);
-      // The MULO libcall is not part of libgcc, only compiler-rt.
-      setLibcallName(RTLIB::MULO_I64, nullptr);
-    }
-
-    // The MULO libcall is not part of libgcc, only compiler-rt.
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-
-    if (TT.isOSMSVCRT()) {
-      // MSVCRT doesn't have powi; fall back to pow
-      setLibcallName(RTLIB::POWI_F32, nullptr);
-      setLibcallName(RTLIB::POWI_F64, nullptr);
-    }
-  }
-}
-
-void RuntimeLibcallsInfo::initCmpLibcallCCs() {
-  std::fill(CmpLibcallCCs, CmpLibcallCCs + RTLIB::UNKNOWN_LIBCALL,
-            ISD::SETCC_INVALID);
-  CmpLibcallCCs[RTLIB::OEQ_F32] = ISD::SETEQ;
-  CmpLibcallCCs[RTLIB::OEQ_F64] = ISD::SETEQ;
-  CmpLibcallCCs[RTLIB::OEQ_F128] = ISD::SETEQ;
-  CmpLibcallCCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
-  CmpLibcallCCs[RTLIB::UNE_F32] = ISD::SETNE;
-  CmpLibcallCCs[RTLIB::UNE_F64] = ISD::SETNE;
-  CmpLibcallCCs[RTLIB::UNE_F128] = ISD::SETNE;
-  CmpLibcallCCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
-  CmpLibcallCCs[RTLIB::OGE_F32] = ISD::SETGE;
-  CmpLibcallCCs[RTLIB::OGE_F64] = ISD::SETGE;
-  CmpLibcallCCs[RTLIB::OGE_F128] = ISD::SETGE;
-  CmpLibcallCCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
-  CmpLibcallCCs[RTLIB::OLT_F32] = ISD::SETLT;
-  CmpLibcallCCs[RTLIB::OLT_F64] = ISD::SETLT;
-  CmpLibcallCCs[RTLIB::OLT_F128] = ISD::SETLT;
-  CmpLibcallCCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
-  CmpLibcallCCs[RTLIB::OLE_F32] = ISD::SETLE;
-  CmpLibcallCCs[RTLIB::OLE_F64] = ISD::SETLE;
-  CmpLibcallCCs[RTLIB::OLE_F128] = ISD::SETLE;
-  CmpLibcallCCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
-  CmpLibcallCCs[RTLIB::OGT_F32] = ISD::SETGT;
-  CmpLibcallCCs[RTLIB::OGT_F64] = ISD::SETGT;
-  CmpLibcallCCs[RTLIB::OGT_F128] = ISD::SETGT;
-  CmpLibcallCCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
-  CmpLibcallCCs[RTLIB::UO_F32] = ISD::SETNE;
-  CmpLibcallCCs[RTLIB::UO_F64] = ISD::SETNE;
-  CmpLibcallCCs[RTLIB::UO_F128] = ISD::SETNE;
-  CmpLibcallCCs[RTLIB::UO_PPCF128] = ISD::SETNE;
-}
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index d303f228aa72c..4ffacbb9f0ffd 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -30,7 +30,6 @@
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/IR/RuntimeLibcalls.h"
 #include "llvm/LTO/LTOBackend.h"
 #include "llvm/LTO/SummaryBasedOptimizations.h"
 #include "llvm/Linker/IRMover.h"
@@ -1358,13 +1357,14 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) {
   return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 }
 
-SmallVector<const char *> LTO::getRuntimeLibcallSymbols(const Triple &TT) {
-  RTLIB::RuntimeLibcallsInfo Libcalls(TT);
+static const char *libcallRoutineNames[] = {
+#define HANDLE_LIBCALL(code, name) name,
+#include "llvm/IR/RuntimeLibcalls.def"
+#undef HANDLE_LIBCALL
+};
 
-  SmallVector<const char *> LibcallSymbols;
-  copy_if(Libcalls.getLibcallNames(), std::back_inserter(LibcallSymbols),
-          [](const char *Name) { return Name; });
-  return LibcallSymbols;
+ArrayRef<const char*> LTO::getRuntimeLibcallSymbols() {
+  return ArrayRef(libcallRoutineNames);
 }
 
 /// This class defines the interface to the ThinLTO backend.
diff --git a/llvm/lib/Object/IRSymtab.cpp b/llvm/lib/Object/IRSymtab.cpp
index 2a2b235461a55..7e29da4d5e948 100644
--- a/llvm/lib/Object/IRSymtab.cpp
+++ b/llvm/lib/Object/IRSymtab.cpp
@@ -22,7 +22,6 @@
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
-#include "llvm/IR/RuntimeLibcalls.h"
 #include "llvm/MC/StringTableBuilder.h"
 #include "llvm/Object/ModuleSymbolTable.h"
 #include "llvm/Object/SymbolicFile.h"
@@ -47,6 +46,9 @@ static cl::opt<bool> DisableBitcodeVersionUpgrade(
     cl::desc("Disable automatic bitcode upgrade for version mismatch"));
 
 static const char *PreservedSymbols[] = {
+#define HANDLE_LIBCALL(code, name) name,
+#include "llvm/IR/RuntimeLibcalls.def"
+#undef HANDLE_LIBCALL
     // There are global variables, so put it here instead of in
     // RuntimeLibcalls.def.
     // TODO: Are there similar such variables?
@@ -213,16 +215,9 @@ Expected<int> Builder::getComdatIndex(const Comdat *C, const Module *M) {
   return P.first->second;
 }
 
-static DenseSet<StringRef> buildPreservedSymbolsSet(const Triple &TT) {
-  DenseSet<StringRef> PreservedSymbolSet(std::begin(PreservedSymbols),
-                                         std::end(PreservedSymbols));
-
-  RTLIB::RuntimeLibcallsInfo Libcalls(TT);
-  for (const char *Name : Libcalls.getLibcallNames()) {
-    if (Name)
-      PreservedSymbolSet.insert(Name);
-  }
-  return PreservedSymbolSet;
+static DenseSet<StringRef> buildPreservedSymbolsSet() {
+  return DenseSet<StringRef>(std::begin(PreservedSymbols),
+                             std::end(PreservedSymbols));
 }
 
 Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
@@ -281,8 +276,7 @@ Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
   setStr(Sym.IRName, GV->getName());
 
   static const DenseSet<StringRef> PreservedSymbolsSet =
-      buildPreservedSymbolsSet(
-          llvm::Triple(GV->getParent()->getTargetTriple()));
+      buildPreservedSymbolsSet();
   bool IsPreservedSymbol = PreservedSymbolsSet.contains(GV->getName());
 
   if (Used.count(GV) || IsPreservedSymbol)
diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index 527496f1a6374..e3c5a143b2889 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -35,7 +35,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/CodeGenTypes/MachineValueType.h"
 #include "llvm/IR/Argument.h"
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 84de1ee8f8923..dbaf31166229b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -48,7 +48,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
diff --git a/llvm/lib/Target/ARM/ARMFastISel.cpp b/llvm/lib/Target/ARM/ARMFastISel.cpp
index 1cc21da51d1e6..61d2928fe6d41 100644
--- a/llvm/lib/Target/ARM/ARMFastISel.cpp
+++ b/llvm/lib/Target/ARM/ARMFastISel.cpp
@@ -40,7 +40,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 75d16a42d0205..2683b5741d459 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -53,7 +53,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
diff --git a/llvm/lib/Target/ARM/ARMLegalizerInfo.h b/llvm/lib/Target/ARM/ARMLegalizerInfo.h
index 9e10638233e6d..d6ce4eb1055b4 100644
--- a/llvm/lib/Target/ARM/ARMLegalizerInfo.h
+++ b/llvm/lib/Target/ARM/ARMLegalizerInfo.h
@@ -16,7 +16,7 @@
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/IR/Instructions.h"
 
 namespace llvm {
diff --git a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
index 275b1c0f8dc01..ffa8b50493510 100644
--- a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
+++ b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
@@ -14,7 +14,7 @@
 #define LLVM_LIB_TARGET_ARM_ARMSELECTIONDAGINFO_H
 
 #include "MCTargetDesc/ARMAddressingModes.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
 
 namespace llvm {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 7aeaebc584c64..918fc127762fc 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -27,7 +27,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
 #include "llvm/CodeGen/ValueTypes.h"
@@ -39,12 +39,12 @@
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsHexagon.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index f6763a35cc0d5..c7848356682d0 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -27,7 +27,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 6072e5e244263..d5d047b3ee1a5 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -22,7 +22,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicsLoongArch.h"
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 0f2047fcac640..8cb64a0e6f4bf 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -40,7 +40,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 898d1f80d0564..564c0f74d975d 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -48,7 +48,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
index ba3ab5164af26..53619eaa6f562 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
@@ -21,7 +21,7 @@
 #include "WebAssemblyRuntimeLibcallSignatures.h"
 #include "WebAssemblySubtarget.h"
 #include "WebAssemblyUtilities.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
index 966b84aa4951b..ff6515d5bf4e6 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
@@ -16,7 +16,7 @@
 
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/CodeGen/RuntimeLibcallUtil.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
 
 namespace llvm {
 
diff --git a/llvm/tools/lto/lto.cpp b/llvm/tools/lto/lto.cpp
index d68cff839604f..ece6dd0f10830 100644
--- a/llvm/tools/lto/lto.cpp
+++ b/llvm/tools/lto/lto.cpp
@@ -691,7 +691,7 @@ extern const char *lto_input_get_dependent_library(lto_input_t input,
 }
 
 extern const char *const *lto_runtime_lib_symbols_list(size_t *size) {
-  auto symbols = lto::LTO::getRuntimeLibcallSymbols(Triple());
+  auto symbols = lto::LTO::getRuntimeLibcallSymbols();
   *size = symbols.size();
   return symbols.data();
 }

From e2965fe512162baeb5bdb66f1a675a47c7fab5ca Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Sat, 20 Jul 2024 12:10:09 +0900
Subject: [PATCH 260/486] Revert "[LLVM][LTO] Add missing dependency"

This reverts commit bb604ae9b8adbc0a0852eb68545eff685902f41b.
(llvmorg-19-init-17804-gbb604ae9b8ad)
See #99610
---
 llvm/lib/IR/CMakeLists.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/lib/IR/CMakeLists.txt b/llvm/lib/IR/CMakeLists.txt
index 4eb08f648760d..6b3224e22ffb6 100644
--- a/llvm/lib/IR/CMakeLists.txt
+++ b/llvm/lib/IR/CMakeLists.txt
@@ -81,7 +81,6 @@ add_llvm_component_library(LLVMCore
   ${LLVM_PTHREAD_LIB}
 
   DEPENDS
-  vt_gen
   intrinsics_gen
 
   LINK_COMPONENTS

From 168ecd706904d6ce221dc5107da92c56aea7c8e9 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Sat, 20 Jul 2024 12:10:56 +0900
Subject: [PATCH 261/486] Revert "[bazel] Fix llvm:Core build (#99054)"

This reverts commit 5b54f36fb607d21c18f9eb56dcf481a9841dee8e.
(llvmorg-19-init-17774-g5b54f36fb607)
See #99610
---
 utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index ae17746c72882..64d36c7b7f664 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -947,14 +947,7 @@ cc_library(
     ]) + [
         # To avoid a dependency cycle.
         "include/llvm/Analysis/IVDescriptors.h",
-        "include/llvm/CodeGen/GenVT.inc",
-    ] + glob(
-        # To avoid a dependency cycle.
-        [
-            "include/llvm/CodeGen/**/*.h",
-            "include/llvm/CodeGenTypes/**/*.h",
-        ],
-    ),
+    ],
     hdrs = glob(
         [
             "include/llvm/*.h",

From 5893b1e297f3a333b30a9d32d0909b77a9fcd31c Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Sat, 20 Jul 2024 12:09:30 +0900
Subject: [PATCH 262/486] Reformat

---
 llvm/include/llvm/CodeGen/RuntimeLibcalls.h   | 172 +++++++++---------
 llvm/include/llvm/LTO/LTO.h                   |   2 +-
 llvm/lib/CodeGen/TargetLoweringBase.cpp       |   6 +-
 llvm/lib/LTO/LTO.cpp                          |   2 +-
 .../Target/Hexagon/HexagonISelLowering.cpp    |   2 +-
 5 files changed, 90 insertions(+), 94 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/RuntimeLibcalls.h b/llvm/include/llvm/CodeGen/RuntimeLibcalls.h
index 3a407c4a4d940..f3170418f3364 100644
--- a/llvm/include/llvm/CodeGen/RuntimeLibcalls.h
+++ b/llvm/include/llvm/CodeGen/RuntimeLibcalls.h
@@ -19,95 +19,91 @@
 
 namespace llvm {
 namespace RTLIB {
-  /// RTLIB::Libcall enum - This enum defines all of the runtime library calls
-  /// the backend can emit.  The various long double types cannot be merged,
-  /// because 80-bit library functions use "xf" and 128-bit use "tf".
-  ///
-  /// When adding PPCF128 functions here, note that their names generally need
-  /// to be overridden for Darwin with the xxx$LDBL128 form.  See
-  /// PPCISelLowering.cpp.
-  ///
-  enum Libcall {
+/// RTLIB::Libcall enum - This enum defines all of the runtime library calls
+/// the backend can emit.  The various long double types cannot be merged,
+/// because 80-bit library functions use "xf" and 128-bit use "tf".
+///
+/// When adding PPCF128 functions here, note that their names generally need
+/// to be overridden for Darwin with the xxx$LDBL128 form.  See
+/// PPCISelLowering.cpp.
+///
+enum Libcall {
 #define HANDLE_LIBCALL(code, name) code,
-    #include "llvm/IR/RuntimeLibcalls.def"
+#include "llvm/IR/RuntimeLibcalls.def"
 #undef HANDLE_LIBCALL
-  };
-
-  /// GetFPLibCall - Helper to return the right libcall for the given floating
-  /// point type, or UNKNOWN_LIBCALL if there is none.
-  Libcall getFPLibCall(EVT VT,
-                       Libcall Call_F32,
-                       Libcall Call_F64,
-                       Libcall Call_F80,
-                       Libcall Call_F128,
-                       Libcall Call_PPCF128);
-
-  /// getFPEXT - Return the FPEXT_*_* value for the given types, or
-  /// UNKNOWN_LIBCALL if there is none.
-  Libcall getFPEXT(EVT OpVT, EVT RetVT);
-
-  /// getFPROUND - Return the FPROUND_*_* value for the given types, or
-  /// UNKNOWN_LIBCALL if there is none.
-  Libcall getFPROUND(EVT OpVT, EVT RetVT);
-
-  /// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
-  /// UNKNOWN_LIBCALL if there is none.
-  Libcall getFPTOSINT(EVT OpVT, EVT RetVT);
-
-  /// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
-  /// UNKNOWN_LIBCALL if there is none.
-  Libcall getFPTOUINT(EVT OpVT, EVT RetVT);
-
-  /// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
-  /// UNKNOWN_LIBCALL if there is none.
-  Libcall getSINTTOFP(EVT OpVT, EVT RetVT);
-
-  /// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
-  /// UNKNOWN_LIBCALL if there is none.
-  Libcall getUINTTOFP(EVT OpVT, EVT RetVT);
-
-  /// getPOWI - Return the POWI_* value for the given types, or
-  /// UNKNOWN_LIBCALL if there is none.
-  Libcall getPOWI(EVT RetVT);
-
-  /// getLDEXP - Return the LDEXP_* value for the given types, or
-  /// UNKNOWN_LIBCALL if there is none.
-  Libcall getLDEXP(EVT RetVT);
-
-  /// getFREXP - Return the FREXP_* value for the given types, or
-  /// UNKNOWN_LIBCALL if there is none.
-  Libcall getFREXP(EVT RetVT);
-
-  /// Return the SYNC_FETCH_AND_* value for the given opcode and type, or
-  /// UNKNOWN_LIBCALL if there is none.
-  Libcall getSYNC(unsigned Opc, MVT VT);
-
-  /// Return the outline atomics value for the given atomic ordering, access
-  /// size and set of libcalls for a given atomic, or UNKNOWN_LIBCALL if there
-  /// is none.
-  Libcall getOutlineAtomicHelper(const Libcall (&LC)[5][4],
-                                 AtomicOrdering Order, uint64_t MemSize);
-
-  /// Return the outline atomics value for the given opcode, atomic ordering
-  /// and type, or UNKNOWN_LIBCALL if there is none.
-  Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT);
-
-  /// getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return
-  /// MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
-  /// UNKNOW_LIBCALL if there is none.
-  Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
-
-  /// getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return
-  /// MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
-  /// UNKNOW_LIBCALL if there is none.
-  Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
-
-  /// getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return
-  /// MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
-  /// UNKNOW_LIBCALL if there is none.
-  Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
-
-}
-}
+};
+
+/// GetFPLibCall - Helper to return the right libcall for the given floating
+/// point type, or UNKNOWN_LIBCALL if there is none.
+Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64,
+                     Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128);
+
+/// getFPEXT - Return the FPEXT_*_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getFPEXT(EVT OpVT, EVT RetVT);
+
+/// getFPROUND - Return the FPROUND_*_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getFPROUND(EVT OpVT, EVT RetVT);
+
+/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getFPTOSINT(EVT OpVT, EVT RetVT);
+
+/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getFPTOUINT(EVT OpVT, EVT RetVT);
+
+/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getSINTTOFP(EVT OpVT, EVT RetVT);
+
+/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getUINTTOFP(EVT OpVT, EVT RetVT);
+
+/// getPOWI - Return the POWI_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getPOWI(EVT RetVT);
+
+/// getLDEXP - Return the LDEXP_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getLDEXP(EVT RetVT);
+
+/// getFREXP - Return the FREXP_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getFREXP(EVT RetVT);
+
+/// Return the SYNC_FETCH_AND_* value for the given opcode and type, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getSYNC(unsigned Opc, MVT VT);
+
+/// Return the outline atomics value for the given atomic ordering, access
+/// size and set of libcalls for a given atomic, or UNKNOWN_LIBCALL if there
+/// is none.
+Libcall getOutlineAtomicHelper(const Libcall (&LC)[5][4], AtomicOrdering Order,
+                               uint64_t MemSize);
+
+/// Return the outline atomics value for the given opcode, atomic ordering
+/// and type, or UNKNOWN_LIBCALL if there is none.
+Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT);
+
+/// getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return
+/// MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
+/// UNKNOW_LIBCALL if there is none.
+Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
+
+/// getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return
+/// MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
+/// UNKNOW_LIBCALL if there is none.
+Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
+
+/// getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return
+/// MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
+/// UNKNOW_LIBCALL if there is none.
+Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
+
+} // namespace RTLIB
+} // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index 94996ae89e35d..63c8435838f27 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -301,7 +301,7 @@ class LTO {
 
   /// Static method that returns a list of libcall symbols that can be generated
   /// by LTO but might not be visible from bitcode symbol table.
-  static ArrayRef<const char*> getRuntimeLibcallSymbols();
+  static ArrayRef<const char *> getRuntimeLibcallSymbols();
 
 private:
   Config Conf;
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index f0fa6aa772e37..914c67abdd617 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -114,8 +114,7 @@ static bool darwinHasSinCos(const Triple &TT) {
 }
 
 void TargetLoweringBase::InitLibcalls(const Triple &TT) {
-#define HANDLE_LIBCALL(code, name) \
-  setLibcallName(RTLIB::code, name);
+#define HANDLE_LIBCALL(code, name) setLibcallName(RTLIB::code, name);
 #include "llvm/IR/RuntimeLibcalls.def"
 #undef HANDLE_LIBCALL
   // Initialize calling conventions to their default.
@@ -985,7 +984,8 @@ TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
   MinCmpXchgSizeInBits = 0;
   SupportsUnalignedAtomics = false;
 
-  std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr);
+  std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames),
+            nullptr);
 
   InitLibcalls(TM.getTargetTriple());
   InitCmpLibcallCCs(CmpLibcallCCs);
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 4ffacbb9f0ffd..94db6660e0ba5 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1363,7 +1363,7 @@ static const char *libcallRoutineNames[] = {
 #undef HANDLE_LIBCALL
 };
 
-ArrayRef<const char*> LTO::getRuntimeLibcallSymbols() {
+ArrayRef<const char *> LTO::getRuntimeLibcallSymbols() {
   return ArrayRef(libcallRoutineNames);
 }
 
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 918fc127762fc..84defa8d7b91b 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -39,12 +39,12 @@
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsHexagon.h"
-#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"

From 72d8c2737bb557af9d0c735b9fa30b1b03485627 Mon Sep 17 00:00:00 2001
From: Haowei <haowei@google.com>
Date: Fri, 19 Jul 2024 21:42:57 -0700
Subject: [PATCH 263/486] [Fuchsia] Remove linker flags from stage2 pass
 through (#99722)

This patch removes CMAKE_XXX_LINKER_FLAGS from list of flags that passed
through to stage2 build.
---
 clang/cmake/caches/Fuchsia.cmake | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/clang/cmake/caches/Fuchsia.cmake b/clang/cmake/caches/Fuchsia.cmake
index 4d3af3ad3f403..2d2dcb9ae6798 100644
--- a/clang/cmake/caches/Fuchsia.cmake
+++ b/clang/cmake/caches/Fuchsia.cmake
@@ -67,9 +67,6 @@ set(_FUCHSIA_BOOTSTRAP_PASSTHROUGH
   SWIG_EXECUTABLE
   CMAKE_FIND_PACKAGE_PREFER_CONFIG
   CMAKE_SYSROOT
-  CMAKE_MODULE_LINKER_FLAGS
-  CMAKE_SHARED_LINKER_FLAGS
-  CMAKE_EXE_LINKER_FLAGS
   LLVM_WINSYSROOT
   LLVM_VFSOVERLAY
 )

From 672cc8bce0a9306b07680b340617b8afd862adf5 Mon Sep 17 00:00:00 2001
From: darkbuck <michael.hliao@gmail.com>
Date: Sat, 20 Jul 2024 02:06:38 -0400
Subject: [PATCH 264/486] [GlobalISel] Allow customizing instruction-select
 pass

- Allow customizing instruction-select pass to add additional analysis
  dependencies.

Reviewers: aemerson, arsenm, aeubanks

Reviewed By: arsenm

Pull Request: https://github.com/llvm/llvm-project/pull/95724
---
 .../llvm/CodeGen/GlobalISel/InstructionSelect.h        |  4 ++--
 llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp      | 10 ++--------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h
index cada7f30072e2..8017f09aa3c8b 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h
@@ -49,8 +49,8 @@ class InstructionSelect : public MachineFunctionPass {
         MachineFunctionProperties::Property::Selected);
   }
 
-  InstructionSelect(CodeGenOptLevel OL);
-  InstructionSelect();
+  InstructionSelect(CodeGenOptLevel OL = CodeGenOptLevel::Default,
+                    char &PassID = ID);
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index 4cb1d01f3e8ca..9a27728dcb4dd 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -62,14 +62,8 @@ INITIALIZE_PASS_END(InstructionSelect, DEBUG_TYPE,
                     "Select target instructions out of generic instructions",
                     false, false)
 
-InstructionSelect::InstructionSelect(CodeGenOptLevel OL)
-    : MachineFunctionPass(ID), OptLevel(OL) {}
-
-// In order not to crash when calling getAnalysis during testing with -run-pass
-// we use the default opt level here instead of None, so that the addRequired()
-// calls are made in getAnalysisUsage().
-InstructionSelect::InstructionSelect()
-    : MachineFunctionPass(ID), OptLevel(CodeGenOptLevel::Default) {}
+InstructionSelect::InstructionSelect(CodeGenOptLevel OL, char &PassID)
+    : MachineFunctionPass(PassID), OptLevel(OL) {}
 
 void InstructionSelect::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<TargetPassConfig>();

From 96fb0ea4df27c46b7127dace37fc47718db52779 Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Sat, 20 Jul 2024 07:16:24 +0100
Subject: [PATCH 265/486] [compiler-rt] remove unneeded comma for
 pthread_atfork declaration (#99739)

---
 compiler-rt/lib/asan/asan_interceptors.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/lib/asan/asan_interceptors.cpp b/compiler-rt/lib/asan/asan_interceptors.cpp
index f8f86a766b204..74af2e65e9bfa 100644
--- a/compiler-rt/lib/asan/asan_interceptors.cpp
+++ b/compiler-rt/lib/asan/asan_interceptors.cpp
@@ -747,7 +747,7 @@ INTERCEPTOR(int, atexit, void (*func)()) {
 extern "C" {
 extern int _pthread_atfork(void (*prepare)(), void (*parent)(),
                            void (*child)());
-};
+}
 
 INTERCEPTOR(int, pthread_atfork, void (*prepare)(), void (*parent)(),
             void (*child)()) {

From 37be437b1237ee1e02e72eb59c915be16ad8ee53 Mon Sep 17 00:00:00 2001
From: alx32 <103613512+alx32@users.noreply.github.com>
Date: Fri, 19 Jul 2024 23:17:59 -0700
Subject: [PATCH 266/486] [lld-macho] Fix erasing category names for ObjC
 categories (#99400)

We were already not deleting category names for Swift classes as those
names can be reused by other parts. However, I have come across a corner
case where this also happens for ObjC categories - so we can't delete
category names for them either. TODO remains to optimize this behavior
for both ObjC and Swift.
---
 lld/MachO/ObjC.cpp                            |  14 +-
 ...jc-category-merging-erase-objc-name-test.s | 306 ++++++++++++++++++
 2 files changed, 315 insertions(+), 5 deletions(-)
 create mode 100644 lld/test/MachO/objc-category-merging-erase-objc-name-test.s

diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp
index 4a6f99654ba13..9c056f40aa943 100644
--- a/lld/MachO/ObjC.cpp
+++ b/lld/MachO/ObjC.cpp
@@ -1303,12 +1303,16 @@ void ObjcCategoryMerger::eraseMergedCategories() {
         continue;
 
       eraseISec(catInfo.catBodyIsec);
-      // We can't erase 'catLayout.nameOffset' for Swift categories because the
-      // name will be referenced for generating relative offsets
-      // See usages of 'l_.str.11.SimpleClass' in objc-category-merging-swift.s
+
+      // We can't erase 'catLayout.nameOffset' for either Swift or ObjC
+      //   categories because the name will sometimes also be used for other
+      //   purposes.
+      // For Swift, see usages of 'l_.str.11.SimpleClass' in
+      //   objc-category-merging-swift.s
+      // For ObjC, see usages of 'l_OBJC_CLASS_NAME_.1' in
+      //   objc-category-merging-erase-objc-name-test.s
       // TODO: handle the above in a smarter way
-      if (catInfo.sourceLanguage != SourceLanguage::Swift)
-        tryEraseDefinedAtIsecOffset(catInfo.catBodyIsec, catLayout.nameOffset);
+
       tryEraseDefinedAtIsecOffset(catInfo.catBodyIsec,
                                   catLayout.instanceMethodsOffset);
       tryEraseDefinedAtIsecOffset(catInfo.catBodyIsec,
diff --git a/lld/test/MachO/objc-category-merging-erase-objc-name-test.s b/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
new file mode 100644
index 0000000000000..01c5c4fd9e0c3
--- /dev/null
+++ b/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
@@ -0,0 +1,306 @@
+; REQUIRES: aarch64
+
+; Here we test that if we defined a protocol MyTestProtocol and also a category MyTestProtocol
+; then when merging the category into the base class (and deleting the category), we don't
+; delete the 'MyTestProtocol' name
+
+; RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o erase-objc-name.o %s
+; RUN: %lld -arch arm64 -dylib -o erase-objc-name.dylib erase-objc-name.o -objc_category_merging
+; RUN: llvm-objdump --objc-meta-data --macho erase-objc-name.dylib | FileCheck %s --check-prefixes=MERGE_CATS
+
+; === Check merge categories enabled ===
+; Check that the original categories are not there
+; MERGE_CATS-NOT: __OBJC_$_CATEGORY_MyBaseClass_$_Category01
+; MERGE_CATS-NOT: __OBJC_$_CATEGORY_MyBaseClass_$_Category02
+
+; Check that we get the expected output - most importantly that the protocol is named `MyTestProtocol`
+; MERGE_CATS:        Contents of (__DATA_CONST,__objc_classlist) section
+; MERGE_CATS-NEXT:   _OBJC_CLASS_$_MyBaseClass
+; MERGE_CATS-NEXT:            isa {{.*}} _OBJC_METACLASS_$_MyBaseClass
+; MERGE_CATS-NEXT:     superclass {{.*}}
+; MERGE_CATS-NEXT:          cache {{.*}}
+; MERGE_CATS-NEXT:         vtable {{.*}}
+; MERGE_CATS-NEXT:           data {{.*}} (struct class_ro_t *)
+; MERGE_CATS-NEXT:                     flags {{.*}} RO_ROOT
+; MERGE_CATS-NEXT:             instanceStart 0
+; MERGE_CATS-NEXT:              instanceSize 0
+; MERGE_CATS-NEXT:                  reserved {{.*}}
+; MERGE_CATS-NEXT:                ivarLayout {{.*}}
+; MERGE_CATS-NEXT:                      name {{.*}} MyBaseClass
+; MERGE_CATS-NEXT:               baseMethods {{.*}} (struct method_list_t *)
+; MERGE_CATS-NEXT:             entsize 24
+; MERGE_CATS-NEXT:               count 2
+; MERGE_CATS-NEXT:                name {{.*}} getValue
+; MERGE_CATS-NEXT:               types {{.*}} i16@0:8
+; MERGE_CATS-NEXT:                 imp -[MyBaseClass(MyTestProtocol) getValue]
+; MERGE_CATS-NEXT:                name {{.*}} baseInstanceMethod
+; MERGE_CATS-NEXT:               types {{.*}} v16@0:8
+; MERGE_CATS-NEXT:                 imp -[MyBaseClass baseInstanceMethod]
+; MERGE_CATS-NEXT:             baseProtocols {{.*}}
+; MERGE_CATS-NEXT:                       count 1
+; MERGE_CATS-NEXT:                list[0] {{.*}} (struct protocol_t *)
+; MERGE_CATS-NEXT:                       isa {{.*}}
+; MERGE_CATS-NEXT:                      name {{.*}} MyTestProtocol
+; MERGE_CATS-NEXT:                 protocols {{.*}}
+; MERGE_CATS-NEXT:            instanceMethods {{.*}} (struct method_list_t *)
+; MERGE_CATS-NEXT:                    entsize 24
+; MERGE_CATS-NEXT:                      count 1
+; MERGE_CATS-NEXT:                       name {{.*}} getValue
+; MERGE_CATS-NEXT:                      types {{.*}} i16@0:8
+; MERGE_CATS-NEXT:                        imp {{.*}}
+; MERGE_CATS-NEXT:               classMethods {{.*}} (struct method_list_t *)
+; MERGE_CATS-NEXT:     optionalInstanceMethods {{.*}}
+; MERGE_CATS-NEXT:        optionalClassMethods {{.*}}
+; MERGE_CATS-NEXT:          instanceProperties {{.*}}
+; MERGE_CATS-NEXT:                     ivars {{.*}}
+; MERGE_CATS-NEXT:            weakIvarLayout {{.*}}
+; MERGE_CATS-NEXT:            baseProperties {{.*}}
+; MERGE_CATS-NEXT: Meta Class
+; MERGE_CATS-NEXT:            isa {{.*}} _OBJC_METACLASS_$_MyBaseClass
+; MERGE_CATS-NEXT:     superclass {{.*}} _OBJC_CLASS_$_MyBaseClass
+; MERGE_CATS-NEXT:          cache {{.*}}
+; MERGE_CATS-NEXT:         vtable {{.*}}
+; MERGE_CATS-NEXT:           data {{.*}} (struct class_ro_t *)
+; MERGE_CATS-NEXT:                     flags {{.*}} RO_META RO_ROOT
+; MERGE_CATS-NEXT:             instanceStart 40
+; MERGE_CATS-NEXT:              instanceSize 40
+; MERGE_CATS-NEXT:                  reserved {{.*}}
+; MERGE_CATS-NEXT:                ivarLayout {{.*}}
+; MERGE_CATS-NEXT:                      name {{.*}} MyBaseClass
+; MERGE_CATS-NEXT:               baseMethods {{.*}} (struct method_list_t *)
+; MERGE_CATS-NEXT:             baseProtocols {{.*}}
+; MERGE_CATS-NEXT:                       count 1
+; MERGE_CATS-NEXT:                list[0] {{.*}} (struct protocol_t *)
+; MERGE_CATS-NEXT:                       isa {{.*}}
+; MERGE_CATS-NEXT:                      name {{.*}} MyTestProtocol
+; MERGE_CATS-NEXT:                 protocols {{.*}}
+; MERGE_CATS-NEXT:            instanceMethods {{.*}} (struct method_list_t *)
+; MERGE_CATS-NEXT:                    entsize 24
+; MERGE_CATS-NEXT:                      count 1
+; MERGE_CATS-NEXT:                       name {{.*}} getValue
+; MERGE_CATS-NEXT:                      types {{.*}} i16@0:8
+; MERGE_CATS-NEXT:                        imp {{.*}}
+; MERGE_CATS-NEXT:               classMethods {{.*}} (struct method_list_t *)
+; MERGE_CATS-NEXT:     optionalInstanceMethods {{.*}}
+; MERGE_CATS-NEXT:        optionalClassMethods {{.*}}
+; MERGE_CATS-NEXT:          instanceProperties {{.*}}
+; MERGE_CATS-NEXT:                     ivars {{.*}}
+; MERGE_CATS-NEXT:            weakIvarLayout {{.*}}
+; MERGE_CATS-NEXT:            baseProperties {{.*}}
+; MERGE_CATS-NEXT: Contents of (__DATA_CONST,__objc_protolist) section
+; MERGE_CATS-NEXT: {{.*}} {{.*}} __OBJC_PROTOCOL_$_MyTestProtocol
+; MERGE_CATS-NEXT: Contents of (__DATA_CONST,__objc_imageinfo) section
+; MERGE_CATS-NEXT:   version 0
+; MERGE_CATS-NEXT:     flags {{.*}} OBJC_IMAGE_HAS_CATEGORY_CLASS_PROPERTIES
+
+
+; ================== repro.sh ====================
+; # Write the Objective-C code to a file
+; cat << EOF > MyClass.m
+; @protocol MyTestProtocol
+; - (int)getValue;
+; @end
+;
+; __attribute__((objc_root_class))
+; @interface MyBaseClass
+; - (void)baseInstanceMethod;
+; @end
+;
+; @implementation MyBaseClass
+; - (void)baseInstanceMethod {}
+; @end
+;
+; @interface MyBaseClass (MyTestProtocol) <MyTestProtocol>
+; @end
+;
+; @implementation MyBaseClass (MyTestProtocol)
+;
+; - (int)getValue {
+;     return 0x30;
+; }
+;
+; @end
+; EOF
+;
+; # Compile the Objective-C file to assembly
+; xcrun clang -S -arch arm64 MyClass.m -o MyClass.s
+; ==============================================
+
+
+       .section      __TEXT,__text,regular,pure_instructions
+       .p2align      2                               ; -- Begin function -[MyBaseClass baseInstanceMethod]
+"-[MyBaseClass baseInstanceMethod]":    ; @"\01-[MyBaseClass baseInstanceMethod]"
+       .cfi_startproc
+; %bb.0:
+       sub    sp, sp, #16
+       .cfi_def_cfa_offset 16
+       str    x0, [sp, #8]
+       str    x1, [sp]
+       add    sp, sp, #16
+       ret
+       .cfi_endproc
+                                        ; -- End function
+       .p2align      2                               ; -- Begin function -[MyBaseClass(MyTestProtocol) getValue]
+"-[MyBaseClass(MyTestProtocol) getValue]": ; @"\01-[MyBaseClass(MyTestProtocol) getValue]"
+       .cfi_startproc
+; %bb.0:
+       sub    sp, sp, #16
+       .cfi_def_cfa_offset 16
+       str    x0, [sp, #8]
+       str    x1, [sp]
+       mov    w0, #48                         ; =0x30
+       add    sp, sp, #16
+       ret
+       .cfi_endproc
+                                        ; -- End function
+       .section      __DATA,__objc_data
+       .globl _OBJC_CLASS_$_MyBaseClass       ; @"OBJC_CLASS_$_MyBaseClass"
+       .p2align      3, 0x0
+_OBJC_CLASS_$_MyBaseClass:
+       .quad  _OBJC_METACLASS_$_MyBaseClass
+       .quad  0
+       .quad  __objc_empty_cache
+       .quad  0
+       .quad  __OBJC_CLASS_RO_$_MyBaseClass
+       .globl _OBJC_METACLASS_$_MyBaseClass   ; @"OBJC_METACLASS_$_MyBaseClass"
+       .p2align      3, 0x0
+_OBJC_METACLASS_$_MyBaseClass:
+       .quad  _OBJC_METACLASS_$_MyBaseClass
+       .quad  _OBJC_CLASS_$_MyBaseClass
+       .quad  __objc_empty_cache
+       .quad  0
+       .quad  __OBJC_METACLASS_RO_$_MyBaseClass
+       .section      __TEXT,__objc_classname,cstring_literals
+l_OBJC_CLASS_NAME_:                     ; @OBJC_CLASS_NAME_
+       .asciz "MyBaseClass"
+       .section      __DATA,__objc_const
+       .p2align      3, 0x0                          ; @"_OBJC_METACLASS_RO_$_MyBaseClass"
+__OBJC_METACLASS_RO_$_MyBaseClass:
+       .long  131                             ; 0x83
+       .long  40                              ; 0x28
+       .long  40                              ; 0x28
+       .space 4
+       .quad  0
+       .quad  l_OBJC_CLASS_NAME_
+       .quad  0
+       .quad  0
+       .quad  0
+       .quad  0
+       .quad  0
+       .section      __TEXT,__objc_methname,cstring_literals
+l_OBJC_METH_VAR_NAME_:                  ; @OBJC_METH_VAR_NAME_
+       .asciz "baseInstanceMethod"
+       .section      __TEXT,__objc_methtype,cstring_literals
+l_OBJC_METH_VAR_TYPE_:                  ; @OBJC_METH_VAR_TYPE_
+       .asciz "v16@0:8"
+       .section      __DATA,__objc_const
+       .p2align      3, 0x0                          ; @"_OBJC_$_INSTANCE_METHODS_MyBaseClass"
+__OBJC_$_INSTANCE_METHODS_MyBaseClass:
+       .long  24                              ; 0x18
+       .long  1                               ; 0x1
+       .quad  l_OBJC_METH_VAR_NAME_
+       .quad  l_OBJC_METH_VAR_TYPE_
+       .quad  "-[MyBaseClass baseInstanceMethod]"
+       .p2align      3, 0x0                          ; @"_OBJC_CLASS_RO_$_MyBaseClass"
+__OBJC_CLASS_RO_$_MyBaseClass:
+       .long  130                             ; 0x82
+       .long  0                               ; 0x0
+       .long  0                               ; 0x0
+       .space 4
+       .quad  0
+       .quad  l_OBJC_CLASS_NAME_
+       .quad  __OBJC_$_INSTANCE_METHODS_MyBaseClass
+       .quad  0
+       .quad  0
+       .quad  0
+       .quad  0
+       .section      __TEXT,__objc_classname,cstring_literals
+l_OBJC_CLASS_NAME_.1:                   ; @OBJC_CLASS_NAME_.1
+       .asciz "MyTestProtocol"
+       .section      __TEXT,__objc_methname,cstring_literals
+l_OBJC_METH_VAR_NAME_.2:                ; @OBJC_METH_VAR_NAME_.2
+       .asciz "getValue"
+       .section      __TEXT,__objc_methtype,cstring_literals
+l_OBJC_METH_VAR_TYPE_.3:                ; @OBJC_METH_VAR_TYPE_.3
+       .asciz "i16@0:8"
+       .section      __DATA,__objc_const
+       .p2align      3, 0x0                          ; @"_OBJC_$_CATEGORY_INSTANCE_METHODS_MyBaseClass_$_MyTestProtocol"
+__OBJC_$_CATEGORY_INSTANCE_METHODS_MyBaseClass_$_MyTestProtocol:
+       .long  24                              ; 0x18
+       .long  1                               ; 0x1
+       .quad  l_OBJC_METH_VAR_NAME_.2
+       .quad  l_OBJC_METH_VAR_TYPE_.3
+       .quad  "-[MyBaseClass(MyTestProtocol) getValue]"
+       .p2align      3, 0x0                          ; @"_OBJC_$_PROTOCOL_INSTANCE_METHODS_MyTestProtocol"
+__OBJC_$_PROTOCOL_INSTANCE_METHODS_MyTestProtocol:
+       .long  24                              ; 0x18
+       .long  1                               ; 0x1
+       .quad  l_OBJC_METH_VAR_NAME_.2
+       .quad  l_OBJC_METH_VAR_TYPE_.3
+       .quad  0
+       .p2align      3, 0x0                          ; @"_OBJC_$_PROTOCOL_METHOD_TYPES_MyTestProtocol"
+__OBJC_$_PROTOCOL_METHOD_TYPES_MyTestProtocol:
+       .quad  l_OBJC_METH_VAR_TYPE_.3
+       .private_extern      __OBJC_PROTOCOL_$_MyTestProtocol ; @"_OBJC_PROTOCOL_$_MyTestProtocol"
+       .section      __DATA,__data
+       .globl __OBJC_PROTOCOL_$_MyTestProtocol
+       .weak_definition     __OBJC_PROTOCOL_$_MyTestProtocol
+       .p2align      3, 0x0
+__OBJC_PROTOCOL_$_MyTestProtocol:
+       .quad  0
+       .quad  l_OBJC_CLASS_NAME_.1
+       .quad  0
+       .quad  __OBJC_$_PROTOCOL_INSTANCE_METHODS_MyTestProtocol
+       .quad  0
+       .quad  0
+       .quad  0
+       .quad  0
+       .long  96                              ; 0x60
+       .long  0                               ; 0x0
+       .quad  __OBJC_$_PROTOCOL_METHOD_TYPES_MyTestProtocol
+       .quad  0
+       .quad  0
+       .private_extern      __OBJC_LABEL_PROTOCOL_$_MyTestProtocol ; @"_OBJC_LABEL_PROTOCOL_$_MyTestProtocol"
+       .section      __DATA,__objc_protolist,coalesced,no_dead_strip
+       .globl __OBJC_LABEL_PROTOCOL_$_MyTestProtocol
+       .weak_definition     __OBJC_LABEL_PROTOCOL_$_MyTestProtocol
+       .p2align      3, 0x0
+__OBJC_LABEL_PROTOCOL_$_MyTestProtocol:
+       .quad  __OBJC_PROTOCOL_$_MyTestProtocol
+       .section      __DATA,__objc_const
+       .p2align      3, 0x0                          ; @"_OBJC_CATEGORY_PROTOCOLS_$_MyBaseClass_$_MyTestProtocol"
+__OBJC_CATEGORY_PROTOCOLS_$_MyBaseClass_$_MyTestProtocol:
+       .quad  1                               ; 0x1
+       .quad  __OBJC_PROTOCOL_$_MyTestProtocol
+       .quad  0
+       .p2align      3, 0x0                          ; @"_OBJC_$_CATEGORY_MyBaseClass_$_MyTestProtocol"
+__OBJC_$_CATEGORY_MyBaseClass_$_MyTestProtocol:
+       .quad  l_OBJC_CLASS_NAME_.1
+       .quad  _OBJC_CLASS_$_MyBaseClass
+       .quad  __OBJC_$_CATEGORY_INSTANCE_METHODS_MyBaseClass_$_MyTestProtocol
+       .quad  0
+       .quad  __OBJC_CATEGORY_PROTOCOLS_$_MyBaseClass_$_MyTestProtocol
+       .quad  0
+       .quad  0
+       .long  64                              ; 0x40
+       .space 4
+       .section      __DATA,__objc_classlist,regular,no_dead_strip
+       .p2align      3, 0x0                          ; @"OBJC_LABEL_CLASS_$"
+l_OBJC_LABEL_CLASS_$:
+       .quad  _OBJC_CLASS_$_MyBaseClass
+       .section      __DATA,__objc_catlist,regular,no_dead_strip
+       .p2align      3, 0x0                          ; @"OBJC_LABEL_CATEGORY_$"
+l_OBJC_LABEL_CATEGORY_$:
+       .quad  __OBJC_$_CATEGORY_MyBaseClass_$_MyTestProtocol
+       .no_dead_strip       __OBJC_PROTOCOL_$_MyTestProtocol
+       .no_dead_strip       __OBJC_LABEL_PROTOCOL_$_MyTestProtocol
+       .section      __DATA,__objc_imageinfo,regular,no_dead_strip
+L_OBJC_IMAGE_INFO:
+       .long  0
+       .long  64
+
+__objc_empty_cache:
+_$sBOWV:
+  .quad 0
+
+.subsections_via_symbols

From 06d2176d81cab1d3ed8d0c17f78c1d3ef65cbab8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Fri, 19 Jul 2024 19:27:13 +0200
Subject: [PATCH 267/486] [clang][Interp][NFC] Move global variable init case
 to the top

of the respective functions. Previously, we did not properly mark
global zero sized arrays as initialized.
---
 clang/lib/AST/Interp/Pointer.cpp | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp
index ff4da0fa805dc..b22b4b1918ba5 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/Interp/Pointer.cpp
@@ -227,6 +227,12 @@ bool Pointer::isInitialized() const {
   if (isIntegralPointer())
     return true;
 
+  if (isRoot() && PointeeStorage.BS.Base == sizeof(GlobalInlineDescriptor)) {
+    const GlobalInlineDescriptor &GD =
+        *reinterpret_cast<const GlobalInlineDescriptor *>(block()->rawData());
+    return GD.InitState == GlobalInitState::Initialized;
+  }
+
   assert(PointeeStorage.BS.Pointee &&
          "Cannot check if null pointer was initialized");
   const Descriptor *Desc = getFieldDesc();
@@ -249,12 +255,6 @@ bool Pointer::isInitialized() const {
   if (asBlockPointer().Base == 0)
     return true;
 
-  if (isRoot() && PointeeStorage.BS.Base == sizeof(GlobalInlineDescriptor)) {
-    const GlobalInlineDescriptor &GD =
-        *reinterpret_cast<const GlobalInlineDescriptor *>(block()->rawData());
-    return GD.InitState == GlobalInitState::Initialized;
-  }
-
   // Field has its bit in an inline descriptor.
   return getInlineDesc()->IsInitialized;
 }
@@ -266,6 +266,13 @@ void Pointer::initialize() const {
   assert(PointeeStorage.BS.Pointee && "Cannot initialize null pointer");
   const Descriptor *Desc = getFieldDesc();
 
+  if (isRoot() && PointeeStorage.BS.Base == sizeof(GlobalInlineDescriptor)) {
+    GlobalInlineDescriptor &GD = *reinterpret_cast<GlobalInlineDescriptor *>(
+        asBlockPointer().Pointee->rawData());
+    GD.InitState = GlobalInitState::Initialized;
+    return;
+  }
+
   assert(Desc);
   if (Desc->isPrimitiveArray()) {
     // Primitive global arrays don't have an initmap.
@@ -294,13 +301,6 @@ void Pointer::initialize() const {
     return;
   }
 
-  if (isRoot() && PointeeStorage.BS.Base == sizeof(GlobalInlineDescriptor)) {
-    GlobalInlineDescriptor &GD = *reinterpret_cast<GlobalInlineDescriptor *>(
-        asBlockPointer().Pointee->rawData());
-    GD.InitState = GlobalInitState::Initialized;
-    return;
-  }
-
   // Field has its bit in an inline descriptor.
   assert(PointeeStorage.BS.Base != 0 &&
          "Only composite fields can be initialised");

From b201ab8adc812f025490388ef89d5547e63b9d1e Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 19 Jul 2024 23:36:31 -0700
Subject: [PATCH 268/486] [MC] Move setIncrementalLinkerCompatible() calls to
 MCWinCOFFStreamer

Similar to setRelaxAll (45b59cb1d42b57a40c79a61afc8d1b8892826480).
---
 llvm/lib/MC/MCWinCOFFStreamer.cpp                          | 7 ++++++-
 .../Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp | 6 ++----
 llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp    | 6 ++----
 llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp    | 6 +-----
 4 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp
index 538244cb8ba6a..a14d3bcf37f3f 100644
--- a/llvm/lib/MC/MCWinCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp
@@ -27,6 +27,7 @@
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSymbolCOFF.h"
+#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
@@ -45,7 +46,11 @@ MCWinCOFFStreamer::MCWinCOFFStreamer(MCContext &Context,
                                      std::unique_ptr<MCCodeEmitter> CE,
                                      std::unique_ptr<MCObjectWriter> OW)
     : MCObjectStreamer(Context, std::move(MAB), std::move(OW), std::move(CE)),
-      CurSymbol(nullptr) {}
+      CurSymbol(nullptr) {
+  auto *TO = Context.getTargetOptions();
+  if (TO && TO->MCIncrementalLinkerCompatible)
+    getAssembler().setIncrementalLinkerCompatible(true);
+}
 
 void MCWinCOFFStreamer::emitInstToData(const MCInst &Inst,
                                        const MCSubtargetInfo &STI) {
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
index c25cc2e99adca..6f2826348d0bc 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
@@ -295,8 +295,6 @@ MCWinCOFFStreamer *llvm::createAArch64WinCOFFStreamer(
     MCContext &Context, std::unique_ptr<MCAsmBackend> MAB,
     std::unique_ptr<MCObjectWriter> OW, std::unique_ptr<MCCodeEmitter> Emitter,
     bool IncrementalLinkerCompatible) {
-  auto *S = new AArch64WinCOFFStreamer(Context, std::move(MAB),
-                                       std::move(Emitter), std::move(OW));
-  S->getAssembler().setIncrementalLinkerCompatible(IncrementalLinkerCompatible);
-  return S;
+  return new AArch64WinCOFFStreamer(Context, std::move(MAB), std::move(Emitter),
+                                    std::move(OW));
 }
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
index 0fcf6eb1a5abb..c7c297ab09799 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
@@ -72,10 +72,8 @@ llvm::createARMWinCOFFStreamer(MCContext &Context,
                                std::unique_ptr<MCObjectWriter> &&OW,
                                std::unique_ptr<MCCodeEmitter> &&Emitter,
                                bool IncrementalLinkerCompatible) {
-  auto *S = new ARMWinCOFFStreamer(Context, std::move(MAB), std::move(Emitter),
-                                   std::move(OW));
-  S->getAssembler().setIncrementalLinkerCompatible(IncrementalLinkerCompatible);
-  return S;
+  return new ARMWinCOFFStreamer(Context, std::move(MAB), std::move(Emitter),
+                                std::move(OW));
 }
 
 namespace {
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp
index b1e5362c5d24b..481866431364d 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp
@@ -77,9 +77,5 @@ MCStreamer *llvm::createX86WinCOFFStreamer(MCContext &C,
                                            std::unique_ptr<MCObjectWriter> &&OW,
                                            std::unique_ptr<MCCodeEmitter> &&CE,
                                            bool IncrementalLinkerCompatible) {
-  X86WinCOFFStreamer *S =
-      new X86WinCOFFStreamer(C, std::move(AB), std::move(CE), std::move(OW));
-  S->getAssembler().setIncrementalLinkerCompatible(IncrementalLinkerCompatible);
-  return S;
+  return new X86WinCOFFStreamer(C, std::move(AB), std::move(CE), std::move(OW));
 }
-

From 79a8279edc2e873fea2de3e736eaa0bfccfb6d1d Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 19 Jul 2024 23:51:57 -0700
Subject: [PATCH 269/486] [MC] Remove IncrementalLinkerCompatible parameters
 from create*WinCOFFStreamer

Follow-up to b201ab8adc812f025490388ef89d5547e63b9d1e.
Similar to RelaxAll (4e340356163aaaf2de83cd73f74bca8db735471b).
---
 llvm/include/llvm/MC/TargetRegistry.h                    | 9 +++------
 .../Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp  | 6 ++----
 .../AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp      | 9 +++++----
 .../Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h | 3 +--
 llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h       | 3 +--
 llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp  | 3 +--
 llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h       | 3 +--
 llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp  | 9 ++++-----
 8 files changed, 18 insertions(+), 27 deletions(-)

diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index 5038b87cd1dc9..75387f84149db 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -206,8 +206,7 @@ class Target {
   using COFFStreamerCtorTy =
       MCStreamer *(*)(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB,
                       std::unique_ptr<MCObjectWriter> &&OW,
-                      std::unique_ptr<MCCodeEmitter> &&Emitter,
-                      bool IncrementalLinkerCompatible);
+                      std::unique_ptr<MCCodeEmitter> &&Emitter);
   using WasmStreamerCtorTy =
       MCStreamer *(*)(const Triple &T, MCContext &Ctx,
                       std::unique_ptr<MCAsmBackend> &&TAB,
@@ -555,13 +554,11 @@ class Target {
   /// \param TAB The target assembler backend object. Takes ownership.
   /// \param OW The stream object.
   /// \param Emitter The target independent assembler object.Takes ownership.
-  /// \param RelaxAll Relax all fixups?
   MCStreamer *createMCObjectStreamer(const Triple &T, MCContext &Ctx,
                                      std::unique_ptr<MCAsmBackend> &&TAB,
                                      std::unique_ptr<MCObjectWriter> &&OW,
                                      std::unique_ptr<MCCodeEmitter> &&Emitter,
-                                     const MCSubtargetInfo &STI, bool,
-                                     bool IncrementalLinkerCompatible,
+                                     const MCSubtargetInfo &STI, bool, bool,
                                      bool DWARFMustBeAtTheEnd) const {
     MCStreamer *S = nullptr;
     switch (T.getObjectFormat()) {
@@ -571,7 +568,7 @@ class Target {
       assert((T.isOSWindows() || T.isUEFI()) &&
              "only Windows and UEFI COFF are supported");
       S = COFFStreamerCtorFn(Ctx, std::move(TAB), std::move(OW),
-                             std::move(Emitter), IncrementalLinkerCompatible);
+                             std::move(Emitter));
       break;
     case Triple::MachO:
       if (MachOStreamerCtorFn)
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index f05e5e6df7f8e..1e367be9fa4ae 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -399,11 +399,9 @@ static MCStreamer *createMachOStreamer(MCContext &Ctx,
 static MCStreamer *
 createWinCOFFStreamer(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB,
                       std::unique_ptr<MCObjectWriter> &&OW,
-                      std::unique_ptr<MCCodeEmitter> &&Emitter,
-                      bool IncrementalLinkerCompatible) {
+                      std::unique_ptr<MCCodeEmitter> &&Emitter) {
   return createAArch64WinCOFFStreamer(Ctx, std::move(TAB), std::move(OW),
-                                      std::move(Emitter),
-                                      IncrementalLinkerCompatible);
+                                      std::move(Emitter));
 }
 
 namespace {
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
index 6f2826348d0bc..208d43502cb88 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
@@ -291,10 +291,11 @@ void AArch64TargetWinCOFFStreamer::emitARM64WinCFISaveAnyRegQPX(unsigned Reg,
   emitARM64WinUnwindCode(Win64EH::UOP_SaveAnyRegQPX, Reg, Offset);
 }
 
-MCWinCOFFStreamer *llvm::createAArch64WinCOFFStreamer(
-    MCContext &Context, std::unique_ptr<MCAsmBackend> MAB,
-    std::unique_ptr<MCObjectWriter> OW, std::unique_ptr<MCCodeEmitter> Emitter,
-    bool IncrementalLinkerCompatible) {
+MCWinCOFFStreamer *
+llvm::createAArch64WinCOFFStreamer(MCContext &Context,
+                                   std::unique_ptr<MCAsmBackend> MAB,
+                                   std::unique_ptr<MCObjectWriter> OW,
+                                   std::unique_ptr<MCCodeEmitter> Emitter) {
   return new AArch64WinCOFFStreamer(Context, std::move(MAB), std::move(Emitter),
                                     std::move(OW));
 }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h
index a13b1a451be5f..5caf520a3aa37 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h
@@ -20,8 +20,7 @@ namespace llvm {
 
 MCWinCOFFStreamer *createAArch64WinCOFFStreamer(
     MCContext &Context, std::unique_ptr<MCAsmBackend> TAB,
-    std::unique_ptr<MCObjectWriter> OW, std::unique_ptr<MCCodeEmitter> Emitter,
-    bool IncrementalLinkerCompatible);
+    std::unique_ptr<MCObjectWriter> OW, std::unique_ptr<MCCodeEmitter> Emitter);
 } // end llvm namespace
 
 #endif
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
index a673d590419ec..66f19237f275f 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
@@ -93,8 +93,7 @@ MCAsmBackend *createARMBEAsmBackend(const Target &T, const MCSubtargetInfo &STI,
 MCStreamer *createARMWinCOFFStreamer(MCContext &Context,
                                      std::unique_ptr<MCAsmBackend> &&MAB,
                                      std::unique_ptr<MCObjectWriter> &&OW,
-                                     std::unique_ptr<MCCodeEmitter> &&Emitter,
-                                     bool IncrementalLinkerCompatible);
+                                     std::unique_ptr<MCCodeEmitter> &&Emitter);
 
 /// Construct an ELF Mach-O object writer.
 std::unique_ptr<MCObjectTargetWriter> createARMELFObjectWriter(uint8_t OSABI);
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
index c7c297ab09799..e66059c2a0e09 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
@@ -70,8 +70,7 @@ MCStreamer *
 llvm::createARMWinCOFFStreamer(MCContext &Context,
                                std::unique_ptr<MCAsmBackend> &&MAB,
                                std::unique_ptr<MCObjectWriter> &&OW,
-                               std::unique_ptr<MCCodeEmitter> &&Emitter,
-                               bool IncrementalLinkerCompatible) {
+                               std::unique_ptr<MCCodeEmitter> &&Emitter) {
   return new ARMWinCOFFStreamer(Context, std::move(MAB), std::move(Emitter),
                                 std::move(OW));
 }
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
index 96a7823b04ad8..4e83e7e437aeb 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
@@ -128,8 +128,7 @@ MCTargetStreamer *createX86ObjectTargetStreamer(MCStreamer &S,
 MCStreamer *createX86WinCOFFStreamer(MCContext &C,
                                      std::unique_ptr<MCAsmBackend> &&AB,
                                      std::unique_ptr<MCObjectWriter> &&OW,
-                                     std::unique_ptr<MCCodeEmitter> &&CE,
-                                     bool IncrementalLinkerCompatible);
+                                     std::unique_ptr<MCCodeEmitter> &&CE);
 
 MCStreamer *createX86ELFStreamer(const Triple &T, MCContext &Context,
                                  std::unique_ptr<MCAsmBackend> &&MAB,
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp
index 481866431364d..1ef10928c05d8 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp
@@ -72,10 +72,9 @@ void X86WinCOFFStreamer::finishImpl() {
 }
 } // namespace
 
-MCStreamer *llvm::createX86WinCOFFStreamer(MCContext &C,
-                                           std::unique_ptr<MCAsmBackend> &&AB,
-                                           std::unique_ptr<MCObjectWriter> &&OW,
-                                           std::unique_ptr<MCCodeEmitter> &&CE,
-                                           bool IncrementalLinkerCompatible) {
+MCStreamer *
+llvm::createX86WinCOFFStreamer(MCContext &C, std::unique_ptr<MCAsmBackend> &&AB,
+                               std::unique_ptr<MCObjectWriter> &&OW,
+                               std::unique_ptr<MCCodeEmitter> &&CE) {
   return new X86WinCOFFStreamer(C, std::move(AB), std::move(CE), std::move(OW));
 }

From c2019a37bdb1375e9f72b2549361cc50ea7729db Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 20 Jul 2024 10:53:41 +0400
Subject: [PATCH 270/486] SelectionDAG: Avoid using MachineFunction::getMMI
 (#99696)

---
 llvm/include/llvm/CodeGen/SelectionDAG.h          |  9 ++++++---
 llvm/include/llvm/CodeGen/SelectionDAGISel.h      |  1 +
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp    |  3 ++-
 .../CodeGen/SelectionDAG/SelectionDAGBuilder.cpp  |  6 +++---
 .../lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 15 +++++++++------
 .../unittests/CodeGen/AArch64SelectionDAGTest.cpp |  3 ++-
 .../CodeGen/SelectionDAGAddressAnalysisTest.cpp   |  3 ++-
 .../CodeGen/SelectionDAGPatternMatchTest.cpp      |  3 ++-
 8 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 16ec65f2e7daa..24eab7b408675 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -245,6 +245,7 @@ class SelectionDAG {
 
   ProfileSummaryInfo *PSI = nullptr;
   BlockFrequencyInfo *BFI = nullptr;
+  MachineModuleInfo *MMI = nullptr;
 
   /// List of non-single value types.
   FoldingSet<SDVTListNode> VTListMap;
@@ -459,14 +460,15 @@ class SelectionDAG {
   void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
             Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
             UniformityInfo *UA, ProfileSummaryInfo *PSIin,
-            BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs);
+            BlockFrequencyInfo *BFIin, MachineModuleInfo &MMI,
+            FunctionVarLocs const *FnVarLocs);
 
   void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
             MachineFunctionAnalysisManager &AM,
             const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA,
             ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin,
-            FunctionVarLocs const *FnVarLocs) {
-    init(NewMF, NewORE, nullptr, LibraryInfo, UA, PSIin, BFIin, FnVarLocs);
+            MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs) {
+    init(NewMF, NewORE, nullptr, LibraryInfo, UA, PSIin, BFIin, MMI, FnVarLocs);
     MFAM = &AM;
   }
 
@@ -500,6 +502,7 @@ class SelectionDAG {
   OptimizationRemarkEmitter &getORE() const { return *ORE; }
   ProfileSummaryInfo *getPSI() const { return PSI; }
   BlockFrequencyInfo *getBFI() const { return BFI; }
+  MachineModuleInfo *getMMI() const { return MMI; }
 
   FlagInserter *getFlagInserter() { return Inserter; }
   void setFlagInserter(FlagInserter *FI) { Inserter = FI; }
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index aa0efa5d9bf5d..fc0590b1a1b69 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -48,6 +48,7 @@ class SelectionDAGISel {
   std::unique_ptr<FunctionLoweringInfo> FuncInfo;
   SwiftErrorValueTracking *SwiftError;
   MachineFunction *MF;
+  MachineModuleInfo *MMI;
   MachineRegisterInfo *RegInfo;
   SelectionDAG *CurDAG;
   std::unique_ptr<SelectionDAGBuilder> SDB;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 0ccae50286345..e4d08228ee903 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1333,7 +1333,7 @@ void SelectionDAG::init(MachineFunction &NewMF,
                         OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
                         const TargetLibraryInfo *LibraryInfo,
                         UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
-                        BlockFrequencyInfo *BFIin,
+                        BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
                         FunctionVarLocs const *VarLocs) {
   MF = &NewMF;
   SDAGISelPass = PassPtr;
@@ -1345,6 +1345,7 @@ void SelectionDAG::init(MachineFunction &NewMF,
   UA = NewUA;
   PSI = PSIin;
   BFI = BFIin;
+  MMI = &MMIin;
   FnVarLocs = VarLocs;
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c179c97029c7a..cf133e162c65f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6707,11 +6707,11 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                              getValue(I.getArgOperand(0))));
     return;
   case Intrinsic::eh_sjlj_callsite: {
-    MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
-    assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
+    assert(DAG.getMMI()->getCurrentCallSite() == 0 &&
+           "Overlapping call sites!");
 
-    MMI.setCurrentCallSite(CI->getZExtValue());
+    DAG.getMMI()->setCurrentCallSite(CI->getZExtValue());
     return;
   }
   case Intrinsic::eh_sjlj_functioncontext: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index df3d207d85d35..2670ff488fbed 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -509,7 +509,7 @@ void SelectionDAGISel::initializeAnalysisResults(
     FnVarLocs = &FAM.getResult<DebugAssignmentTrackingAnalysis>(Fn);
 
   auto *UA = FAM.getCachedResult<UniformityInfoAnalysis>(Fn);
-  CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, FnVarLocs);
+  CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, *MMI, FnVarLocs);
 
   // Now get the optional analyzes if we want to.
   // This is based on the possibly changed OptLevel (after optnone is taken
@@ -562,7 +562,11 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
   UniformityInfo *UA = nullptr;
   if (auto *UAPass = MFP.getAnalysisIfAvailable<UniformityInfoWrapperPass>())
     UA = &UAPass->getUniformityInfo();
-  CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, FnVarLocs);
+
+  MachineModuleInfo &MMI =
+      MFP.getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
+
+  CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, MMI, FnVarLocs);
 
   // Now get the optional analyzes if we want to.
   // This is based on the possibly changed OptLevel (after optnone is taken
@@ -796,7 +800,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
   }
 
   // Determine if floating point is used for msvc
-  computeUsesMSVCFloatingPoint(TM.getTargetTriple(), Fn, MF->getMMI());
+  computeUsesMSVCFloatingPoint(TM.getTargetTriple(), Fn, *CurDAG->getMMI());
 
   // Release function-specific state. SDB and CurDAG are already cleared
   // at this point.
@@ -1443,7 +1447,6 @@ bool SelectionDAGISel::PrepareEHLandingPad() {
 
 // Mark and Report IPToState for each Block under IsEHa
 void SelectionDAGISel::reportIPToStateForBlocks(MachineFunction *MF) {
-  MachineModuleInfo &MMI = MF->getMMI();
   llvm::WinEHFuncInfo *EHInfo = MF->getWinEHFuncInfo();
   if (!EHInfo)
     return;
@@ -1458,8 +1461,8 @@ void SelectionDAGISel::reportIPToStateForBlocks(MachineFunction *MF) {
         continue;
 
       // Insert EH Labels
-      MCSymbol *BeginLabel = MMI.getContext().createTempSymbol();
-      MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
+      MCSymbol *BeginLabel = MF->getContext().createTempSymbol();
+      MCSymbol *EndLabel = MF->getContext().createTempSymbol();
       EHInfo->addIPToStateRange(State, BeginLabel, EndLabel);
       BuildMI(MBB, MBBb, SDB->getCurDebugLoc(),
               TII->get(TargetOpcode::EH_LABEL))
diff --git a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
index cc574b5887efc..877793a28b769 100644
--- a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
+++ b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
@@ -68,7 +68,8 @@ class AArch64SelectionDAGTest : public testing::Test {
     if (!DAG)
       report_fatal_error("DAG?");
     OptimizationRemarkEmitter ORE(F);
-    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr);
+    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
+              nullptr);
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {
diff --git a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
index 88c4dbc2eec78..ef30bd491d352 100644
--- a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
@@ -78,7 +78,8 @@ class SelectionDAGAddressAnalysisTest : public testing::Test {
     if (!DAG)
       report_fatal_error("DAG?");
     OptimizationRemarkEmitter ORE(F);
-    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr);
+    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
+              nullptr);
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {
diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index a3d5e5f94b610..e318f467bbf27 100644
--- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -77,7 +77,8 @@ class SelectionDAGPatternMatchTest : public testing::Test {
     if (!DAG)
       report_fatal_error("DAG?");
     OptimizationRemarkEmitter ORE(F);
-    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr);
+    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
+              nullptr);
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {

From 867ff2d4268ca7eed89a24f100b67b68f5329439 Mon Sep 17 00:00:00 2001
From: Vincent Lee <thevinster@users.noreply.github.com>
Date: Sat, 20 Jul 2024 00:13:20 -0700
Subject: [PATCH 271/486] [lit] Add a flag to disable lit time tests (#98270)

LLVM lit assumes control of the test parallelism when running a test
suite. This style of testing doesn't play nicely with build systems like
Buck or Bazel since it prefers finer grained actions on a per-test
level. In order for external build systems to control the test
parallelism, add an option to disable `.lit_test_times.txt` under the
`--skip-test-time-recording` flag, thus allowing other build systems
to determine the parallelism and avoid race conditions when writing
to that file. I went for `--skip-test-time-recording` instead of `--time-tests` in
order to preserve the original functionality of writing to `.lit_test_times.txt`
as the default behavior and only opt-in for those who do _not_ want
`.lit_test_times.txt` file.
---
 llvm/docs/CommandGuide/lit.rst                 |  4 ++++
 llvm/utils/lit/lit/cl_arguments.py             | 16 +++++++++++-----
 llvm/utils/lit/lit/main.py                     |  3 ++-
 llvm/utils/lit/tests/Inputs/time-tests/a.txt   |  1 +
 llvm/utils/lit/tests/Inputs/time-tests/lit.cfg |  7 +++++++
 llvm/utils/lit/tests/time-tests.py             | 15 +++++++++++++++
 6 files changed, 40 insertions(+), 6 deletions(-)
 create mode 100644 llvm/utils/lit/tests/Inputs/time-tests/a.txt
 create mode 100644 llvm/utils/lit/tests/Inputs/time-tests/lit.cfg
 create mode 100644 llvm/utils/lit/tests/time-tests.py

diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 799ee34e9f9ff..c9d5baba3e2f4 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -151,6 +151,10 @@ EXECUTION OPTIONS
  feature that can be used to conditionally disable (or expect failure in)
  certain tests.
 
+.. option:: --skip-test-time-recording
+
+ Disable tracking the wall time individual tests take to execute.
+
 .. option:: --time-tests
 
  Track the wall time individual tests take to execute and includes the results
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index b9122d07afd8a..ed78256ee414b 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -154,11 +154,6 @@ def parse_args():
         action="append",
         default=[],
     )
-    execution_group.add_argument(
-        "--time-tests",
-        help="Track elapsed wall time for each test",
-        action="store_true",
-    )
     execution_group.add_argument(
         "--no-execute",
         dest="noExecute",
@@ -209,6 +204,17 @@ def parse_args():
         action="store_true",
         help="Exit with status zero even if some tests fail",
     )
+    execution_test_time_group = execution_group.add_mutually_exclusive_group()
+    execution_test_time_group.add_argument(
+        "--skip-test-time-recording",
+        help="Do not track elapsed wall time for each test",
+        action="store_true",
+    )
+    execution_test_time_group.add_argument(
+        "--time-tests",
+        help="Track elapsed wall time for each test printed in a histogram",
+        action="store_true",
+    )
 
     selection_group = parser.add_argument_group("Test Selection")
     selection_group.add_argument(
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index db9f24f748d9e..24ba804f0c363 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -124,7 +124,8 @@ def main(builtin_params={}):
     run_tests(selected_tests, lit_config, opts, len(discovered_tests))
     elapsed = time.time() - start
 
-    record_test_times(selected_tests, lit_config)
+    if not opts.skip_test_time_recording:
+        record_test_times(selected_tests, lit_config)
 
     selected_tests, discovered_tests = GoogleTest.post_process_shard_results(
         selected_tests, discovered_tests
diff --git a/llvm/utils/lit/tests/Inputs/time-tests/a.txt b/llvm/utils/lit/tests/Inputs/time-tests/a.txt
new file mode 100644
index 0000000000000..b80b60b7a2794
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/time-tests/a.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/time-tests/lit.cfg b/llvm/utils/lit/tests/Inputs/time-tests/lit.cfg
new file mode 100644
index 0000000000000..e6ae41833874a
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/time-tests/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "time-tests"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/time-tests.py b/llvm/utils/lit/tests/time-tests.py
new file mode 100644
index 0000000000000..20b83a64330f0
--- /dev/null
+++ b/llvm/utils/lit/tests/time-tests.py
@@ -0,0 +1,15 @@
+## Check that --skip-test-time-recording skips .lit_test_times.txt recording.
+
+# RUN: %{lit-no-order-opt} --skip-test-time-recording %{inputs}/time-tests
+# RUN: not ls %{inputs}/time-tests/.lit_test_times.txt
+
+## Check that --time-tests generates a printed histogram.
+
+# RUN: %{lit-no-order-opt} --time-tests %{inputs}/time-tests > %t.out
+# RUN: FileCheck < %t.out %s
+# RUN: rm %{inputs}/time-tests/.lit_test_times.txt
+
+# CHECK:      Tests Times:
+# CHECK-NEXT: --------------------------------------------------------------------------
+# CHECK-NEXT: [    Range    ] :: [               Percentage               ] :: [Count]
+# CHECK-NEXT: --------------------------------------------------------------------------

From d386a5582b286bbd8a52f2fd3dbc5c8f70a8f60d Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Sat, 20 Jul 2024 00:56:52 -0700
Subject: [PATCH 272/486] [libc] Make static_assert available even if NDEBUG is
 set (#99742)

This addresses an issue introduced in #98826 where static_assert was
made defined only when NDEBUG is not set which is different from all
other C libraries and breaks any code that uses static_assert and
doesn't guard it with NDEBUG.
---
 libc/include/assert.h.def | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/libc/include/assert.h.def b/libc/include/assert.h.def
index 9c924c7f58545..d5ae14a1cd810 100644
--- a/libc/include/assert.h.def
+++ b/libc/include/assert.h.def
@@ -12,22 +12,19 @@
 // This file may be usefully included multiple times to change assert()'s
 // definition based on NDEBUG.
 
-
-#undef assert
-#ifdef NDEBUG
-#define assert(e) (void)0
-#else
-
 #ifndef __cplusplus
 #undef static_assert
 #define static_assert _Static_assert
 #endif
 
+#undef assert
+#ifdef NDEBUG
+#define assert(e) (void)0
+#else
 #ifdef __cplusplus
 extern "C"
 #endif
 _Noreturn void __assert_fail(const char *, const char *, unsigned, const char *) __NOEXCEPT;
-
 #define assert(e)  \
   ((e) ? (void)0 : __assert_fail(#e, __FILE__, __LINE__, __PRETTY_FUNCTION__))
 #endif

From 0d26f65414afe496b00ee803cc24722a9bf3f41d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sat, 20 Jul 2024 08:25:23 +0200
Subject: [PATCH 273/486] [clang][Interp] Emit diagnostics if final ltor
 conversion fails

---
 clang/lib/AST/Interp/EvalEmitter.cpp | 4 +++-
 clang/test/AST/Interp/cxx11.cpp      | 8 ++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/Interp/EvalEmitter.cpp b/clang/lib/AST/Interp/EvalEmitter.cpp
index 59e78686b78ad..221bbfdc542ff 100644
--- a/clang/lib/AST/Interp/EvalEmitter.cpp
+++ b/clang/lib/AST/Interp/EvalEmitter.cpp
@@ -169,7 +169,9 @@ template <> bool EvalEmitter::emitRet<PT_Ptr>(const SourceInfo &Info) {
       return false;
     // Never allow reading from a non-const pointer, unless the memory
     // has been created in this evaluation.
-    if (!Ptr.isConst() && Ptr.block()->getEvalID() != Ctx.getEvalID())
+    if (!Ptr.isZero() && Ptr.isBlockPointer() &&
+        Ptr.block()->getEvalID() != Ctx.getEvalID() &&
+        (!CheckLoad(S, OpPC, Ptr, AK_Read) || !Ptr.isConst()))
       return false;
 
     if (std::optional<APValue> V =
diff --git a/clang/test/AST/Interp/cxx11.cpp b/clang/test/AST/Interp/cxx11.cpp
index 92ab9b605f30d..cf2dfba079ef7 100644
--- a/clang/test/AST/Interp/cxx11.cpp
+++ b/clang/test/AST/Interp/cxx11.cpp
@@ -152,3 +152,11 @@ void A::f(SortOrder order) {
     return;
 }
 }
+
+namespace FinalLtorDiags {
+  template<int*> struct A {}; // both-note {{template parameter is declared here}}
+  int k;
+  int *q = &k; // both-note {{declared here}}
+  A<q> c; // both-error {{non-type template argument of type 'int *' is not a constant expression}} \
+          // both-note {{read of non-constexpr variable 'q' is not allowed in a constant expression}}
+}

From bbd4af5da2b741672a8e6f625eb12ea5c2d6220f Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Sat, 20 Jul 2024 10:12:13 +0200
Subject: [PATCH 274/486] [mlir][Transforms] Dialect conversion: Simplify
 handling of dropped arguments (#97213)

This commit simplifies the handling of dropped arguments and updates
some dialect conversion documentation that is outdated.

When converting a block signature, a `BlockTypeConversionRewrite` object
and potentially multiple `ReplaceBlockArgRewrite` are created. During
the "commit" phase, uses of the old block arguments are replaced with
the new block arguments, but the old implementation was written in an
inconsistent way: some block arguments were replaced in
`BlockTypeConversionRewrite::commit` and some were replaced in
`ReplaceBlockArgRewrite::commit`. The new
`BlockTypeConversionRewrite::commit` implementation is much simpler and
no longer modifies any IR; that is done only in `ReplaceBlockArgRewrite`
now. The `ConvertedArgInfo` data structure is no longer needed.

To that end, materializations of dropped arguments are now built in
`applySignatureConversion` instead of `materializeLiveConversions`; the
latter function no longer has to deal with dropped arguments.

Other minor improvements:
- Add more comments to `applySignatureConversion`.

Note: Error messages around failed materializations for dropped basic
block arguments changed slightly. That is because those materializations
are now built in `legalizeUnresolvedMaterialization` instead of
`legalizeConvertedArgumentTypes`.

This commit is in preparation of decoupling argument/source/target
materializations from the dialect conversion.

This is a re-upload of #96207.
---
 .../Transforms/Utils/DialectConversion.cpp    | 173 ++++++------------
 .../test-legalize-type-conversion.mlir        |   6 +-
 2 files changed, 57 insertions(+), 122 deletions(-)

diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 1e0afee2373a9..0b552a7e1ca3b 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -432,34 +432,14 @@ class MoveBlockRewrite : public BlockRewrite {
   Block *insertBeforeBlock;
 };
 
-/// This structure contains the information pertaining to an argument that has
-/// been converted.
-struct ConvertedArgInfo {
-  ConvertedArgInfo(unsigned newArgIdx, unsigned newArgSize,
-                   Value castValue = nullptr)
-      : newArgIdx(newArgIdx), newArgSize(newArgSize), castValue(castValue) {}
-
-  /// The start index of in the new argument list that contains arguments that
-  /// replace the original.
-  unsigned newArgIdx;
-
-  /// The number of arguments that replaced the original argument.
-  unsigned newArgSize;
-
-  /// The cast value that was created to cast from the new arguments to the
-  /// old. This only used if 'newArgSize' > 1.
-  Value castValue;
-};
-
 /// Block type conversion. This rewrite is partially reflected in the IR.
 class BlockTypeConversionRewrite : public BlockRewrite {
 public:
-  BlockTypeConversionRewrite(
-      ConversionPatternRewriterImpl &rewriterImpl, Block *block,
-      Block *origBlock, SmallVector<std::optional<ConvertedArgInfo>, 1> argInfo,
-      const TypeConverter *converter)
+  BlockTypeConversionRewrite(ConversionPatternRewriterImpl &rewriterImpl,
+                             Block *block, Block *origBlock,
+                             const TypeConverter *converter)
       : BlockRewrite(Kind::BlockTypeConversion, rewriterImpl, block),
-        origBlock(origBlock), argInfo(argInfo), converter(converter) {}
+        origBlock(origBlock), converter(converter) {}
 
   static bool classof(const IRRewrite *rewrite) {
     return rewrite->getKind() == Kind::BlockTypeConversion;
@@ -479,10 +459,6 @@ class BlockTypeConversionRewrite : public BlockRewrite {
   /// The original block that was requested to have its signature converted.
   Block *origBlock;
 
-  /// The conversion information for each of the arguments. The information is
-  /// std::nullopt if the argument was dropped during conversion.
-  SmallVector<std::optional<ConvertedArgInfo>, 1> argInfo;
-
   /// The type converter used to convert the arguments.
   const TypeConverter *converter;
 };
@@ -691,12 +667,16 @@ class CreateOperationRewrite : public OperationRewrite {
 /// The type of materialization.
 enum MaterializationKind {
   /// This materialization materializes a conversion for an illegal block
-  /// argument type, to a legal one.
+  /// argument type, to the original one.
   Argument,
 
   /// This materialization materializes a conversion from an illegal type to a
   /// legal one.
-  Target
+  Target,
+
+  /// This materialization materializes a conversion from a legal type back to
+  /// an illegal one.
+  Source
 };
 
 /// An unresolved materialization, i.e., a "builtin.unrealized_conversion_cast"
@@ -736,7 +716,7 @@ class UnresolvedMaterializationRewrite : public OperationRewrite {
 private:
   /// The corresponding type converter to use when resolving this
   /// materialization, and the kind of this materialization.
-  llvm::PointerIntPair<const TypeConverter *, 1, MaterializationKind>
+  llvm::PointerIntPair<const TypeConverter *, 2, MaterializationKind>
       converterAndKind;
 };
 } // namespace
@@ -855,11 +835,6 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
                                        ValueRange inputs, Type outputType,
                                        const TypeConverter *converter);
 
-  Value buildUnresolvedArgumentMaterialization(Block *block, Location loc,
-                                               ValueRange inputs,
-                                               Type outputType,
-                                               const TypeConverter *converter);
-
   Value buildUnresolvedTargetMaterialization(Location loc, Value input,
                                              Type outputType,
                                              const TypeConverter *converter);
@@ -989,28 +964,6 @@ void BlockTypeConversionRewrite::commit(RewriterBase &rewriter) {
           dyn_cast_or_null<RewriterBase::Listener>(rewriter.getListener()))
     for (Operation *op : block->getUsers())
       listener->notifyOperationModified(op);
-
-  // Process the remapping for each of the original arguments.
-  for (auto [origArg, info] :
-       llvm::zip_equal(origBlock->getArguments(), argInfo)) {
-    // Handle the case of a 1->0 value mapping.
-    if (!info) {
-      if (Value newArg =
-              rewriterImpl.mapping.lookupOrNull(origArg, origArg.getType()))
-        rewriter.replaceAllUsesWith(origArg, newArg);
-      continue;
-    }
-
-    // Otherwise this is a 1->1+ value mapping.
-    Value castValue = info->castValue;
-    assert(info->newArgSize >= 1 && castValue && "expected 1->1+ mapping");
-
-    // If the argument is still used, replace it with the generated cast.
-    if (!origArg.use_empty()) {
-      rewriter.replaceAllUsesWith(origArg, rewriterImpl.mapping.lookupOrDefault(
-                                               castValue, origArg.getType()));
-    }
-  }
 }
 
 void BlockTypeConversionRewrite::rollback() {
@@ -1035,14 +988,12 @@ LogicalResult BlockTypeConversionRewrite::materializeLiveConversions(
       continue;
 
     Value replacementValue = rewriterImpl.mapping.lookupOrDefault(origArg);
-    bool isDroppedArg = replacementValue == origArg;
-    if (!isDroppedArg)
-      builder.setInsertionPointAfterValue(replacementValue);
+    assert(replacementValue && "replacement value not found");
     Value newArg;
     if (converter) {
+      builder.setInsertionPointAfterValue(replacementValue);
       newArg = converter->materializeSourceConversion(
-          builder, origArg.getLoc(), origArg.getType(),
-          isDroppedArg ? ValueRange() : ValueRange(replacementValue));
+          builder, origArg.getLoc(), origArg.getType(), replacementValue);
       assert((!newArg || newArg.getType() == origArg.getType()) &&
              "materialization hook did not provide a value of the expected "
              "type");
@@ -1053,8 +1004,6 @@ LogicalResult BlockTypeConversionRewrite::materializeLiveConversions(
           << "failed to materialize conversion for block argument #"
           << it.index() << " that remained live after conversion, type was "
           << origArg.getType();
-      if (!isDroppedArg)
-        diag << ", with target type " << replacementValue.getType();
       diag.attachNote(liveUser->getLoc())
           << "see existing live user here: " << *liveUser;
       return failure();
@@ -1340,73 +1289,64 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion(
   // Replace all uses of the old block with the new block.
   block->replaceAllUsesWith(newBlock);
 
-  // Remap each of the original arguments as determined by the signature
-  // conversion.
-  SmallVector<std::optional<ConvertedArgInfo>, 1> argInfo;
-  argInfo.resize(origArgCount);
-
   for (unsigned i = 0; i != origArgCount; ++i) {
-    auto inputMap = signatureConversion.getInputMapping(i);
-    if (!inputMap)
-      continue;
     BlockArgument origArg = block->getArgument(i);
+    Type origArgType = origArg.getType();
+
+    std::optional<TypeConverter::SignatureConversion::InputMapping> inputMap =
+        signatureConversion.getInputMapping(i);
+    if (!inputMap) {
+      // This block argument was dropped and no replacement value was provided.
+      // Materialize a replacement value "out of thin air".
+      Value repl = buildUnresolvedMaterialization(
+          MaterializationKind::Source, newBlock, newBlock->begin(),
+          origArg.getLoc(), /*inputs=*/ValueRange(),
+          /*outputType=*/origArgType, converter);
+      mapping.map(origArg, repl);
+      appendRewrite<ReplaceBlockArgRewrite>(block, origArg);
+      continue;
+    }
 
-    // If inputMap->replacementValue is not nullptr, then the argument is
-    // dropped and a replacement value is provided to be the remappedValue.
-    if (inputMap->replacementValue) {
+    if (Value repl = inputMap->replacementValue) {
+      // This block argument was dropped and a replacement value was provided.
       assert(inputMap->size == 0 &&
              "invalid to provide a replacement value when the argument isn't "
              "dropped");
-      mapping.map(origArg, inputMap->replacementValue);
+      mapping.map(origArg, repl);
       appendRewrite<ReplaceBlockArgRewrite>(block, origArg);
       continue;
     }
 
-    // Otherwise, this is a 1->1+ mapping.
+    // This is a 1->1+ mapping. 1->N mappings are not fully supported in the
+    // dialect conversion. Therefore, we need an argument materialization to
+    // turn the replacement block arguments into a single SSA value that can be
+    // used as a replacement.
     auto replArgs =
         newBlock->getArguments().slice(inputMap->inputNo, inputMap->size);
-    Value newArg;
+    Value argMat = buildUnresolvedMaterialization(
+        MaterializationKind::Argument, newBlock, newBlock->begin(),
+        origArg.getLoc(), /*inputs=*/replArgs, origArgType, converter);
+    mapping.map(origArg, argMat);
+    appendRewrite<ReplaceBlockArgRewrite>(block, origArg);
 
-    // If this is a 1->1 mapping and the types of new and replacement arguments
-    // match (i.e. it's an identity map), then the argument is mapped to its
-    // original type.
     // FIXME: We simply pass through the replacement argument if there wasn't a
     // converter, which isn't great as it allows implicit type conversions to
     // appear. We should properly restructure this code to handle cases where a
     // converter isn't provided and also to properly handle the case where an
     // argument materialization is actually a temporary source materialization
     // (e.g. in the case of 1->N).
-    if (replArgs.size() == 1 &&
-        (!converter || replArgs[0].getType() == origArg.getType())) {
-      newArg = replArgs.front();
-      mapping.map(origArg, newArg);
-    } else {
-      // Build argument materialization: new block arguments -> old block
-      // argument type.
-      Value argMat = buildUnresolvedArgumentMaterialization(
-          newBlock, origArg.getLoc(), replArgs, origArg.getType(), converter);
-      mapping.map(origArg, argMat);
-
-      // Build target materialization: old block argument type -> legal type.
-      // Note: This function returns an "empty" type if no valid conversion to
-      // a legal type exists. In that case, we continue the conversion with the
-      // original block argument type.
-      Type legalOutputType = converter->convertType(origArg.getType());
-      if (legalOutputType && legalOutputType != origArg.getType()) {
-        newArg = buildUnresolvedTargetMaterialization(
-            origArg.getLoc(), argMat, legalOutputType, converter);
-        mapping.map(argMat, newArg);
-      } else {
-        newArg = argMat;
-      }
+    Type legalOutputType;
+    if (converter)
+      legalOutputType = converter->convertType(origArgType);
+    if (legalOutputType && legalOutputType != origArgType) {
+      Value targetMat = buildUnresolvedTargetMaterialization(
+          origArg.getLoc(), argMat, legalOutputType, converter);
+      mapping.map(argMat, targetMat);
     }
-
     appendRewrite<ReplaceBlockArgRewrite>(block, origArg);
-    argInfo[i] = ConvertedArgInfo(inputMap->inputNo, inputMap->size, newArg);
   }
 
-  appendRewrite<BlockTypeConversionRewrite>(newBlock, block, argInfo,
-                                            converter);
+  appendRewrite<BlockTypeConversionRewrite>(newBlock, block, converter);
 
   // Erase the old block. (It is just unlinked for now and will be erased during
   // cleanup.)
@@ -1437,13 +1377,6 @@ Value ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
   appendRewrite<UnresolvedMaterializationRewrite>(convertOp, converter, kind);
   return convertOp.getResult(0);
 }
-Value ConversionPatternRewriterImpl::buildUnresolvedArgumentMaterialization(
-    Block *block, Location loc, ValueRange inputs, Type outputType,
-    const TypeConverter *converter) {
-  return buildUnresolvedMaterialization(MaterializationKind::Argument, block,
-                                        block->begin(), loc, inputs, outputType,
-                                        converter);
-}
 Value ConversionPatternRewriterImpl::buildUnresolvedTargetMaterialization(
     Location loc, Value input, Type outputType,
     const TypeConverter *converter) {
@@ -2862,6 +2795,10 @@ static LogicalResult legalizeUnresolvedMaterialization(
       newMaterialization = converter->materializeTargetConversion(
           rewriter, op->getLoc(), outputType, inputOperands);
       break;
+    case MaterializationKind::Source:
+      newMaterialization = converter->materializeSourceConversion(
+          rewriter, op->getLoc(), outputType, inputOperands);
+      break;
     }
     if (newMaterialization) {
       assert(newMaterialization.getType() == outputType &&
@@ -2874,8 +2811,8 @@ static LogicalResult legalizeUnresolvedMaterialization(
 
   InFlightDiagnostic diag = op->emitError()
                             << "failed to legalize unresolved materialization "
-                               "from "
-                            << inputOperands.getTypes() << " to " << outputType
+                               "from ("
+                            << inputOperands.getTypes() << ") to " << outputType
                             << " that remained live after conversion";
   if (Operation *liveUser = findLiveUser(op->getUsers())) {
     diag.attachNote(liveUser->getLoc())
diff --git a/mlir/test/Transforms/test-legalize-type-conversion.mlir b/mlir/test/Transforms/test-legalize-type-conversion.mlir
index b35cda8e724f6..8254be68912c8 100644
--- a/mlir/test/Transforms/test-legalize-type-conversion.mlir
+++ b/mlir/test/Transforms/test-legalize-type-conversion.mlir
@@ -2,9 +2,8 @@
 
 
 func.func @test_invalid_arg_materialization(
-  // expected-error@below {{failed to materialize conversion for block argument #0 that remained live after conversion, type was 'i16'}}
+  // expected-error@below {{failed to legalize unresolved materialization from () to 'i16' that remained live after conversion}}
   %arg0: i16) {
-  // expected-note@below {{see existing live user here}}
   "foo.return"(%arg0) : (i16) -> ()
 }
 
@@ -104,9 +103,8 @@ func.func @test_block_argument_not_converted() {
 // Make sure argument type changes aren't implicitly forwarded.
 func.func @test_signature_conversion_no_converter() {
   "test.signature_conversion_no_converter"() ({
-  // expected-error@below {{failed to materialize conversion for block argument #0 that remained live after conversion}}
+  // expected-error@below {{failed to legalize unresolved materialization from ('f64') to 'f32' that remained live after conversion}}
   ^bb0(%arg0: f32):
-    // expected-note@below {{see existing live user here}}
     "test.type_consumer"(%arg0) : (f32) -> ()
     "test.return"(%arg0) : (f32) -> ()
   }) : () -> ()

From 62aa596ba19af26f73c292c00d9c7f50df71d836 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson@amd.com>
Date: Sat, 20 Jul 2024 17:26:58 +0900
Subject: [PATCH 275/486] [AMDGPU] Add no return image_sample intrinsics and
 instructions (#97542)

An appropriately configured image resource descriptor can trigger
image_sample instructions to store outputs directly to a linked memory
location instead of returning to VGPRs.

This is opaque to the backend as instruction encoding is unchanged;
however, a mechanism is require to allow frontends to communicate that
these instructions do not require destination VGPRs and store to memory.
Flagging these as stores means they will not be optimized away.
---
 llvm/include/llvm/IR/IntrinsicsAMDGPU.td      |  26 +-
 .../AMDGPU/AMDGPUInstructionSelector.cpp      |   3 +
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |  17 +-
 .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp  |   2 +
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |   3 +-
 llvm/lib/Target/AMDGPU/MIMGInstructions.td    | 223 ++++++--
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  31 +-
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   |   3 +-
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |   2 +
 llvm/lib/Target/AMDGPU/SIInstructions.td      |   8 +
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h |   1 +
 .../AMDGPU/llvm.amdgcn.image.sample.noret.ll  | 479 ++++++++++++++++++
 12 files changed, 746 insertions(+), 52 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll

diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 71b1e832bde3c..ca85ff30f683f 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -769,9 +769,10 @@ class AMDGPUDimProfileCopy<AMDGPUDimProfile base> : AMDGPUDimProfile<base.OpMod,
 
 class AMDGPUDimSampleProfile<string opmod,
                              AMDGPUDimProps dim,
-                             AMDGPUSampleVariant sample> : AMDGPUDimProfile<opmod, dim> {
+                             AMDGPUSampleVariant sample,
+                             bit has_return = true> : AMDGPUDimProfile<opmod, dim> {
   let IsSample = true;
-  let RetTypes = [llvm_any_ty];
+  let RetTypes = !if(has_return, [llvm_any_ty], []);
   let ExtraAddrArgs = sample.ExtraAddrArgs;
   let Offset = sample.Offset;
   let Bias = sample.Bias;
@@ -780,6 +781,12 @@ class AMDGPUDimSampleProfile<string opmod,
   let LodClampMip = sample.LodOrClamp;
 }
 
+class AMDGPUDimSampleNoReturnProfile<string opmod,
+                             AMDGPUDimProps dim,
+                             AMDGPUSampleVariant sample>
+    : AMDGPUDimSampleProfile<opmod, dim, sample, false> {
+}
+
 class AMDGPUDimNoSampleProfile<string opmod,
                                AMDGPUDimProps dim,
                                list<LLVMType> retty,
@@ -970,6 +977,21 @@ defset list<AMDGPUImageDimIntrinsic> AMDGPUImageDimIntrinsics = {
         AMDGPUImageDMaskIntrinsic;
   }
 
+  multiclass AMDGPUImageDimSampleNoReturnDims<string opmod,
+                                      AMDGPUSampleVariant sample> {
+    foreach dim = AMDGPUDims.NoMsaa in {
+      def !strconcat(NAME, "_", dim.Name, "_nortn") : AMDGPUImageDimIntrinsic<
+          AMDGPUDimSampleNoReturnProfile<opmod, dim, sample>,
+          [IntrWillReturn], [SDNPMemOperand]>;
+    }
+  }
+  foreach sample = AMDGPUSampleVariants in {
+    defm int_amdgcn_image_sample # sample.LowerCaseMod
+      : AMDGPUImageDimSampleNoReturnDims<
+        "SAMPLE" # sample.UpperCaseMod # "_nortn", sample>,
+        AMDGPUImageDMaskIntrinsic;
+  }
+
   defm int_amdgcn_image_getlod
     : AMDGPUImageDimSampleDims<"GET_LOD", AMDGPUSample, 1>,
       AMDGPUImageDMaskIntrinsic;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index da3e8c0a62b08..63048c7d1a0a1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1870,6 +1870,8 @@ bool AMDGPUInstructionSelector::selectImageIntrinsic(
       VDataIn = MI.getOperand(1).getReg();
       VDataTy = MRI->getType(VDataIn);
       NumVDataDwords = (VDataTy.getSizeInBits() + 31) / 32;
+    } else if (BaseOpcode->NoReturn) {
+      NumVDataDwords = 0;
     } else {
       VDataOut = MI.getOperand(0).getReg();
       VDataTy = MRI->getType(VDataOut);
@@ -3616,6 +3618,7 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
     return selectG_INSERT_VECTOR_ELT(I);
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
+  case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
     const AMDGPU::ImageDimIntrinsicInfo *Intr =
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 88e40da110555..89ef0f299feab 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6334,8 +6334,13 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
   const LLT V2S16 = LLT::fixed_vector(2, 16);
 
   unsigned DMask = 0;
-  Register VData = MI.getOperand(NumDefs == 0 ? 1 : 0).getReg();
-  LLT Ty = MRI->getType(VData);
+  Register VData;
+  LLT Ty;
+
+  if (!BaseOpcode->NoReturn || BaseOpcode->Store) {
+    VData = MI.getOperand(NumDefs == 0 ? 1 : 0).getReg();
+    Ty = MRI->getType(VData);
+  }
 
   const bool IsAtomicPacked16Bit =
       (BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
@@ -6373,7 +6378,11 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
                                      : AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE;
   const unsigned LoadOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16
                                     : AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD;
-  unsigned NewOpcode = NumDefs == 0 ? StoreOpcode : LoadOpcode;
+  unsigned NewOpcode = LoadOpcode;
+  if (BaseOpcode->Store)
+    NewOpcode = StoreOpcode;
+  else if (BaseOpcode->NoReturn)
+    NewOpcode = AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET;
 
   // Track that we legalized this
   MI.setDesc(B.getTII().get(NewOpcode));
@@ -6503,7 +6512,7 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
     Flags |= 2;
   MI.addOperand(MachineOperand::CreateImm(Flags));
 
-  if (BaseOpcode->Store) { // No TFE for stores?
+  if (BaseOpcode->NoReturn) { // No TFE for stores?
     // TODO: Handle dmask trim
     if (!Ty.isVector() || !IsD16)
       return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 73796edb5d3e3..68f4767458703 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3172,6 +3172,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
   }
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
+  case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
     const AMDGPU::RsrcIntrinsic *RSrcIntrin =
@@ -4842,6 +4843,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   }
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
+  case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
     auto IntrID = AMDGPU::getIntrinsicID(MI);
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 217487b2cc7e6..92c3b26ca4d6f 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -3868,7 +3868,8 @@ bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst,
   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
   int TFEIdx   = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);
 
-  assert(VDataIdx != -1);
+  if (VDataIdx == -1 && isGFX10Plus()) // no return image_sample
+    return true;
 
   if ((DMaskIdx == -1 || TFEIdx == -1) && isGFX10_AEncoding()) // intersect_ray
     return true;
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 15fd36ebd10a4..b4e58cfd98a23 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -51,6 +51,7 @@ class MIMGBaseOpcode : PredicateControl {
   bit MSAA = 0;
   bit BVH = 0;
   bit A16 = 0;
+  bit NoReturn = 0;
 }
 
 def MIMGBaseOpcode : GenericEnum {
@@ -62,7 +63,7 @@ def MIMGBaseOpcodesTable : GenericTable {
   let CppTypeName = "MIMGBaseOpcodeInfo";
   let Fields = ["BaseOpcode", "Store", "Atomic", "AtomicX2", "Sampler",
                 "Gather4", "NumExtraArgs", "Gradients", "G16", "Coordinates",
-                "LodOrClampOrMip", "HasD16", "MSAA", "BVH", "A16"];
+                "LodOrClampOrMip", "HasD16", "MSAA", "BVH", "A16", "NoReturn"];
   string TypeOf_BaseOpcode = "MIMGBaseOpcode";
 
   let PrimaryKey = ["BaseOpcode"];
@@ -521,6 +522,25 @@ class VSAMPLE_Sampler_gfx12<mimgopc op, string opcode, RegisterClass DataRC,
                     #!if(BaseOpcode.HasD16, "$d16", "");
 }
 
+class VSAMPLE_Sampler_nortn_gfx12<mimgopc op, string opcode,
+                            int num_addrs, RegisterClass Addr3RC = VGPR_32,
+                            string dns="">
+  : VSAMPLE_gfx12<op.GFX12, (outs), num_addrs, dns, Addr3RC> {
+  let InOperandList = !con(AddrIns,
+                           (ins SReg_256:$rsrc),
+                           !if(BaseOpcode.Sampler, (ins SReg_128:$samp), (ins)),
+                           (ins DMask:$dmask, Dim:$dim, UNorm:$unorm,
+                                CPol:$cpol, R128A16:$r128, A16:$a16, TFE:$tfe,
+                                LWE:$lwe),
+                           !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));
+  let AsmString = opcode#" off, "#AddrAsm#", $rsrc"
+                    #!if(BaseOpcode.Sampler, ", $samp", "")
+                    #"$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"
+                    #!if(BaseOpcode.HasD16, "$d16", "");
+  // Force vdata to VGPR0 as no result will be returned.
+  let vdata = 0;
+}
+
 multiclass MIMG_NoSampler_Src_Helper <mimgopc op, string asm,
                                       RegisterClass dst_rc, bit enableDisasm,
                                       bit ExtendedImageInst = 1,
@@ -835,6 +855,7 @@ multiclass MIMG_Store <mimgopc op, string asm, bit has_d16, bit mip = 0> {
     let Store = 1;
     let LodOrClampOrMip = mip;
     let HasD16 = has_d16;
+    let NoReturn = 1;
   }
 
   let BaseOpcode = !cast<MIMGBaseOpcode>(NAME) in {
@@ -1136,44 +1157,62 @@ class MIMG_Sampler_gfx90a<mimgopc op, string asm, RegisterClass dst_rc,
                       #!if(BaseOpcode.HasD16, "$d16", "");
 }
 
+class MIMG_Sampler_OpList_gfx10p<dag OpPrefix, bit HasD16> {
+  dag ret = !con(OpPrefix,
+                 (ins SReg_256:$srsrc, SReg_128:$ssamp,
+                  DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,
+                  R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),
+                 !if(HasD16, (ins D16:$d16), (ins)));
+}
+
+class MIMG_Sampler_Asm_gfx10p<string opcode, string AsmPrefix, bit HasD16> {
+  string ret = opcode#" "#AsmPrefix#", $srsrc, $ssamp$dmask$dim$unorm"
+               #"$cpol$r128$a16$tfe$lwe"
+               #!if(HasD16, "$d16", "");
+}
+
 class MIMG_Sampler_gfx10<mimgopc op, string opcode,
                          RegisterClass DataRC, RegisterClass AddrRC,
                          string dns="">
   : MIMG_gfx10<op.GFX10M, (outs DataRC:$vdata), dns> {
-  let InOperandList = !con((ins AddrRC:$vaddr0, SReg_256:$srsrc, SReg_128:$ssamp,
-                                DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,
-                                R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),
-                           !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));
-  let AsmString = opcode#" $vdata, $vaddr0, $srsrc, $ssamp$dmask$dim$unorm"
-                    #"$cpol$r128$a16$tfe$lwe"
-                    #!if(BaseOpcode.HasD16, "$d16", "");
+  let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;
+  let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "$vdata, $vaddr0", BaseOpcode.HasD16>.ret;
 }
 
 class MIMG_Sampler_nsa_gfx10<mimgopc op, string opcode,
                              RegisterClass DataRC, int num_addrs,
                              string dns="">
   : MIMG_nsa_gfx10<op.GFX10M, (outs DataRC:$vdata), num_addrs, dns> {
-  let InOperandList = !con(AddrIns,
-                           (ins SReg_256:$srsrc, SReg_128:$ssamp, DMask:$dmask,
-                                Dim:$dim, UNorm:$unorm, CPol:$cpol,
-                                R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),
-                           !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));
-  let AsmString = opcode#" $vdata, "#AddrAsm#", $srsrc, $ssamp$dmask$dim$unorm"
-                    #"$cpol$r128$a16$tfe$lwe"
-                    #!if(BaseOpcode.HasD16, "$d16", "");
+  let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;
+  let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, " $vdata, "#AddrAsm, BaseOpcode.HasD16>.ret;
+}
+
+class MIMG_Sampler_nortn_gfx10<mimgopc op, string opcode,
+                         RegisterClass AddrRC,
+                         string dns="">
+  : MIMG_gfx10<op.GFX10M, (outs), dns> {
+  let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;
+  let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "off, $vaddr0", BaseOpcode.HasD16>.ret;
+  // Force vdata to VGPR0 as no result will be returned.
+  let vdata = 0;
+}
+
+class MIMG_Sampler_nortn_nsa_gfx10<mimgopc op, string opcode,
+                         int num_addrs,
+                         string dns="">
+  : MIMG_nsa_gfx10<op.GFX10M, (outs), num_addrs, dns> {
+  let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;
+  let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, " off, "#AddrAsm, BaseOpcode.HasD16>.ret;
+  // Force vdata to VGPR0 as no result will be returned.
+  let vdata = 0;
 }
 
 class MIMG_Sampler_gfx11<mimgopc op, string opcode,
                          RegisterClass DataRC, RegisterClass AddrRC,
                          string dns="">
   : MIMG_gfx11<op.GFX11, (outs DataRC:$vdata), dns> {
-  let InOperandList = !con((ins AddrRC:$vaddr0, SReg_256:$srsrc, SReg_128:$ssamp,
-                                DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,
-                                R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),
-                           !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));
-  let AsmString = opcode#" $vdata, $vaddr0, $srsrc, $ssamp$dmask$dim$unorm"
-                    #"$cpol$r128$a16$tfe$lwe"
-                    #!if(BaseOpcode.HasD16, "$d16", "");
+  let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;
+  let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "$vdata, $vaddr0", BaseOpcode.HasD16>.ret;
 }
 
 class MIMG_Sampler_nsa_gfx11<mimgopc op, string opcode,
@@ -1181,14 +1220,26 @@ class MIMG_Sampler_nsa_gfx11<mimgopc op, string opcode,
                              RegisterClass LastVAddrSize, string dns="">
   : MIMG_nsa_gfx11<op.GFX11, (outs DataRC:$vdata), num_addrs, dns, [],
                    LastVAddrSize> {
-  let InOperandList = !con(AddrIns,
-                           (ins SReg_256:$srsrc, SReg_128:$ssamp, DMask:$dmask,
-                                Dim:$dim, UNorm:$unorm, CPol:$cpol,
-                                R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),
-                           !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));
-  let AsmString = opcode#" $vdata, "#AddrAsm#", $srsrc, $ssamp$dmask$dim$unorm"
-                    #"$cpol$r128$a16$tfe$lwe"
-                    #!if(BaseOpcode.HasD16, "$d16", "");
+  let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;
+  let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, " $vdata, "#AddrAsm, BaseOpcode.HasD16>.ret;
+}
+
+class MIMG_Sampler_nortn_gfx11<mimgopc op, string opcode,
+                                  RegisterClass AddrRC,
+                                  string dns="">
+  : MIMG_gfx11<op.GFX11, (outs), dns> {
+  let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;
+  let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "off, $vaddr0", BaseOpcode.HasD16>.ret;
+  let vdata = 0;
+}
+
+class MIMG_Sampler_nortn_nsa_gfx11<mimgopc op, string opcode,
+                                      int num_addrs,
+                                      RegisterClass LastVAddrSize, string dns="">
+  : MIMG_nsa_gfx11<op.GFX11, (outs), num_addrs, dns, [], LastVAddrSize> {
+  let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;
+  let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "off, "#AddrAsm, BaseOpcode.HasD16>.ret;
+  let vdata = 0;
 }
 
 class MIMGAddrSize<int dw, bit enable_disasm, int AddrDW = dw> {
@@ -1366,6 +1417,57 @@ class MIMG_Sampler_BaseOpcode<AMDGPUSampleVariant sample>
   let LodOrClampOrMip = !ne(sample.LodOrClamp, "");
 }
 
+multiclass MIMG_Sampler_NoReturn <mimgopc op, AMDGPUSampleVariant sample, bit wqm = 0, bit isG16, string asm> {
+  def "" : MIMG_Sampler_BaseOpcode<sample> {
+    let HasD16 = 1;
+    let G16 = isG16;
+    let NoReturn = 1;
+  }
+
+  let BaseOpcode = !cast<MIMGBaseOpcode>(NAME), WQM = wqm,
+      mayLoad = 1, mayStore = 1, VDataDwords = 0 in {
+    foreach addr = MIMG_Sampler_AddrSizes<sample, isG16>.MachineInstrs in {
+      let VAddrDwords = addr.NumWords in {
+        if op.HAS_GFX10M then {
+          def _V # addr.NumWords # _gfx10
+            : MIMG_Sampler_nortn_gfx10 <op, asm, addr.RegClass>;
+        }
+        if op.HAS_GFX11 then {
+          def _V # addr.NumWords # _gfx11
+            : MIMG_Sampler_nortn_gfx11 <op, asm, addr.RegClass>;
+        }
+      }
+    }
+
+    foreach addr = MIMG_Sampler_AddrSizes<sample, isG16>.NSAInstrs in {
+      let VAddrDwords = addr.NumWords in {
+        if op.HAS_GFX10M then {
+          def _V # addr.NumWords # _nsa_gfx10
+            : MIMG_Sampler_nortn_nsa_gfx10<op, asm, addr.NumWords>;
+        }
+      }
+    }
+
+    foreach addr = MIMG_Sampler_AddrSizes<sample, isG16, 5/*MaxNSASize*/>.PartialNSAInstrs in {
+      let VAddrDwords = addr.NumWords in {
+        if op.HAS_GFX11 then {
+          def _V # addr.NumWords # _nsa_gfx11
+            : MIMG_Sampler_nortn_nsa_gfx11<op, asm, addr.NumWords, addr.RegClass>;
+        }
+      }
+    }
+
+    foreach addr = MIMG_Sampler_AddrSizes<sample, isG16, 4/*MaxNSASize*/, 1>.PartialNSAInstrs in {
+      let VAddrDwords = addr.NumWords in {
+        if op.HAS_GFX12 then {
+          def _V # addr.NumWords # _gfx12
+            : VSAMPLE_Sampler_nortn_gfx12<op, asm, addr.NumWords, addr.RegClass>;
+        }
+      }
+    }
+  }
+}
+
 multiclass MIMG_Sampler <mimgopc op, AMDGPUSampleVariant sample, bit wqm = 0,
                          bit isG16 = 0, bit isGetLod = 0,
                          string asm = "image_sample"#sample.LowerCaseMod#!if(isG16, "_g16", ""),
@@ -1388,6 +1490,9 @@ multiclass MIMG_Sampler <mimgopc op, AMDGPUSampleVariant sample, bit wqm = 0,
     let VDataDwords = 5 in
     defm _V5 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_160, 0, ExtendedImageInst, isG16>;
   }
+
+  if !not(isGetLod) then
+  defm "_nortn" : MIMG_Sampler_NoReturn <op, sample, wqm, isG16, asm>;
 }
 
 multiclass MIMG_Sampler_WQM <mimgopc op, AMDGPUSampleVariant sample>
@@ -1755,6 +1860,10 @@ def : MIMGLZMapping<IMAGE_GATHER4_L, IMAGE_GATHER4_LZ>;
 def : MIMGLZMapping<IMAGE_GATHER4_C_L, IMAGE_GATHER4_C_LZ>;
 def : MIMGLZMapping<IMAGE_GATHER4_L_O, IMAGE_GATHER4_LZ_O>;
 def : MIMGLZMapping<IMAGE_GATHER4_C_L_O, IMAGE_GATHER4_C_LZ_O>;
+def : MIMGLZMapping<IMAGE_SAMPLE_L_nortn, IMAGE_SAMPLE_LZ_nortn>;
+def : MIMGLZMapping<IMAGE_SAMPLE_C_L_nortn, IMAGE_SAMPLE_C_LZ_nortn>;
+def : MIMGLZMapping<IMAGE_SAMPLE_L_O_nortn, IMAGE_SAMPLE_LZ_O_nortn>;
+def : MIMGLZMapping<IMAGE_SAMPLE_C_L_O_nortn, IMAGE_SAMPLE_C_LZ_O_nortn>;
 
 // MIP to NONMIP Optimization Mapping
 def : MIMGMIPMapping<IMAGE_LOAD_MIP, IMAGE_LOAD>;
@@ -1777,6 +1886,14 @@ def : MIMGBiasMapping<IMAGE_GATHER4_B_O, IMAGE_GATHER4_O>;
 def : MIMGBiasMapping<IMAGE_GATHER4_B_CL_O, IMAGE_GATHER4_CL_O>;
 def : MIMGBiasMapping<IMAGE_GATHER4_C_B_O, IMAGE_GATHER4_C_O>;
 def : MIMGBiasMapping<IMAGE_GATHER4_C_B_CL_O, IMAGE_GATHER4_C_CL_O>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_B_nortn, IMAGE_SAMPLE_nortn>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_B_CL_nortn, IMAGE_SAMPLE_CL_nortn>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_nortn, IMAGE_SAMPLE_C_nortn>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_CL_nortn, IMAGE_SAMPLE_C_CL_nortn>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_B_O_nortn, IMAGE_SAMPLE_O_nortn>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_B_CL_O_nortn, IMAGE_SAMPLE_CL_O_nortn>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_O_nortn, IMAGE_SAMPLE_C_O_nortn>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_CL_O_nortn, IMAGE_SAMPLE_C_CL_O_nortn>;
 
 // Offset to NoOffset Optimization Mapping
 def : MIMGOffsetMapping<IMAGE_SAMPLE_O, IMAGE_SAMPLE>;
@@ -1819,6 +1936,34 @@ def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_O_G16, IMAGE_SAMPLE_CD_G16>;
 def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_CL_O_G16, IMAGE_SAMPLE_CD_CL_G16>;
 def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_O_G16, IMAGE_SAMPLE_C_CD_G16>;
 def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_CL_O_G16, IMAGE_SAMPLE_C_CD_CL_G16>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_O_nortn, IMAGE_SAMPLE_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_CL_O_nortn, IMAGE_SAMPLE_CL_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_D_O_nortn, IMAGE_SAMPLE_D_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_D_CL_O_nortn, IMAGE_SAMPLE_D_CL_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_D_O_G16_nortn, IMAGE_SAMPLE_D_G16_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_D_CL_O_G16_nortn, IMAGE_SAMPLE_D_CL_G16_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_L_O_nortn, IMAGE_SAMPLE_L_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_B_O_nortn, IMAGE_SAMPLE_B_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_B_CL_O_nortn, IMAGE_SAMPLE_B_CL_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_LZ_O_nortn, IMAGE_SAMPLE_LZ_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_O_nortn, IMAGE_SAMPLE_C_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CL_O_nortn, IMAGE_SAMPLE_C_CL_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_O_nortn, IMAGE_SAMPLE_C_D_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_CL_O_nortn, IMAGE_SAMPLE_C_D_CL_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_O_G16_nortn, IMAGE_SAMPLE_C_D_G16_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_CL_O_G16_nortn, IMAGE_SAMPLE_C_D_CL_G16_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_L_O_nortn, IMAGE_SAMPLE_C_L_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_B_CL_O_nortn, IMAGE_SAMPLE_C_B_CL_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_B_O_nortn, IMAGE_SAMPLE_C_B_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_LZ_O_nortn, IMAGE_SAMPLE_C_LZ_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_O_nortn, IMAGE_SAMPLE_CD>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_CL_O_nortn, IMAGE_SAMPLE_CD_CL_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_O_nortn, IMAGE_SAMPLE_C_CD_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_CL_O_nortn, IMAGE_SAMPLE_C_CD_CL_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_O_G16_nortn, IMAGE_SAMPLE_CD_G16_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_CL_O_G16_nortn, IMAGE_SAMPLE_CD_CL_G16_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_O_G16_nortn, IMAGE_SAMPLE_C_CD_G16_nortn>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_CL_O_G16_nortn, IMAGE_SAMPLE_C_CD_CL_G16_nortn>;
 
 // G to G16 Optimization Mapping
 def : MIMGG16Mapping<IMAGE_SAMPLE_D, IMAGE_SAMPLE_D_G16>;
@@ -1837,3 +1982,19 @@ def : MIMGG16Mapping<IMAGE_SAMPLE_CD_O, IMAGE_SAMPLE_CD_O_G16>;
 def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_O, IMAGE_SAMPLE_CD_CL_O_G16>;
 def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_O, IMAGE_SAMPLE_C_CD_O_G16>;
 def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_O, IMAGE_SAMPLE_C_CD_CL_O_G16>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_D_nortn, IMAGE_SAMPLE_D_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_D_CL_nortn, IMAGE_SAMPLE_D_CL_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_nortn, IMAGE_SAMPLE_C_D_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_CL_nortn, IMAGE_SAMPLE_C_D_CL_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_D_O_nortn, IMAGE_SAMPLE_D_O_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_D_CL_O_nortn, IMAGE_SAMPLE_D_CL_O_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_O_nortn, IMAGE_SAMPLE_C_D_O_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_CL_O_nortn, IMAGE_SAMPLE_C_D_CL_O_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_CD_nortn, IMAGE_SAMPLE_CD_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_nortn, IMAGE_SAMPLE_CD_CL_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_nortn, IMAGE_SAMPLE_C_CD_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_nortn, IMAGE_SAMPLE_C_CD_CL_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_CD_O_nortn, IMAGE_SAMPLE_CD_O_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_O_nortn, IMAGE_SAMPLE_CD_CL_O_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_O_nortn, IMAGE_SAMPLE_C_CD_O_G16_nortn>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_O_nortn, IMAGE_SAMPLE_C_CD_CL_O_G16_nortn>;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index d5ffb4478bee1..a09e0ad2c0c29 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1190,8 +1190,13 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     // TODO: Should images get their own address space?
     Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
 
-    if (RsrcIntr->IsImage)
+    const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
+    if (RsrcIntr->IsImage) {
+      const AMDGPU::ImageDimIntrinsicInfo *Intr =
+          AMDGPU::getImageDimIntrinsicInfo(IntrID);
+      BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
       Info.align.reset();
+    }
 
     Value *RsrcArg = CI.getArgOperand(RsrcIntr->RsrcArg);
     if (auto *RsrcPtrTy = dyn_cast<PointerType>(RsrcArg->getType())) {
@@ -1212,11 +1217,6 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
       if (RsrcIntr->IsImage) {
         unsigned MaxNumLanes = 4;
 
-        const AMDGPU::ImageDimIntrinsicInfo *Intr
-          = AMDGPU::getImageDimIntrinsicInfo(IntrID);
-        const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
-          AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
-
         if (!BaseOpcode->Gather4) {
           // If this isn't a gather, we may have excess loaded elements in the
           // IR type. Check the dmask for the real number of elements loaded.
@@ -1250,7 +1250,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 
       Info.flags |= MachineMemOperand::MOStore;
     } else {
-      // Atomic
+      // Atomic or NoReturn Sampler
       Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID :
                                             ISD::INTRINSIC_W_CHAIN;
       Info.flags |= MachineMemOperand::MOLoad |
@@ -1259,9 +1259,14 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 
       switch (IntrID) {
       default:
-        Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
-        // XXX - Should this be volatile without known ordering?
-        Info.flags |= MachineMemOperand::MOVolatile;
+        if (RsrcIntr->IsImage && BaseOpcode->NoReturn) {
+          // Fake memory access type for no return sampler intrinsics
+          Info.memVT = MVT::i32;
+        } else {
+          // XXX - Should this be volatile without known ordering?
+          Info.flags |= MachineMemOperand::MOVolatile;
+          Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
+        }
         break;
       case Intrinsic::amdgcn_raw_buffer_load_lds:
       case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
@@ -7900,7 +7905,7 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
   bool IsG16 = false;
   bool IsA16 = false;
   SDValue VData;
-  int NumVDataDwords;
+  int NumVDataDwords = 0;
   bool AdjustRetType = false;
   bool IsAtomicPacked16Bit = false;
 
@@ -7949,7 +7954,7 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
       }
 
       NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
-    } else {
+    } else if (!BaseOpcode->NoReturn) {
       // Work out the num dwords based on the dmask popcount and underlying type
       // and whether packing is supported.
       MVT LoadVT = ResultTypes[0].getSimpleVT();
@@ -8242,7 +8247,7 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
     DAG.ExtractVectorElements(SDValue(NewNode, 0), Elt, 0, 1);
     return DAG.getMergeValues({Elt[0], SDValue(NewNode, 1)}, DL);
   }
-  if (BaseOpcode->Store)
+  if (BaseOpcode->NoReturn)
     return SDValue(NewNode, 0);
   return constructRetValue(DAG, NewNode, OrigResultTypes, IsTexFail,
                            Subtarget->hasUnpackedD16VMem(), IsD16, DMaskLanes,
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index a18da72b02ebe..1315aa0855788 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -699,7 +699,8 @@ class SIInsertWaitcnts : public MachineFunctionPass {
     // these should use VM_CNT.
     if (!ST->hasVscnt() || SIInstrInfo::mayWriteLDSThroughDMA(Inst))
       return VMEM_ACCESS;
-    if (Inst.mayStore() && !SIInstrInfo::isAtomicRet(Inst)) {
+    if (Inst.mayStore() &&
+        (!Inst.mayLoad() || SIInstrInfo::isAtomicNoRet(Inst))) {
       // FLAT and SCRATCH instructions may access scratch. Other VMEM
       // instructions do not.
       if (SIInstrInfo::isFLAT(Inst) && mayAccessScratchThroughFlat(Inst))
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 52044791e6c66..463737f645d45 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -472,6 +472,8 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
     Offset = 0;
     // Get appropriate operand, and compute width accordingly.
     DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
+    if (DataOpIdx == -1)
+      return false; // no return sampler
     Width = getOpSize(LdSt, DataOpIdx);
     return true;
   }
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 2e617e5646c59..15078bc941292 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3953,6 +3953,14 @@ def G_AMDGPU_INTRIN_IMAGE_LOAD_D16 : AMDGPUGenericInstruction {
   let mayStore = 1;
 }
 
+def G_AMDGPU_INTRIN_IMAGE_LOAD_NORET : AMDGPUGenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins unknown:$intrin, variable_ops);
+  let hasSideEffects = 0;
+  let mayLoad = 1;
+  let mayStore = 1;
+}
+
 // This is equivalent to the G_INTRINSIC*, but the operands may have
 // been legalized depending on the subtarget requirements.
 def G_AMDGPU_INTRIN_IMAGE_STORE : AMDGPUGenericInstruction {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index af2f0bc1a6306..429c3ad335d21 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -404,6 +404,7 @@ struct MIMGBaseOpcodeInfo {
   bool MSAA;
   bool BVH;
   bool A16;
+  bool NoReturn;
 };
 
 LLVM_READONLY
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
new file mode 100644
index 0000000000000..90dfab501d0a4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
@@ -0,0 +1,479 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10PLUS-SDAG,GFX10,GFX10-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10PLUS-GISEL,GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10PLUS-SDAG,GFX11,GFX11-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10PLUS-GISEL,GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+
+define amdgpu_ps void @sample_1d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+; GFX10PLUS-LABEL: sample_1d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_1d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.1d.nortn.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_2d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
+; GFX10PLUS-LABEL: sample_2d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    image_sample off, v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_2d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    image_sample off, [v0, v1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.2d.nortn.f32(i32 15, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_3d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %r) {
+; GFX10PLUS-LABEL: sample_3d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    image_sample off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_3d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    image_sample off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.3d.nortn.f32(i32 15, float %s, float %t, float %r, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_cube_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %face) {
+; GFX10PLUS-LABEL: sample_cube_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    image_sample off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_cube_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    image_sample off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.cube.nortn.f32(i32 15, float %s, float %t, float %face, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_1darray_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %slice) {
+; GFX10PLUS-LABEL: sample_1darray_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    image_sample off, v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_1darray_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    image_sample off, [v0, v1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.1darray.nortn.f32(i32 15, float %s, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_2darray_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %slice) {
+; GFX10PLUS-LABEL: sample_2darray_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    image_sample off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_2darray_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    image_sample off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.2darray.nortn.f32(i32 15, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_b_1d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s) {
+; GFX10PLUS-LABEL: sample_b_1d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    image_sample_b off, v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_b_1d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    image_sample_b off, [v0, v1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.b.1d.nortn.f32(i32 15, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_b_2d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t) {
+; GFX10PLUS-LABEL: sample_b_2d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    image_sample_b off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_b_2d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    image_sample_b off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.b.2d.nortn.f32(i32 15, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_c_1d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s) {
+; GFX10PLUS-LABEL: sample_c_1d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    image_sample_c off, v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_c_1d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    image_sample_c off, [v0, v1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.c.1d.nortn.f32(i32 15, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_c_2d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t) {
+; GFX10PLUS-LABEL: sample_c_2d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    image_sample_c off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_c_2d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    image_sample_c off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.c.2d.nortn.f32(i32 15, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_d_1d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dsdv, float %s) {
+; GFX10PLUS-LABEL: sample_d_1d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    image_sample_d off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_d_1d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    image_sample_d off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.d.1d.nortn.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_d_2d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t) {
+; GFX10PLUS-LABEL: sample_d_2d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    image_sample_d off, v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_d_2d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    image_sample_d off, [v0, v1, v2, v[3:5]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.d.2d.nortn.f32.f32(i32 15, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_l_1d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %lod) {
+; GFX10PLUS-LABEL: sample_l_1d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    image_sample_l off, v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_l_1d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    image_sample_l off, [v0, v1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.l.1d.nortn.f32(i32 15, float %s, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @sample_l_2d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %lod) {
+; GFX10PLUS-LABEL: sample_l_2d_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    image_sample_l off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_l_2d_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    image_sample_l off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.l.2d.nortn.f32(i32 15, float %s, float %t, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps <4 x float> @sample_nortn_mix_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+; GFX10PLUS-LABEL: sample_nortn_mix_1:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0)
+; GFX10PLUS-NEXT:    ; return to shader part epilog
+;
+; GFX12-LABEL: sample_nortn_mix_1:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_mov_b32 s12, exec_lo
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX12-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    ; return to shader part epilog
+main_body:
+  call void @llvm.amdgcn.image.sample.1d.nortn.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_nortn_mix_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+; GFX10PLUS-LABEL: sample_nortn_mix_2:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10PLUS-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT:    v_mov_b32_e32 v4, v0
+; GFX10PLUS-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0)
+; GFX10PLUS-NEXT:    ; return to shader part epilog
+;
+; GFX12-LABEL: sample_nortn_mix_2:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_mov_b32 s12, exec_lo
+; GFX12-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT:    v_mov_b32_e32 v4, v0
+; GFX12-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX12-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  call void @llvm.amdgcn.image.sample.1d.nortn.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_nortn_mix_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+; GFX10PLUS-SDAG-LABEL: sample_nortn_mix_3:
+; GFX10PLUS-SDAG:       ; %bb.0: ; %main_body
+; GFX10PLUS-SDAG-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10PLUS-SDAG-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-SDAG-NEXT:    image_sample v1, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-SDAG-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX10PLUS-SDAG-NEXT:    image_sample v[0:3], v1, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX10PLUS-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX10PLUS-GISEL-LABEL: sample_nortn_mix_3:
+; GFX10PLUS-GISEL:       ; %bb.0: ; %main_body
+; GFX10PLUS-GISEL-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10PLUS-GISEL-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-GISEL-NEXT:    image_sample v[1:4], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-GISEL-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX10PLUS-GISEL-NEXT:    image_sample v[0:3], v1, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX10PLUS-GISEL-NEXT:    ; return to shader part epilog
+;
+; GFX12-SDAG-LABEL: sample_nortn_mix_3:
+; GFX12-SDAG:       ; %bb.0: ; %main_body
+; GFX12-SDAG-NEXT:    s_mov_b32 s12, exec_lo
+; GFX12-SDAG-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-SDAG-NEXT:    image_sample v1, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX12-SDAG-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x1
+; GFX12-SDAG-NEXT:    image_sample v[0:3], v1, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: sample_nortn_mix_3:
+; GFX12-GISEL:       ; %bb.0: ; %main_body
+; GFX12-GISEL-NEXT:    s_mov_b32 s12, exec_lo
+; GFX12-GISEL-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-GISEL-NEXT:    image_sample v[1:4], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX12-GISEL-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x1
+; GFX12-GISEL-NEXT:    image_sample v[0:3], v1, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v.0 = extractelement <4 x float> %v, i32 0
+  call void @llvm.amdgcn.image.sample.1d.nortn.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %u = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %v.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret <4 x float> %u
+}
+
+define amdgpu_ps <4 x float> @sample_nortn_mix_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+; GFX10PLUS-SDAG-LABEL: sample_nortn_mix_4:
+; GFX10PLUS-SDAG:       ; %bb.0: ; %main_body
+; GFX10PLUS-SDAG-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10PLUS-SDAG-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-SDAG-NEXT:    image_sample v4, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX10PLUS-SDAG-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT:    image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-SDAG-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT:    s_waitcnt vmcnt(2)
+; GFX10PLUS-SDAG-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX10PLUS-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX10PLUS-GISEL-LABEL: sample_nortn_mix_4:
+; GFX10PLUS-GISEL:       ; %bb.0: ; %main_body
+; GFX10PLUS-GISEL-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10PLUS-GISEL-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-GISEL-NEXT:    image_sample v[4:7], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX10PLUS-GISEL-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT:    image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-GISEL-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT:    s_waitcnt vmcnt(2)
+; GFX10PLUS-GISEL-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX10PLUS-GISEL-NEXT:    ; return to shader part epilog
+;
+; GFX12-SDAG-LABEL: sample_nortn_mix_4:
+; GFX12-SDAG:       ; %bb.0: ; %main_body
+; GFX12-SDAG-NEXT:    s_mov_b32 s12, exec_lo
+; GFX12-SDAG-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-SDAG-NEXT:    image_sample v4, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x1
+; GFX12-SDAG-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT:    image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX12-SDAG-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x2
+; GFX12-SDAG-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: sample_nortn_mix_4:
+; GFX12-GISEL:       ; %bb.0: ; %main_body
+; GFX12-GISEL-NEXT:    s_mov_b32 s12, exec_lo
+; GFX12-GISEL-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX12-GISEL-NEXT:    image_sample v[4:7], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x1
+; GFX12-GISEL-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT:    image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX12-GISEL-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT:    image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x2
+; GFX12-GISEL-NEXT:    image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v.0 = extractelement <4 x float> %v, i32 0
+  %v.1 = extractelement <4 x float> %v, i32 0
+  %v.2 = extractelement <4 x float> %v, i32 0
+  %v.3 = extractelement <4 x float> %v, i32 0
+  call void @llvm.amdgcn.image.sample.1d.nortn.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  call void @llvm.amdgcn.image.sample.1d.nortn.f32(i32 15, float %v.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %u = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %v.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %u.0 = extractelement <4 x float> %u, i32 0
+  call void @llvm.amdgcn.image.sample.1d.nortn.f32(i32 15, float %v.2, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  call void @llvm.amdgcn.image.sample.1d.nortn.f32(i32 15, float %v.3, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  call void @llvm.amdgcn.image.sample.1d.nortn.f32(i32 15, float %u.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret <4 x float> %u
+}
+
+define amdgpu_ps void @sample_d_1d_g16_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
+; GFX10PLUS-LABEL: sample_d_1d_g16_nortn:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    image_sample_d_g16 off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT:    s_endpgm
+;
+; GFX12-LABEL: sample_d_1d_g16_nortn:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    image_sample_d_g16 off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.sample.d.1d.nortn.f16.f32(i32 15, half %dsdh, half %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.image.sample.1d.nortn.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.2d.nortn.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.3d.nortn.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.cube.nortn.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.1darray.nortn.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.2darray.nortn.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+declare void @llvm.amdgcn.image.sample.b.1d.nortn.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.b.2d.nortn.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+declare void @llvm.amdgcn.image.sample.c.1d.nortn.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.c.2d.nortn.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+declare void @llvm.amdgcn.image.sample.d.1d.f32.nortn.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.d.2d.f32.nortn.f32(i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+declare void @llvm.amdgcn.image.sample.l.1d.nortn.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.l.2d.nortn.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare void @llvm.amdgcn.image.sample.d.1d.nortn.f16.f32(i32, half, half, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
+; GFX10-GISEL: {{.*}}
+; GFX10-SDAG: {{.*}}
+; GFX11: {{.*}}
+; GFX11-GISEL: {{.*}}
+; GFX11-SDAG: {{.*}}

From 155f6b49d90357d4062aa97f035f42617565ee26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sat, 20 Jul 2024 10:39:06 +0200
Subject: [PATCH 276/486] [clang][Interp] Fix reporting invalid new/delete
 expressions

This should be a CCEDiag call and we do *not* abort because of it.
---
 clang/lib/AST/Interp/Interp.cpp      | 4 ++--
 clang/test/AST/Interp/new-delete.cpp | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index cd6fc60400ebd..6fcd90e5f5849 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -728,8 +728,8 @@ bool CheckDynamicMemoryAllocation(InterpState &S, CodePtr OpPC) {
     return true;
 
   const SourceInfo &E = S.Current->getSource(OpPC);
-  S.FFDiag(E, diag::note_constexpr_new);
-  return false;
+  S.CCEDiag(E, diag::note_constexpr_new);
+  return true;
 }
 
 bool CheckNewDeleteForms(InterpState &S, CodePtr OpPC, bool NewWasArray,
diff --git a/clang/test/AST/Interp/new-delete.cpp b/clang/test/AST/Interp/new-delete.cpp
index cb46426c0e3be..7a85def784920 100644
--- a/clang/test/AST/Interp/new-delete.cpp
+++ b/clang/test/AST/Interp/new-delete.cpp
@@ -560,4 +560,9 @@ constexpr int a() { // both-error {{never produces a constant expression}}
 }
 static_assert(a() == 1, ""); // both-error {{not an integral constant expression}} \
                              // both-note {{in call to 'a()'}}
+
+
+static_assert(true ? *new int : 4, ""); // both-error {{expression is not an integral constant expression}} \
+                                        // both-note {{read of uninitialized object is not allowed in a constant expression}}
+
 #endif

From a8a7d62d0497cf0f9a8b343e0a2e5098f363ae23 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 19 Jul 2024 22:59:23 +0400
Subject: [PATCH 277/486] AArch64: Avoid using MachineFunction::getMMI

---
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp    | 6 ++----
 llvm/lib/Target/AArch64/AArch64FrameLowering.cpp | 3 +--
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 63358c1568a35..1e60ce9c40df8 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -1728,8 +1728,7 @@ void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
     assert(GAOp.getOffset() == 0 &&
            "non-zero offset for $auth_ptr$ stub slots is not supported");
     const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
-    AuthPtrStubSym =
-        TLOF.getAuthPtrSlotSymbol(TM, &MF->getMMI(), GASym, Key, Disc);
+    AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, GASym, Key, Disc);
   } else {
     assert(TM.getTargetTriple().isOSBinFormatMachO() &&
            "LOADauthptrstatic is implemented only for MachO/ELF");
@@ -1740,8 +1739,7 @@ void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
     assert(GAOp.getOffset() == 0 &&
            "non-zero offset for $auth_ptr$ stub slots is not supported");
     const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
-    AuthPtrStubSym =
-        TLOF.getAuthPtrSlotSymbol(TM, &MF->getMMI(), GASym, Key, Disc);
+    AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, GASym, Key, Disc);
   }
 
   MachineOperand StubMOHi =
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index dac03bc3c1d9e..8c58f0bdf2f53 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1714,7 +1714,6 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
   const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
 
-  MachineModuleInfo &MMI = MF.getMMI();
   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
   bool EmitCFI = AFI->needsDwarfUnwindInfo(MF);
   bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
@@ -1882,7 +1881,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
                       MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
       if (EmitCFI) {
         // Label used to tie together the PROLOG_LABEL and the MachineMoves.
-        MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
+        MCSymbol *FrameLabel = MF.getContext().createTempSymbol();
         // Encode the stack size of the leaf function.
         unsigned CFIIndex = MF.addFrameInst(
             MCCFIInstruction::cfiDefCfaOffset(FrameLabel, NumBytes));

From b4f3a9662d308c869ac97e4f147edb38bc4f0626 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 20 Jul 2024 00:20:52 +0400
Subject: [PATCH 278/486] X86: Avoid using MachineFunction::getMMI

---
 llvm/lib/Target/X86/X86MCInstLower.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index df20ecd1b9b21..77ddd2366e629 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -147,7 +147,7 @@ X86MCInstLower::X86MCInstLower(const MachineFunction &mf,
       AsmPrinter(asmprinter) {}
 
 MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
-  return MF.getMMI().getObjFileInfo<MachineModuleInfoMachO>();
+  return AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoMachO>();
 }
 
 /// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol
@@ -203,7 +203,7 @@ MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const {
     break;
   case X86II::MO_COFFSTUB: {
     MachineModuleInfoCOFF &MMICOFF =
-        MF.getMMI().getObjFileInfo<MachineModuleInfoCOFF>();
+        AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoCOFF>();
     MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym);
     if (!StubSym.getPointer()) {
       assert(MO.isGlobal() && "Extern symbol not handled yet");

From f65d7fdcf81f1fb01df3446b254f3304589f19c4 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 20 Jul 2024 12:55:44 +0200
Subject: [PATCH 279/486] [libc++][vector] Fixes shrink_to_fit. (#97895)

This assures shrink_to_fit does not increase the allocated size.

Partly addresses https://github.com/llvm/llvm-project/issues/95161

---------

Co-authored-by: Mital Ashok <mital.vaja@googlemail.com>
---
 libcxx/include/vector                         |  6 ++-
 .../vector.capacity/shrink_to_fit.pass.cpp    | 47 ++++++++++++++++++-
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/libcxx/include/vector b/libcxx/include/vector
index aaf51d18fe30f..45980043a3c15 100644
--- a/libcxx/include/vector
+++ b/libcxx/include/vector
@@ -1443,7 +1443,11 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::shrink_to_fit() _NOE
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
       allocator_type& __a = this->__alloc();
       __split_buffer<value_type, allocator_type&> __v(size(), size(), __a);
-      __swap_out_circular_buffer(__v);
+      // The Standard mandates shrink_to_fit() does not increase the capacity.
+      // With equal capacity keep the existing buffer. This avoids extra work
+      // due to swapping the elements.
+      if (__v.capacity() < capacity())
+        __swap_out_circular_buffer(__v);
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
     } catch (...) {
     }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit.pass.cpp
index 8851e2a9ed0c7..e39afb2d48f0a 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit.pass.cpp
@@ -71,11 +71,56 @@ TEST_CONSTEXPR_CXX20 bool tests() {
     return true;
 }
 
+#if TEST_STD_VER >= 23
+template <typename T>
+struct increasing_allocator {
+  using value_type         = T;
+  std::size_t min_elements = 1000;
+  increasing_allocator()   = default;
+
+  template <typename U>
+  constexpr increasing_allocator(const increasing_allocator<U>& other) noexcept : min_elements(other.min_elements) {}
+
+  constexpr std::allocation_result<T*> allocate_at_least(std::size_t n) {
+    if (n < min_elements)
+      n = min_elements;
+    min_elements += 1000;
+    return std::allocator<T>{}.allocate_at_least(n);
+  }
+  constexpr T* allocate(std::size_t n) { return allocate_at_least(n).ptr; }
+  constexpr void deallocate(T* p, std::size_t n) noexcept { std::allocator<T>{}.deallocate(p, n); }
+};
+
+template <typename T, typename U>
+bool operator==(increasing_allocator<T>, increasing_allocator<U>) {
+  return true;
+}
+
+// https://github.com/llvm/llvm-project/issues/95161
+constexpr bool test_increasing_allocator() {
+  std::vector<int, increasing_allocator<int>> v;
+  v.push_back(1);
+  assert(is_contiguous_container_asan_correct(v));
+  std::size_t capacity = v.capacity();
+  v.shrink_to_fit();
+  assert(v.capacity() <= capacity);
+  assert(v.size() == 1);
+  assert(is_contiguous_container_asan_correct(v));
+
+  return true;
+}
+#endif // TEST_STD_VER >= 23
+
 int main(int, char**)
 {
-    tests();
+  tests();
 #if TEST_STD_VER > 17
     static_assert(tests());
 #endif
+#if TEST_STD_VER >= 23
+    test_increasing_allocator();
+    static_assert(test_increasing_allocator());
+#endif
+
     return 0;
 }

From a03935b9841c4e30b27dd9399e0b93191ad443f3 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 20 Jul 2024 15:56:38 +0400
Subject: [PATCH 280/486] M68k: Remove hasDebugInfo check when deciding to emit
 CFI (#99750)

No other target checks this, and this is untested. I am trying
to remove the MachineModuleInfo reference from MachineFunction,
and this is the stickiest blocker.
---
 llvm/lib/Target/M68k/M68kFrameLowering.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/lib/Target/M68k/M68kFrameLowering.cpp b/llvm/lib/Target/M68k/M68kFrameLowering.cpp
index 36443f9d33451..c548346f35a2f 100644
--- a/llvm/lib/Target/M68k/M68kFrameLowering.cpp
+++ b/llvm/lib/Target/M68k/M68kFrameLowering.cpp
@@ -246,9 +246,7 @@ MachineBasicBlock::iterator M68kFrameLowering::eliminateCallFramePseudoInstr(
     unsigned StackAlign = getStackAlignment();
     Amount = alignTo(Amount, StackAlign);
 
-    MachineModuleInfo &MMI = MF.getMMI();
-    const auto &Fn = MF.getFunction();
-    bool DwarfCFI = MMI.hasDebugInfo() || Fn.needsUnwindTableEntry();
+    bool DwarfCFI = MF.needsFrameMoves();
 
     // If we have any exception handlers in this function, and we adjust
     // the SP before calls, we may need to indicate this to the unwinder

From d8116cfbd6708a79a19ae2e4d15f2d602407972b Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 20 Jul 2024 15:59:35 +0400
Subject: [PATCH 281/486] AsmPrinter: Avoid use of MachineFunction::getMMI
 (#99751)

---
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp        | 11 ++++++-----
 llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index b46a6d348413b..91b5703944f3d 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1669,8 +1669,8 @@ void AsmPrinter::emitPCSections(const MachineFunction &MF) {
 }
 
 /// Returns true if function begin and end labels should be emitted.
-static bool needFuncLabels(const MachineFunction &MF) {
-  MachineModuleInfo &MMI = MF.getMMI();
+static bool needFuncLabels(const MachineFunction &MF,
+                           const MachineModuleInfo &MMI) {
   if (!MF.getLandingPads().empty() || MF.hasEHFunclets() ||
       MMI.hasDebugInfo() ||
       MF.getFunction().hasMetadata(LLVMContext::MD_pcsections))
@@ -1944,7 +1944,7 @@ void AsmPrinter::emitFunctionBody() {
   // are automatically sized.
   bool EmitFunctionSize = MAI->hasDotTypeDotSizeDirective() && !TT.isWasm();
 
-  if (needFuncLabels(*MF) || EmitFunctionSize) {
+  if (needFuncLabels(*MF, *MMI) || EmitFunctionSize) {
     // Create a symbol for the end of function.
     CurrentFnEnd = createTempSymbol("func_end");
     OutStreamer->emitLabel(CurrentFnEnd);
@@ -2587,8 +2587,9 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
   bool NeedsLocalForSize = MAI->needsLocalForSize();
   if (F.hasFnAttribute("patchable-function-entry") ||
       F.hasFnAttribute("function-instrument") ||
-      F.hasFnAttribute("xray-instruction-threshold") || needFuncLabels(MF) ||
-      NeedsLocalForSize || MF.getTarget().Options.EmitStackSizeSection ||
+      F.hasFnAttribute("xray-instruction-threshold") ||
+      needFuncLabels(MF, *MMI) || NeedsLocalForSize ||
+      MF.getTarget().Options.EmitStackSizeSection ||
       MF.getTarget().Options.BBAddrMap || MF.hasBBLabels()) {
     CurrentFnBegin = createTempSymbol("func_begin");
     if (NeedsLocalForSize)
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
index 49f3fc1a1fa59..087ee02a7f2b3 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
@@ -90,7 +90,7 @@ void DwarfCFIException::beginFunction(const MachineFunction *MF) {
   shouldEmitLSDA = shouldEmitPersonality &&
     LSDAEncoding != dwarf::DW_EH_PE_omit;
 
-  const MCAsmInfo &MAI = *MF->getMMI().getContext().getAsmInfo();
+  const MCAsmInfo &MAI = *MF->getContext().getAsmInfo();
   if (MAI.getExceptionHandlingType() != ExceptionHandling::None)
     shouldEmitCFI =
         MAI.usesCFIForEH() && (shouldEmitPersonality || shouldEmitMoves);

From 54de554ab8654c7a1f39199c1a491bc80edbba53 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Sat, 20 Jul 2024 07:19:04 -0500
Subject: [PATCH 282/486] [Clang] Fix C library wrappers for offloading
 (#99716)

Summary:
This block of code wraps around the standard C library includes.
However, the order C library includes are presented is actually
important. If they are visible before the `libc++` headers then it will
cause errors. This patch simply moves the logic to just before it is
normally done. A more optimal solution would be to put this in the
toolchain, however doing it correctly would require knowing the
offloading kind and that would require rewriting the function signature
in all 30 or so ToolChains.
---
 clang/lib/Driver/ToolChains/Clang.cpp | 56 ++++++++++++++-------------
 clang/test/Driver/gpu-libc-headers.c  |  8 ++--
 2 files changed, 33 insertions(+), 31 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index c9f0fe0a3d60f..71cdaa10416f4 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1077,33 +1077,6 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
   if (JA.isOffloading(Action::OFK_HIP))
     getToolChain().AddHIPIncludeArgs(Args, CmdArgs);
 
-  // If we are compiling for a GPU target we want to override the system headers
-  // with ones created by the 'libc' project if present.
-  if (!Args.hasArg(options::OPT_nostdinc) &&
-      !Args.hasArg(options::OPT_nogpuinc) &&
-      !Args.hasArg(options::OPT_nobuiltininc)) {
-    // Without an offloading language we will include these headers directly.
-    // Offloading languages will instead only use the declarations stored in
-    // the resource directory at clang/lib/Headers/llvm_libc_wrappers.
-    if ((getToolChain().getTriple().isNVPTX() ||
-         getToolChain().getTriple().isAMDGCN()) &&
-        C.getActiveOffloadKinds() == Action::OFK_None) {
-      SmallString<128> P(llvm::sys::path::parent_path(D.Dir));
-      llvm::sys::path::append(P, "include");
-      llvm::sys::path::append(P, getToolChain().getTripleString());
-      CmdArgs.push_back("-internal-isystem");
-      CmdArgs.push_back(Args.MakeArgString(P));
-    } else if (C.getActiveOffloadKinds() == Action::OFK_OpenMP) {
-      // TODO: CUDA / HIP include their own headers for some common functions
-      // implemented here. We'll need to clean those up so they do not conflict.
-      SmallString<128> P(D.ResourceDir);
-      llvm::sys::path::append(P, "include");
-      llvm::sys::path::append(P, "llvm_libc_wrappers");
-      CmdArgs.push_back("-internal-isystem");
-      CmdArgs.push_back(Args.MakeArgString(P));
-    }
-  }
-
   // If we are offloading to a target via OpenMP we need to include the
   // openmp_wrappers folder which contains alternative system headers.
   if (JA.isDeviceOffloading(Action::OFK_OpenMP) &&
@@ -1276,6 +1249,35 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
         });
   }
 
+  // If we are compiling for a GPU target we want to override the system headers
+  // with ones created by the 'libc' project if present.
+  // TODO: This should be moved to `AddClangSystemIncludeArgs` by passing the
+  //       OffloadKind as an argument.
+  if (!Args.hasArg(options::OPT_nostdinc) &&
+      !Args.hasArg(options::OPT_nogpuinc) &&
+      !Args.hasArg(options::OPT_nobuiltininc)) {
+    // Without an offloading language we will include these headers directly.
+    // Offloading languages will instead only use the declarations stored in
+    // the resource directory at clang/lib/Headers/llvm_libc_wrappers.
+    if ((getToolChain().getTriple().isNVPTX() ||
+         getToolChain().getTriple().isAMDGCN()) &&
+        C.getActiveOffloadKinds() == Action::OFK_None) {
+      SmallString<128> P(llvm::sys::path::parent_path(D.Dir));
+      llvm::sys::path::append(P, "include");
+      llvm::sys::path::append(P, getToolChain().getTripleString());
+      CmdArgs.push_back("-internal-isystem");
+      CmdArgs.push_back(Args.MakeArgString(P));
+    } else if (C.getActiveOffloadKinds() == Action::OFK_OpenMP) {
+      // TODO: CUDA / HIP include their own headers for some common functions
+      // implemented here. We'll need to clean those up so they do not conflict.
+      SmallString<128> P(D.ResourceDir);
+      llvm::sys::path::append(P, "include");
+      llvm::sys::path::append(P, "llvm_libc_wrappers");
+      CmdArgs.push_back("-internal-isystem");
+      CmdArgs.push_back(Args.MakeArgString(P));
+    }
+  }
+
   // Add system include arguments for all targets but IAMCU.
   if (!IsIAMCU)
     forAllAssociatedToolChains(C, JA, getToolChain(),
diff --git a/clang/test/Driver/gpu-libc-headers.c b/clang/test/Driver/gpu-libc-headers.c
index 32a5edb175e61..53c016837dde6 100644
--- a/clang/test/Driver/gpu-libc-headers.c
+++ b/clang/test/Driver/gpu-libc-headers.c
@@ -4,15 +4,15 @@
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --sysroot=./ \
 // RUN:     -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target=nvptx64-nvidia-cuda --offload-arch=sm_70  \
 // RUN:     -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-HEADERS
-// CHECK-HEADERS: "-cc1"{{.*}}"-internal-isystem" "{{.*}}include{{.*}}llvm_libc_wrappers"{{.*}}"-isysroot" "./"
-// CHECK-HEADERS: "-cc1"{{.*}}"-internal-isystem" "{{.*}}include{{.*}}llvm_libc_wrappers"{{.*}}"-isysroot" "./"
+// CHECK-HEADERS: "-cc1"{{.*}}"-isysroot" "./"{{.*}}"-internal-isystem" "{{.*}}include{{.*}}llvm_libc_wrappers"
+// CHECK-HEADERS: "-cc1"{{.*}}"-isysroot" "./"{{.*}}"-internal-isystem" "{{.*}}include{{.*}}llvm_libc_wrappers"
 
 // RUN:   %clang -### --target=amdgcn-amd-amdhsa -mcpu=gfx90a --sysroot=./ \
 // RUN:     -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-HEADERS-AMDGPU
 // RUN:   %clang -### --target=nvptx64-nvidia-cuda -march=sm_89 --sysroot=./ \
 // RUN:     -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-HEADERS-NVPTX
-// CHECK-HEADERS-AMDGPU: "-cc1"{{.*}}"-internal-isystem" "{{.*}}include{{.*}}amdgcn-amd-amdhsa"{{.*}}"-isysroot" "./"
-// CHECK-HEADERS-NVPTX: "-cc1"{{.*}}"-internal-isystem" "{{.*}}include{{.*}}nvptx64-nvidia-cuda"{{.*}}"-isysroot" "./"
+// CHECK-HEADERS-AMDGPU: "-cc1"{{.*}}"-isysroot" "./"{{.*}}"-internal-isystem" "{{.*}}include{{.*}}amdgcn-amd-amdhsa"
+// CHECK-HEADERS-NVPTX: "-cc1"{{.*}}"-isysroot" "./"{{.*}}"-internal-isystem" "{{.*}}include{{.*}}nvptx64-nvidia-cuda"
 
 // RUN:   %clang -### --target=amdgcn-amd-amdhsa -mcpu=gfx1030 -nogpulib \
 // RUN:     -nogpuinc %s 2>&1 | FileCheck %s --check-prefix=CHECK-HEADERS-DISABLED

From 710dab6e18ad9ed22c2529b9125d7b8813165ede Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 20 Jul 2024 13:21:32 +0100
Subject: [PATCH 283/486] [VPlan] Remove VPPredInstPHIRecipes without users
 after region merging.

After merging replicate regions, VPPredInstPHIRecipes may become unused.
Remove them directly instead of moving them to the merged region.
---
 llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp  |  5 +++++
 .../Transforms/LoopVectorize/AArch64/pr73894.ll    |  2 --
 .../PowerPC/vplan-force-tail-with-evl.ll           |  2 --
 .../LoopVectorize/X86/divs-with-tail-folding.ll    |  8 --------
 .../LoopVectorize/X86/x86-predication.ll           | 10 ----------
 ...first-order-recurrence-sink-replicate-region.ll |  7 -------
 .../Transforms/LoopVectorize/if-pred-non-void.ll   |  8 --------
 .../LoopVectorize/interleave-and-scalarize-only.ll |  3 ---
 .../LoopVectorize/pr45679-fold-tail-by-masking.ll  | 12 ------------
 .../LoopVectorize/select-cmp-multiuse.ll           | 14 --------------
 .../LoopVectorize/vplan-sink-scalars-and-merge.ll  |  6 ------
 11 files changed, 5 insertions(+), 72 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d668ae2aa5c08..c91fd0f118e31 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -278,6 +278,11 @@ static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
         return UI && UI->getParent() == Then2;
       });
 
+      // Remove phi recipes that are unused after merging the regions.
+      if (Phi1ToMove.getVPSingleValue()->getNumUsers() == 0) {
+        Phi1ToMove.eraseFromParent();
+        continue;
+      }
       Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
     }
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
index 5116a85e4f2a1..a70eafb6078a0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
@@ -32,7 +32,6 @@ define i32 @pr70988(ptr %src, i32 %n) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
 ; CHECK:       pred.load.continue:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi ptr [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
 ; CHECK-NEXT:    br i1 [[ACTIVE_LANE_MASK2]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5]]
 ; CHECK:       pred.load.if4:
@@ -42,7 +41,6 @@ define i32 @pr70988(ptr %src, i32 %n) {
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE5]]
 ; CHECK:       pred.load.continue5:
-; CHECK-NEXT:    [[TMP13:%.*]] = phi ptr [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF4]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF4]] ]
 ; CHECK-NEXT:    [[TMP15:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP8]], i32 [[VEC_PHI]])
 ; CHECK-NEXT:    [[TMP16:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP14]], i32 [[VEC_PHI3]])
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll
index 7f258d57e7018..0b8a2d2f02057 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll
@@ -40,8 +40,6 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; CHECK-NEXT:    Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:    pred.store.continue:
-; CHECK-NEXT:      PHI-PREDICATED-INSTRUCTION vp<[[P1:%.+]]> = ir<%0>
-; CHECK-NEXT:      PHI-PREDICATED-INSTRUCTION vp<[[P2:%.+]]> = ir<%1>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): for.body.2
diff --git a/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
index 42a9ab0ca270f..133510fbb2db8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
@@ -288,9 +288,7 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK-NEXT:    br label %[[PRED_UREM_CONTINUE]]
 ; CHECK:       [[PRED_UREM_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP9]], %[[PRED_UREM_IF]] ]
-; CHECK-NEXT:    [[TMP16:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP10]], %[[PRED_UREM_IF]] ]
 ; CHECK-NEXT:    [[TMP17:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP11]], %[[PRED_UREM_IF]] ]
-; CHECK-NEXT:    [[TMP18:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP12]], %[[PRED_UREM_IF]] ]
 ; CHECK-NEXT:    [[TMP19:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP13]], %[[PRED_UREM_IF]] ]
 ; CHECK-NEXT:    [[TMP20:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP14]], %[[PRED_UREM_IF]] ]
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP5]], i32 1
@@ -307,9 +305,7 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK-NEXT:    br label %[[PRED_UREM_CONTINUE2]]
 ; CHECK:       [[PRED_UREM_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP30:%.*]] = phi <4 x i64> [ [[TMP15]], %[[PRED_UREM_CONTINUE]] ], [ [[TMP24]], %[[PRED_UREM_IF1]] ]
-; CHECK-NEXT:    [[TMP31:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE]] ], [ [[TMP25]], %[[PRED_UREM_IF1]] ]
 ; CHECK-NEXT:    [[TMP32:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE]] ], [ [[TMP26]], %[[PRED_UREM_IF1]] ]
-; CHECK-NEXT:    [[TMP33:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE]] ], [ [[TMP27]], %[[PRED_UREM_IF1]] ]
 ; CHECK-NEXT:    [[TMP34:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE]] ], [ [[TMP28]], %[[PRED_UREM_IF1]] ]
 ; CHECK-NEXT:    [[TMP35:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE]] ], [ [[TMP29]], %[[PRED_UREM_IF1]] ]
 ; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <4 x i1> [[TMP5]], i32 2
@@ -326,9 +322,7 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK-NEXT:    br label %[[PRED_UREM_CONTINUE4]]
 ; CHECK:       [[PRED_UREM_CONTINUE4]]:
 ; CHECK-NEXT:    [[TMP45:%.*]] = phi <4 x i64> [ [[TMP30]], %[[PRED_UREM_CONTINUE2]] ], [ [[TMP39]], %[[PRED_UREM_IF3]] ]
-; CHECK-NEXT:    [[TMP46:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE2]] ], [ [[TMP40]], %[[PRED_UREM_IF3]] ]
 ; CHECK-NEXT:    [[TMP47:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE2]] ], [ [[TMP41]], %[[PRED_UREM_IF3]] ]
-; CHECK-NEXT:    [[TMP48:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE2]] ], [ [[TMP42]], %[[PRED_UREM_IF3]] ]
 ; CHECK-NEXT:    [[TMP49:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE2]] ], [ [[TMP43]], %[[PRED_UREM_IF3]] ]
 ; CHECK-NEXT:    [[TMP50:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE2]] ], [ [[TMP44]], %[[PRED_UREM_IF3]] ]
 ; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <4 x i1> [[TMP5]], i32 3
@@ -345,9 +339,7 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK-NEXT:    br label %[[PRED_UREM_CONTINUE6]]
 ; CHECK:       [[PRED_UREM_CONTINUE6]]:
 ; CHECK-NEXT:    [[TMP60:%.*]] = phi <4 x i64> [ [[TMP45]], %[[PRED_UREM_CONTINUE4]] ], [ [[TMP54]], %[[PRED_UREM_IF5]] ]
-; CHECK-NEXT:    [[TMP61:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE4]] ], [ [[TMP55]], %[[PRED_UREM_IF5]] ]
 ; CHECK-NEXT:    [[TMP62:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE4]] ], [ [[TMP56]], %[[PRED_UREM_IF5]] ]
-; CHECK-NEXT:    [[TMP63:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE4]] ], [ [[TMP57]], %[[PRED_UREM_IF5]] ]
 ; CHECK-NEXT:    [[TMP64:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE4]] ], [ [[TMP58]], %[[PRED_UREM_IF5]] ]
 ; CHECK-NEXT:    [[TMP65:%.*]] = phi i64 [ poison, %[[PRED_UREM_CONTINUE4]] ], [ [[TMP59]], %[[PRED_UREM_IF5]] ]
 ; CHECK-NEXT:    [[TMP66:%.*]] = extractelement <4 x i64> [[TMP60]], i32 0
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
index feaa5fa2fc4d3..eee1b6f35d1b7 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
@@ -238,7 +238,6 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i32 0
 ; CHECK-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; CHECK:       pred.udiv.continue:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
@@ -250,7 +249,6 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP13]], i32 1
 ; CHECK-NEXT:    br label [[PRED_UDIV_CONTINUE2]]
 ; CHECK:       pred.udiv.continue2:
-; CHECK-NEXT:    [[TMP15:%.*]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], [[PRED_UDIV_IF1]] ]
 ; CHECK-NEXT:    [[TMP16:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP14]], [[PRED_UDIV_IF1]] ]
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> [[TMP16]], <2 x i32> [[BROADCAST_SPLAT4]]
 ; CHECK-NEXT:    [[TMP18]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
@@ -314,7 +312,6 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> poison, i32 [[TMP5]], i32 0
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; SINK-GATHER:       pred.udiv.continue:
-; SINK-GATHER-NEXT:    [[TMP7:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ]
 ; SINK-GATHER-NEXT:    [[TMP8:%.*]] = phi <8 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
 ; SINK-GATHER-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 1
 ; SINK-GATHER-NEXT:    br i1 [[TMP9]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2:%.*]]
@@ -326,7 +323,6 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    [[TMP14:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[TMP13]], i32 1
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE2]]
 ; SINK-GATHER:       pred.udiv.continue2:
-; SINK-GATHER-NEXT:    [[TMP15:%.*]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], [[PRED_UDIV_IF1]] ]
 ; SINK-GATHER-NEXT:    [[TMP16:%.*]] = phi <8 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP14]], [[PRED_UDIV_IF1]] ]
 ; SINK-GATHER-NEXT:    [[TMP17:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 2
 ; SINK-GATHER-NEXT:    br i1 [[TMP17]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
@@ -338,7 +334,6 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    [[TMP22:%.*]] = insertelement <8 x i32> [[TMP16]], i32 [[TMP21]], i32 2
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE4]]
 ; SINK-GATHER:       pred.udiv.continue4:
-; SINK-GATHER-NEXT:    [[TMP23:%.*]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE2]] ], [ [[TMP20]], [[PRED_UDIV_IF3]] ]
 ; SINK-GATHER-NEXT:    [[TMP24:%.*]] = phi <8 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE2]] ], [ [[TMP22]], [[PRED_UDIV_IF3]] ]
 ; SINK-GATHER-NEXT:    [[TMP25:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 3
 ; SINK-GATHER-NEXT:    br i1 [[TMP25]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
@@ -350,7 +345,6 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    [[TMP30:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP29]], i32 3
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE6]]
 ; SINK-GATHER:       pred.udiv.continue6:
-; SINK-GATHER-NEXT:    [[TMP31:%.*]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[TMP28]], [[PRED_UDIV_IF5]] ]
 ; SINK-GATHER-NEXT:    [[TMP32:%.*]] = phi <8 x i32> [ [[TMP24]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP30]], [[PRED_UDIV_IF5]] ]
 ; SINK-GATHER-NEXT:    [[TMP33:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 4
 ; SINK-GATHER-NEXT:    br i1 [[TMP33]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
@@ -362,7 +356,6 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    [[TMP38:%.*]] = insertelement <8 x i32> [[TMP32]], i32 [[TMP37]], i32 4
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE8]]
 ; SINK-GATHER:       pred.udiv.continue8:
-; SINK-GATHER-NEXT:    [[TMP39:%.*]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE6]] ], [ [[TMP36]], [[PRED_UDIV_IF7]] ]
 ; SINK-GATHER-NEXT:    [[TMP40:%.*]] = phi <8 x i32> [ [[TMP32]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP38]], [[PRED_UDIV_IF7]] ]
 ; SINK-GATHER-NEXT:    [[TMP41:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 5
 ; SINK-GATHER-NEXT:    br i1 [[TMP41]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
@@ -374,7 +367,6 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    [[TMP46:%.*]] = insertelement <8 x i32> [[TMP40]], i32 [[TMP45]], i32 5
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE10]]
 ; SINK-GATHER:       pred.udiv.continue10:
-; SINK-GATHER-NEXT:    [[TMP47:%.*]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE8]] ], [ [[TMP44]], [[PRED_UDIV_IF9]] ]
 ; SINK-GATHER-NEXT:    [[TMP48:%.*]] = phi <8 x i32> [ [[TMP40]], [[PRED_UDIV_CONTINUE8]] ], [ [[TMP46]], [[PRED_UDIV_IF9]] ]
 ; SINK-GATHER-NEXT:    [[TMP49:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 6
 ; SINK-GATHER-NEXT:    br i1 [[TMP49]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
@@ -386,7 +378,6 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    [[TMP54:%.*]] = insertelement <8 x i32> [[TMP48]], i32 [[TMP53]], i32 6
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE12]]
 ; SINK-GATHER:       pred.udiv.continue12:
-; SINK-GATHER-NEXT:    [[TMP55:%.*]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE10]] ], [ [[TMP52]], [[PRED_UDIV_IF11]] ]
 ; SINK-GATHER-NEXT:    [[TMP56:%.*]] = phi <8 x i32> [ [[TMP48]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP54]], [[PRED_UDIV_IF11]] ]
 ; SINK-GATHER-NEXT:    [[TMP57:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 7
 ; SINK-GATHER-NEXT:    br i1 [[TMP57]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14]]
@@ -398,7 +389,6 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    [[TMP62:%.*]] = insertelement <8 x i32> [[TMP56]], i32 [[TMP61]], i32 7
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE14]]
 ; SINK-GATHER:       pred.udiv.continue14:
-; SINK-GATHER-NEXT:    [[TMP63:%.*]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE12]] ], [ [[TMP60]], [[PRED_UDIV_IF13]] ]
 ; SINK-GATHER-NEXT:    [[TMP64:%.*]] = phi <8 x i32> [ [[TMP56]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP62]], [[PRED_UDIV_IF13]] ]
 ; SINK-GATHER-NEXT:    [[PREDPHI:%.*]] = select <8 x i1> [[BROADCAST_SPLAT]], <8 x i32> [[TMP64]], <8 x i32> [[BROADCAST_SPLAT16]]
 ; SINK-GATHER-NEXT:    [[TMP66]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index bff730f263ab2..048b670f0fec8 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -60,7 +60,6 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.store.continue:
-; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 ; CHECK-NEXT: Successor(s): loop.2
@@ -143,7 +142,6 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.store.continue:
-; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 ; CHECK-NEXT: Successor(s): loop.1
@@ -332,8 +330,6 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK:        pred.store.continue:
-; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%rem>
-; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.2>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 ; CHECK-NEXT:   Successor(s): loop.3
@@ -426,8 +422,6 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.store.continue:
-; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
-; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem.div>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 ; CHECK-NEXT: Successor(s): loop.3
@@ -510,7 +504,6 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
 ; CHECK-NEXT:     Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:     pred.store.continue:
-; CHECK-NEXT:       PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = ir<%val>
 ; CHECK-NEXT:     No successors
 ; CHECK-NEXT:   }
 ; CHECK-NEXT:   Successor(s): loop.1
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
index eade22f3fe11f..ecb57c539a40e 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
@@ -388,7 +388,6 @@ define void @test_scalar2scalar(ptr nocapture %asd, ptr nocapture %bsd) {
 ; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i32> poison, i32 [[TMP12]], i32 0
 ; CHECK-NEXT:    br label [[PRED_SDIV_CONTINUE]]
 ; CHECK:       pred.sdiv.continue:
-; CHECK-NEXT:    [[TMP14:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP10]], [[PRED_SDIV_IF]] ]
 ; CHECK-NEXT:    [[TMP15:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP13]], [[PRED_SDIV_IF]] ]
 ; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[PRED_SDIV_IF3:%.*]], label [[PRED_SDIV_CONTINUE4]]
@@ -401,7 +400,6 @@ define void @test_scalar2scalar(ptr nocapture %asd, ptr nocapture %bsd) {
 ; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[TMP21]], i32 1
 ; CHECK-NEXT:    br label [[PRED_SDIV_CONTINUE4]]
 ; CHECK:       pred.sdiv.continue4:
-; CHECK-NEXT:    [[TMP23:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP19]], [[PRED_SDIV_IF3]] ]
 ; CHECK-NEXT:    [[TMP24:%.*]] = phi <2 x i32> [ [[TMP15]], [[PRED_SDIV_CONTINUE]] ], [ [[TMP22]], [[PRED_SDIV_IF3]] ]
 ; CHECK-NEXT:    [[TMP25:%.*]] = xor <2 x i1> [[TMP6]], <i1 true, i1 true>
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP25]], <2 x i32> [[TMP5]], <2 x i32> [[TMP24]]
@@ -466,7 +464,6 @@ define void @test_scalar2scalar(ptr nocapture %asd, ptr nocapture %bsd) {
 ; UNROLL-NO-VF-NEXT:    [[TMP15:%.*]] = sdiv i32 [[TMP8]], [[TMP14]]
 ; UNROLL-NO-VF-NEXT:    br label [[PRED_SDIV_CONTINUE]]
 ; UNROLL-NO-VF:       pred.sdiv.continue:
-; UNROLL-NO-VF-NEXT:    [[TMP16:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP14]], [[PRED_SDIV_IF]] ]
 ; UNROLL-NO-VF-NEXT:    [[TMP17:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP15]], [[PRED_SDIV_IF]] ]
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP13]], label [[PRED_SDIV_IF2:%.*]], label [[PRED_SDIV_CONTINUE3]]
 ; UNROLL-NO-VF:       pred.sdiv.if2:
@@ -474,7 +471,6 @@ define void @test_scalar2scalar(ptr nocapture %asd, ptr nocapture %bsd) {
 ; UNROLL-NO-VF-NEXT:    [[TMP19:%.*]] = sdiv i32 [[TMP9]], [[TMP18]]
 ; UNROLL-NO-VF-NEXT:    br label [[PRED_SDIV_CONTINUE3]]
 ; UNROLL-NO-VF:       pred.sdiv.continue3:
-; UNROLL-NO-VF-NEXT:    [[TMP20:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP18]], [[PRED_SDIV_IF2]] ]
 ; UNROLL-NO-VF-NEXT:    [[TMP21:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP19]], [[PRED_SDIV_IF2]] ]
 ; UNROLL-NO-VF-NEXT:    [[TMP22:%.*]] = xor i1 [[TMP12]], true
 ; UNROLL-NO-VF-NEXT:    [[TMP23:%.*]] = xor i1 [[TMP13]], true
@@ -577,7 +573,6 @@ define void @pr30172(ptr nocapture %asd, ptr nocapture %bsd) !dbg !5 {;
 ; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <2 x i32> poison, i32 [[TMP16]], i32 0
 ; CHECK-NEXT:    br label [[PRED_SDIV_CONTINUE]]
 ; CHECK:       pred.sdiv.continue:
-; CHECK-NEXT:    [[TMP18:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP14]], [[PRED_SDIV_IF]] ]
 ; CHECK-NEXT:    [[TMP19:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP17]], [[PRED_SDIV_IF]] ]
 ; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x i1> [[TMP10]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[PRED_SDIV_IF3:%.*]], label [[PRED_SDIV_CONTINUE4]]
@@ -590,7 +585,6 @@ define void @pr30172(ptr nocapture %asd, ptr nocapture %bsd) !dbg !5 {;
 ; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP25]], i32 1
 ; CHECK-NEXT:    br label [[PRED_SDIV_CONTINUE4]]
 ; CHECK:       pred.sdiv.continue4:
-; CHECK-NEXT:    [[TMP27:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP23]], [[PRED_SDIV_IF3]] ]
 ; CHECK-NEXT:    [[TMP28:%.*]] = phi <2 x i32> [ [[TMP19]], [[PRED_SDIV_CONTINUE]] ], [ [[TMP26]], [[PRED_SDIV_IF3]] ]
 ; CHECK-NEXT:    [[TMP29:%.*]] = xor <2 x i1> [[TMP7]], <i1 true, i1 true>, !dbg [[DBG35]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = select <2 x i1> [[TMP8]], <2 x i1> [[TMP29]], <2 x i1> zeroinitializer, !dbg [[DBG35]]
@@ -666,7 +660,6 @@ define void @pr30172(ptr nocapture %asd, ptr nocapture %bsd) !dbg !5 {;
 ; UNROLL-NO-VF-NEXT:    [[TMP23:%.*]] = sdiv i32 [[TMP8]], [[TMP22]]
 ; UNROLL-NO-VF-NEXT:    br label [[PRED_SDIV_CONTINUE]]
 ; UNROLL-NO-VF:       pred.sdiv.continue:
-; UNROLL-NO-VF-NEXT:    [[TMP24:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP22]], [[PRED_SDIV_IF]] ]
 ; UNROLL-NO-VF-NEXT:    [[TMP25:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP23]], [[PRED_SDIV_IF]] ]
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP21]], label [[PRED_SDIV_IF2:%.*]], label [[PRED_SDIV_CONTINUE3]]
 ; UNROLL-NO-VF:       pred.sdiv.if2:
@@ -674,7 +667,6 @@ define void @pr30172(ptr nocapture %asd, ptr nocapture %bsd) !dbg !5 {;
 ; UNROLL-NO-VF-NEXT:    [[TMP27:%.*]] = sdiv i32 [[TMP9]], [[TMP26]]
 ; UNROLL-NO-VF-NEXT:    br label [[PRED_SDIV_CONTINUE3]]
 ; UNROLL-NO-VF:       pred.sdiv.continue3:
-; UNROLL-NO-VF-NEXT:    [[TMP28:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP26]], [[PRED_SDIV_IF2]] ]
 ; UNROLL-NO-VF-NEXT:    [[TMP29:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP27]], [[PRED_SDIV_IF2]] ]
 ; UNROLL-NO-VF-NEXT:    [[TMP30:%.*]] = xor i1 [[TMP14]], true, !dbg [[DBG35]]
 ; UNROLL-NO-VF-NEXT:    [[TMP31:%.*]] = xor i1 [[TMP15]], true, !dbg [[DBG35]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
index 7ed42edfc7753..2503520c0ff9d 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
@@ -96,7 +96,6 @@ declare i32 @llvm.smin.i32(i32, i32)
 ; DBG-NEXT:     Successor(s): pred.store.continue
 ; DBG-EMPTY:
 ; DBG-NEXT:     pred.store.continue:
-; DBG-NEXT:       PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = ir<%l>
 ; DBG-NEXT:     No successors
 ; DBG-NEXT:   }
 ; DBG-NEXT:   Successor(s): cond.false.1
@@ -137,7 +136,6 @@ define void @test_scalarize_with_branch_cond(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr [[TMP1]], align 4
 ; CHECK-NEXT:    br label %pred.store.continue
 ; CHECK:       pred.store.continue:
-; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ poison, %vector.body ], [ [[TMP4]], %pred.store.if ]
 ; CHECK-NEXT:    br i1 [[INDUCTION3]], label %pred.store.if4, label %pred.store.continue5
 ; CHECK:       pred.store.if4:
 ; CHECK-NEXT:    [[INDUCTION5:%.*]] = add i64 [[INDEX]], 1
@@ -147,7 +145,6 @@ define void @test_scalarize_with_branch_cond(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    store i32 [[TMP7]], ptr [[TMP2]], align 4
 ; CHECK-NEXT:    br label %pred.store.continue5
 ; CHECK:       pred.store.continue5:
-; CHECK-NEXT:    [[TMP8:%.*]] = phi i32 [ poison, %pred.store.continue ], [ [[TMP7]], %pred.store.if4 ]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; CHECK-NEXT:    br i1 [[TMP9]], label %middle.block, label %vector.body
diff --git a/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll b/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
index f05ec30619c5d..7c23b603b6e91 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
@@ -224,7 +224,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; CHECK-NEXT:    store i64 [[TMP4]], ptr [[B:%.*]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
-; CHECK-NEXT:    [[TMP5:%.*]] = phi i64 [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_STORE_IF]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
 ; CHECK:       pred.store.if1:
@@ -234,7 +233,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; CHECK-NEXT:    store i64 [[TMP9]], ptr [[B]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
 ; CHECK:       pred.store.continue2:
-; CHECK-NEXT:    [[TMP10:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE]] ], [ [[TMP9]], [[PRED_STORE_IF1]] ]
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; CHECK:       pred.store.if3:
@@ -244,7 +242,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; CHECK-NEXT:    store i64 [[TMP14]], ptr [[B]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; CHECK:       pred.store.continue4:
-; CHECK-NEXT:    [[TMP15:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE2]] ], [ [[TMP14]], [[PRED_STORE_IF3]] ]
 ; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.if5:
@@ -254,7 +251,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; CHECK-NEXT:    store i64 [[TMP19]], ptr [[B]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.continue6:
-; CHECK-NEXT:    [[TMP20:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE4]] ], [ [[TMP19]], [[PRED_STORE_IF5]] ]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
@@ -295,7 +291,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF2UF2-NEXT:    store i64 [[TMP5]], ptr [[B:%.*]], align 8
 ; VF2UF2-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; VF2UF2:       pred.store.continue:
-; VF2UF2-NEXT:    [[TMP6:%.*]] = phi i64 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_STORE_IF]] ]
 ; VF2UF2-NEXT:    [[TMP7:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
 ; VF2UF2-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3:%.*]]
 ; VF2UF2:       pred.store.if2:
@@ -305,7 +300,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF2UF2-NEXT:    store i64 [[TMP10]], ptr [[B]], align 8
 ; VF2UF2-NEXT:    br label [[PRED_STORE_CONTINUE3]]
 ; VF2UF2:       pred.store.continue3:
-; VF2UF2-NEXT:    [[TMP11:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE]] ], [ [[TMP10]], [[PRED_STORE_IF2]] ]
 ; VF2UF2-NEXT:    [[TMP12:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
 ; VF2UF2-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5:%.*]]
 ; VF2UF2:       pred.store.if4:
@@ -315,7 +309,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF2UF2-NEXT:    store i64 [[TMP15]], ptr [[B]], align 8
 ; VF2UF2-NEXT:    br label [[PRED_STORE_CONTINUE5]]
 ; VF2UF2:       pred.store.continue5:
-; VF2UF2-NEXT:    [[TMP16:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE3]] ], [ [[TMP15]], [[PRED_STORE_IF4]] ]
 ; VF2UF2-NEXT:    [[TMP17:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
 ; VF2UF2-NEXT:    br i1 [[TMP17]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7]]
 ; VF2UF2:       pred.store.if6:
@@ -325,7 +318,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF2UF2-NEXT:    store i64 [[TMP20]], ptr [[B]], align 8
 ; VF2UF2-NEXT:    br label [[PRED_STORE_CONTINUE7]]
 ; VF2UF2:       pred.store.continue7:
-; VF2UF2-NEXT:    [[TMP21:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE5]] ], [ [[TMP20]], [[PRED_STORE_IF6]] ]
 ; VF2UF2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
 ; VF2UF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD]], <i64 2, i64 2>
 ; VF2UF2-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
@@ -368,7 +360,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF1UF4-NEXT:    store i64 [[TMP9]], ptr [[B:%.*]], align 8
 ; VF1UF4-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; VF1UF4:       pred.store.continue:
-; VF1UF4-NEXT:    [[TMP10:%.*]] = phi i64 [ poison, [[VECTOR_BODY]] ], [ [[TMP9]], [[PRED_STORE_IF]] ]
 ; VF1UF4-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
 ; VF1UF4:       pred.store.if1:
 ; VF1UF4-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
@@ -376,7 +367,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF1UF4-NEXT:    store i64 [[TMP12]], ptr [[B]], align 8
 ; VF1UF4-NEXT:    br label [[PRED_STORE_CONTINUE2]]
 ; VF1UF4:       pred.store.continue2:
-; VF1UF4-NEXT:    [[TMP13:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE]] ], [ [[TMP12]], [[PRED_STORE_IF1]] ]
 ; VF1UF4-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; VF1UF4:       pred.store.if3:
 ; VF1UF4-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
@@ -384,7 +374,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF1UF4-NEXT:    store i64 [[TMP15]], ptr [[B]], align 8
 ; VF1UF4-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; VF1UF4:       pred.store.continue4:
-; VF1UF4-NEXT:    [[TMP16:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE2]] ], [ [[TMP15]], [[PRED_STORE_IF3]] ]
 ; VF1UF4-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
 ; VF1UF4:       pred.store.if5:
 ; VF1UF4-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
@@ -392,7 +381,6 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF1UF4-NEXT:    store i64 [[TMP18]], ptr [[B]], align 8
 ; VF1UF4-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; VF1UF4:       pred.store.continue6:
-; VF1UF4-NEXT:    [[TMP19:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE4]] ], [ [[TMP18]], [[PRED_STORE_IF5]] ]
 ; VF1UF4-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
 ; VF1UF4-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
 ; VF1UF4-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
index 8983c80bf3ef4..9eb90099214e1 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
@@ -510,7 +510,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC1-NEXT:    store i32 [[TMP11]], ptr [[TMP9]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC1-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK-VF4-IC1:       pred.store.continue:
-; CHECK-VF4-IC1-NEXT:    [[TMP12:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP10]], [[PRED_STORE_IF]] ]
 ; CHECK-VF4-IC1-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
 ; CHECK-VF4-IC1-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; CHECK-VF4-IC1:       pred.store.if3:
@@ -521,7 +520,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC1-NEXT:    store i32 [[TMP17]], ptr [[TMP15]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC1-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; CHECK-VF4-IC1:       pred.store.continue4:
-; CHECK-VF4-IC1-NEXT:    [[TMP18:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE]] ], [ [[TMP16]], [[PRED_STORE_IF3]] ]
 ; CHECK-VF4-IC1-NEXT:    [[TMP19:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
 ; CHECK-VF4-IC1-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
 ; CHECK-VF4-IC1:       pred.store.if5:
@@ -532,7 +530,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC1-NEXT:    store i32 [[TMP23]], ptr [[TMP21]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC1-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK-VF4-IC1:       pred.store.continue6:
-; CHECK-VF4-IC1-NEXT:    [[TMP24:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE4]] ], [ [[TMP22]], [[PRED_STORE_IF5]] ]
 ; CHECK-VF4-IC1-NEXT:    [[TMP25:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
 ; CHECK-VF4-IC1-NEXT:    br i1 [[TMP25]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8]]
 ; CHECK-VF4-IC1:       pred.store.if7:
@@ -543,7 +540,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC1-NEXT:    store i32 [[TMP29]], ptr [[TMP27]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC1-NEXT:    br label [[PRED_STORE_CONTINUE8]]
 ; CHECK-VF4-IC1:       pred.store.continue8:
-; CHECK-VF4-IC1-NEXT:    [[TMP30:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE6]] ], [ [[TMP28]], [[PRED_STORE_IF7]] ]
 ; CHECK-VF4-IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-VF4-IC1-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4-IC1-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
@@ -636,7 +632,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    store i32 [[TMP18]], ptr [[TMP16]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC2-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK-VF4-IC2:       pred.store.continue:
-; CHECK-VF4-IC2-NEXT:    [[TMP19:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP17]], [[PRED_STORE_IF]] ]
 ; CHECK-VF4-IC2-NEXT:    [[TMP20:%.*]] = extractelement <4 x i1> [[TMP7]], i32 1
 ; CHECK-VF4-IC2-NEXT:    br i1 [[TMP20]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7:%.*]]
 ; CHECK-VF4-IC2:       pred.store.if6:
@@ -647,7 +642,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    store i32 [[TMP24]], ptr [[TMP22]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC2-NEXT:    br label [[PRED_STORE_CONTINUE7]]
 ; CHECK-VF4-IC2:       pred.store.continue7:
-; CHECK-VF4-IC2-NEXT:    [[TMP25:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE]] ], [ [[TMP23]], [[PRED_STORE_IF6]] ]
 ; CHECK-VF4-IC2-NEXT:    [[TMP26:%.*]] = extractelement <4 x i1> [[TMP7]], i32 2
 ; CHECK-VF4-IC2-NEXT:    br i1 [[TMP26]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
 ; CHECK-VF4-IC2:       pred.store.if8:
@@ -658,7 +652,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    store i32 [[TMP30]], ptr [[TMP28]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC2-NEXT:    br label [[PRED_STORE_CONTINUE9]]
 ; CHECK-VF4-IC2:       pred.store.continue9:
-; CHECK-VF4-IC2-NEXT:    [[TMP31:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE7]] ], [ [[TMP29]], [[PRED_STORE_IF8]] ]
 ; CHECK-VF4-IC2-NEXT:    [[TMP32:%.*]] = extractelement <4 x i1> [[TMP7]], i32 3
 ; CHECK-VF4-IC2-NEXT:    br i1 [[TMP32]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
 ; CHECK-VF4-IC2:       pred.store.if10:
@@ -669,7 +662,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    store i32 [[TMP36]], ptr [[TMP34]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC2-NEXT:    br label [[PRED_STORE_CONTINUE11]]
 ; CHECK-VF4-IC2:       pred.store.continue11:
-; CHECK-VF4-IC2-NEXT:    [[TMP37:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE9]] ], [ [[TMP35]], [[PRED_STORE_IF10]] ]
 ; CHECK-VF4-IC2-NEXT:    [[TMP38:%.*]] = extractelement <4 x i1> [[TMP8]], i32 0
 ; CHECK-VF4-IC2-NEXT:    br i1 [[TMP38]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]]
 ; CHECK-VF4-IC2:       pred.store.if12:
@@ -679,7 +671,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    store i32 [[TMP41]], ptr [[TMP39]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC2-NEXT:    br label [[PRED_STORE_CONTINUE13]]
 ; CHECK-VF4-IC2:       pred.store.continue13:
-; CHECK-VF4-IC2-NEXT:    [[TMP42:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE11]] ], [ [[TMP40]], [[PRED_STORE_IF12]] ]
 ; CHECK-VF4-IC2-NEXT:    [[TMP43:%.*]] = extractelement <4 x i1> [[TMP8]], i32 1
 ; CHECK-VF4-IC2-NEXT:    br i1 [[TMP43]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]]
 ; CHECK-VF4-IC2:       pred.store.if14:
@@ -690,7 +681,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    store i32 [[TMP47]], ptr [[TMP45]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC2-NEXT:    br label [[PRED_STORE_CONTINUE15]]
 ; CHECK-VF4-IC2:       pred.store.continue15:
-; CHECK-VF4-IC2-NEXT:    [[TMP48:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE13]] ], [ [[TMP46]], [[PRED_STORE_IF14]] ]
 ; CHECK-VF4-IC2-NEXT:    [[TMP49:%.*]] = extractelement <4 x i1> [[TMP8]], i32 2
 ; CHECK-VF4-IC2-NEXT:    br i1 [[TMP49]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17:%.*]]
 ; CHECK-VF4-IC2:       pred.store.if16:
@@ -701,7 +691,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    store i32 [[TMP53]], ptr [[TMP51]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC2-NEXT:    br label [[PRED_STORE_CONTINUE17]]
 ; CHECK-VF4-IC2:       pred.store.continue17:
-; CHECK-VF4-IC2-NEXT:    [[TMP54:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE15]] ], [ [[TMP52]], [[PRED_STORE_IF16]] ]
 ; CHECK-VF4-IC2-NEXT:    [[TMP55:%.*]] = extractelement <4 x i1> [[TMP8]], i32 3
 ; CHECK-VF4-IC2-NEXT:    br i1 [[TMP55]], label [[PRED_STORE_IF18:%.*]], label [[PRED_STORE_CONTINUE19]]
 ; CHECK-VF4-IC2:       pred.store.if18:
@@ -712,7 +701,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    store i32 [[TMP59]], ptr [[TMP57]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF4-IC2-NEXT:    br label [[PRED_STORE_CONTINUE19]]
 ; CHECK-VF4-IC2:       pred.store.continue19:
-; CHECK-VF4-IC2-NEXT:    [[TMP60:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE17]] ], [ [[TMP58]], [[PRED_STORE_IF18]] ]
 ; CHECK-VF4-IC2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-VF4-IC2-NEXT:    [[TMP61:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4-IC2-NEXT:    br i1 [[TMP61]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
@@ -804,7 +792,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF1-IC2-NEXT:    store i32 [[TMP17]], ptr [[TMP15]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF1-IC2-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK-VF1-IC2:       pred.store.continue:
-; CHECK-VF1-IC2-NEXT:    [[TMP18:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP16]], [[PRED_STORE_IF]] ]
 ; CHECK-VF1-IC2-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
 ; CHECK-VF1-IC2:       pred.store.if5:
 ; CHECK-VF1-IC2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP2]]
@@ -813,7 +800,6 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF1-IC2-NEXT:    store i32 [[TMP21]], ptr [[TMP19]], align 4, !alias.scope [[META9]], !noalias [[META6]]
 ; CHECK-VF1-IC2-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK-VF1-IC2:       pred.store.continue6:
-; CHECK-VF1-IC2-NEXT:    [[TMP22:%.*]] = phi i32 [ poison, [[PRED_STORE_CONTINUE]] ], [ [[TMP20]], [[PRED_STORE_IF5]] ]
 ; CHECK-VF1-IC2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-VF1-IC2-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF1-IC2-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 313be091f5f09..9c07281a9a8a9 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -46,7 +46,6 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 
 ; CHECK:      pred.store.continue:
-; CHECK-NEXT:   PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 
@@ -768,8 +767,6 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.store.continue:
-; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a>
-; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 ; CHECK-NEXT: Successor(s): loop.2
@@ -854,7 +851,6 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.store.continue:
-; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 ; CHECK-NEXT: Successor(s): loop.2
@@ -914,7 +910,6 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.store.continue:
-; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%l1>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 ; CHECK-NEXT: Successor(s): loop.then.1
@@ -1053,7 +1048,6 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
 ; CHECK-NEXT:     Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:     pred.store.continue:
-; CHECK-NEXT:       PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = ir<%l>
 ; CHECK-NEXT:     No successors
 ; CHECK-NEXT:   }
 ; CHECK-NEXT:   Successor(s): loop.1

From e77a01d79a48e15c94c89e4aa4bd27424a96b49b Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777@gmail.com>
Date: Sat, 20 Jul 2024 20:41:00 +0800
Subject: [PATCH 284/486] =?UTF-8?q?[Clang]=20Implement=20P3034R1=20Module?=
 =?UTF-8?q?=20Declarations=20Shouldn=E2=80=99t=20be=20Macros=20(#90574)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR implement [P3034R1 Module Declarations Shouldn’t be
Macros](https://wg21.link/P3034R1), and refactor the convoluted state
machines in module name lexical analysis.

---------

Signed-off-by: yronglin <yronglin777@gmail.com>
Co-authored-by: Aaron Ballman <aaron@aaronballman.com>
Co-authored-by: cor3ntin <corentinjabot@gmail.com>
---
 clang/docs/ReleaseNotes.rst                   |   2 +
 .../include/clang/Basic/DiagnosticLexKinds.td |   5 +
 clang/include/clang/Basic/IdentifierTable.h   |  24 +-
 clang/include/clang/Basic/TokenKinds.def      |   3 +
 clang/include/clang/Lex/Preprocessor.h        |  83 +++-
 clang/include/clang/Lex/Token.h               |   3 +
 clang/include/clang/Parse/Parser.h            |   2 +-
 clang/lib/Basic/IdentifierTable.cpp           |   3 +-
 .../lib/Frontend/PrintPreprocessedOutput.cpp  |  12 +-
 clang/lib/Lex/PPLexerChange.cpp               |   9 +-
 clang/lib/Lex/Preprocessor.cpp                | 444 ++++++++++++------
 clang/lib/Lex/TokenConcatenation.cpp          |  10 +
 clang/lib/Parse/ParseDecl.cpp                 |   8 +-
 clang/lib/Parse/Parser.cpp                    |  93 ++--
 clang/test/CXX/cpp/cpp.module/p2.cppm         |  88 ++++
 .../basic/basic.link/module-declaration.cpp   |  61 +--
 .../dcl.module/dcl.module.import/p1.cppm      |  39 +-
 clang/test/SemaCXX/modules.cppm               |  89 ++--
 clang/www/cxx_status.html                     |   2 +-
 19 files changed, 717 insertions(+), 263 deletions(-)
 create mode 100644 clang/test/CXX/cpp/cpp.module/p2.cppm

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index aa91663260755..4638b91b48f95 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -285,6 +285,8 @@ C++2c Feature Support
 
 - Implemented `P2963R3 Ordering of constraints involving fold expressions <https://wg21.link/P2963R3>`_.
 
+- Implemented `P3034R1 Module Declarations Shouldn’t be Macros <https://wg21.link/P3034R1>`_.
+
 
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td
index 12d7b8c0205ee..08ece01009387 100644
--- a/clang/include/clang/Basic/DiagnosticLexKinds.td
+++ b/clang/include/clang/Basic/DiagnosticLexKinds.td
@@ -952,6 +952,11 @@ def warn_module_conflict : Warning<
   InGroup<ModuleConflict>;
 
 // C++20 modules
+def err_module_decl_cannot_be_macros : Error<
+  "the module name in a module%select{| partition}0 declaration cannot contain "
+  "an object-like macro %1">;
+def err_unxepected_paren_in_module_decl : Error<
+  "unexpected '(' after the module name in a module%select{| partition}0 declaration">;
 def err_header_import_semi_in_macro : Error<
   "semicolon terminating header import declaration cannot be produced "
   "by a macro">;
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index ae9ebd9f59154..f40f74d0355ad 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -180,6 +180,10 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsModulesImport : 1;
 
+  // True if this is the 'module' contextual keyword.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned IsModulesDecl : 1;
+
   // True if this is a mangled OpenMP variant name.
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsMangledOpenMPVariantName : 1;
@@ -196,7 +200,7 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsFinal : 1;
 
-  // 22 bits left in a 64-bit word.
+  // 21 bits left in a 64-bit word.
 
   // Managed by the language front-end.
   void *FETokenInfo = nullptr;
@@ -212,8 +216,8 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
         IsCPPOperatorKeyword(false), NeedsHandleIdentifier(false),
         IsFromAST(false), ChangedAfterLoad(false), FEChangedAfterLoad(false),
         RevertedTokenID(false), OutOfDate(false), IsModulesImport(false),
-        IsMangledOpenMPVariantName(false), IsDeprecatedMacro(false),
-        IsRestrictExpansion(false), IsFinal(false) {}
+        IsModulesDecl(false), IsMangledOpenMPVariantName(false),
+        IsDeprecatedMacro(false), IsRestrictExpansion(false), IsFinal(false) {}
 
 public:
   IdentifierInfo(const IdentifierInfo &) = delete;
@@ -520,6 +524,18 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
       RecomputeNeedsHandleIdentifier();
   }
 
+  /// Determine whether this is the contextual keyword \c module.
+  bool isModulesDeclaration() const { return IsModulesDecl; }
+
+  /// Set whether this identifier is the contextual keyword \c module.
+  void setModulesDeclaration(bool I) {
+    IsModulesDecl = I;
+    if (I)
+      NeedsHandleIdentifier = true;
+    else
+      RecomputeNeedsHandleIdentifier();
+  }
+
   /// Determine whether this is the mangled name of an OpenMP variant.
   bool isMangledOpenMPVariantName() const { return IsMangledOpenMPVariantName; }
 
@@ -740,6 +756,8 @@ class IdentifierTable {
     // If this is the 'import' contextual keyword, mark it as such.
     if (Name == "import")
       II->setModulesImport(true);
+    else if (Name == "module")
+      II->setModulesDeclaration(true);
 
     return *II;
   }
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 7f4912b9bcd96..8db18c049b6d0 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -1003,6 +1003,9 @@ ANNOTATION(module_include)
 ANNOTATION(module_begin)
 ANNOTATION(module_end)
 
+// Annotations for C++, Clang and Objective-C named modules.
+ANNOTATION(module_name)
+
 // Annotation for a header_name token that has been looked up and transformed
 // into the name of a header unit.
 ANNOTATION(header_unit)
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index fc7d0053f2323..56aef99a3f38a 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -615,10 +615,6 @@ class Preprocessor {
 
   ModuleDeclSeq ModuleDeclState;
 
-  /// Whether the module import expects an identifier next. Otherwise,
-  /// it expects a '.' or ';'.
-  bool ModuleImportExpectsIdentifier = false;
-
   /// The identifier and source location of the currently-active
   /// \#pragma clang arc_cf_code_audited begin.
   std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
@@ -1744,11 +1740,14 @@ class Preprocessor {
   /// Lex a token, forming a header-name token if possible.
   bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
 
+  /// Lex a module name or a partition name.
+  bool LexModuleName(Token &Result, bool IsImport);
+
   /// Lex the parameters for an #embed directive, returns nullopt on error.
   std::optional<LexEmbedParametersResult> LexEmbedParameters(Token &Current,
                                                              bool ForHasEmbed);
-
   bool LexAfterModuleImport(Token &Result);
+  bool LexAfterModuleDecl(Token &Result);
   void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
 
   void makeModuleVisible(Module *M, SourceLocation Loc);
@@ -3039,6 +3038,9 @@ class Preprocessor {
   static bool CLK_LexAfterModuleImport(Preprocessor &P, Token &Result) {
     return P.LexAfterModuleImport(Result);
   }
+  static bool CLK_LexAfterModuleDecl(Preprocessor &P, Token &Result) {
+    return P.LexAfterModuleDecl(Result);
+  }
 };
 
 /// Abstract base class that describes a handler that will receive
@@ -3071,6 +3073,77 @@ struct EmbedAnnotationData {
 /// Registry of pragma handlers added by plugins
 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
 
+/// Represents module or partition name token sequance.
+///
+///     module-name:
+///           module-name-qualifier[opt] identifier
+///
+///     partition-name: [C++20]
+///           : module-name-qualifier[opt] identifier
+///
+///     module-name-qualifier
+///           module-name-qualifier[opt] identifier .
+///
+/// This class can only be created by the preprocessor and guarantees that the
+/// two source array being contiguous in memory and only contains 3 kind of
+/// tokens (identifier, '.' and ':'). And only available when the preprocessor
+/// returns annot_module_name token.
+///
+/// For exmaple:
+///
+/// export module m.n:c.d
+///
+/// The module name array has 3 tokens ['m', '.', 'n'].
+/// The partition name array has 4 tokens [':', 'c', '.', 'd'].
+///
+/// When import a partition in a named module fragment (Eg. import :part1;),
+/// the module name array will be empty, and the partition name array has 2
+/// tokens.
+///
+/// When we meet a private-module-fragment (Eg. module :private;), preprocessor
+/// will not return a annot_module_name token, but will return 2 separate tokens
+/// [':', 'kw_private'].
+
+class ModuleNameInfo {
+  friend class Preprocessor;
+  ArrayRef<Token> ModuleName;
+  ArrayRef<Token> PartitionName;
+
+  ModuleNameInfo(ArrayRef<Token> AnnotToks, std::optional<unsigned> ColonIndex);
+
+public:
+  /// Return the contiguous token array.
+  ArrayRef<Token> getTokens() const {
+    if (ModuleName.empty())
+      return PartitionName;
+    if (PartitionName.empty())
+      return ModuleName;
+    return ArrayRef(ModuleName.begin(), PartitionName.end());
+  }
+  bool hasModuleName() const { return !ModuleName.empty(); }
+  bool hasPartitionName() const { return !PartitionName.empty(); }
+  ArrayRef<Token> getModuleName() const { return ModuleName; }
+  ArrayRef<Token> getPartitionName() const { return PartitionName; }
+  Token getColonToken() const {
+    assert(hasPartitionName() && "Do not have a partition name");
+    return getPartitionName().front();
+  }
+
+  /// Under the standard C++ Modules, the dot is just part of the module name,
+  /// and not a real hierarchy separator. Flatten such module names now.
+  std::string getFlatName() const;
+
+  /// Build a module id path from the contiguous token array, both include
+  /// module name and partition name.
+  void getModuleIdPath(
+      SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path) const;
+
+  /// Build a module id path from \param ModuleName.
+  static void getModuleIdPath(
+      ArrayRef<Token> ModuleName,
+      SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path);
+};
+
 } // namespace clang
 
 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index 4f29fb7d11415..2be3ad39529f0 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -235,6 +235,9 @@ class Token {
     assert(isAnnotation() && "Used AnnotVal on non-annotation token");
     return PtrData;
   }
+  template <class T> T getAnnotationValueAs() const {
+    return static_cast<T>(getAnnotationValue());
+  }
   void setAnnotationValue(void *val) {
     assert(isAnnotation() && "Used AnnotVal on non-annotation token");
     PtrData = val;
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 93e60be512aae..afcdacf02583a 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3876,7 +3876,7 @@ class Parser : public CodeCompletionHandler {
   }
 
   bool ParseModuleName(
-      SourceLocation UseLoc,
+      SourceLocation UseLoc, ArrayRef<Token> ModuleName,
       SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path,
       bool IsImport);
 
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index 4f7ccaf4021d6..97d830214f890 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -322,8 +322,9 @@ void IdentifierTable::AddKeywords(const LangOptions &LangOpts) {
   if (LangOpts.IEEE128)
     AddKeyword("__ieee128", tok::kw___float128, KEYALL, LangOpts, *this);
 
-  // Add the 'import' contextual keyword.
+  // Add the 'import' and 'module' contextual keyword.
   get("import").setModulesImport(true);
+  get("module").setModulesDeclaration(true);
 }
 
 /// Checks if the specified token kind represents a keyword in the
diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 0592423c12eca..1fff88ccf0405 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -758,9 +758,10 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok,
   // These tokens are not expanded to anything and don't need whitespace before
   // them.
   if (Tok.is(tok::eof) ||
-      (Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) &&
-       !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) &&
-       !Tok.is(tok::annot_repl_input_end) && !Tok.is(tok::annot_embed)))
+      (Tok.isAnnotation() && Tok.isNot(tok::annot_header_unit) &&
+       Tok.isNot(tok::annot_module_begin) && Tok.isNot(tok::annot_module_end) &&
+       Tok.isNot(tok::annot_module_name) &&
+       Tok.isNot(tok::annot_repl_input_end) && Tok.isNot(tok::annot_embed)))
     return;
 
   // EmittedDirectiveOnThisLine takes priority over RequireSameLine.
@@ -951,6 +952,11 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
       PP.Lex(Tok);
       IsStartOfLine = true;
       continue;
+    } else if (Tok.is(tok::annot_module_name)) {
+      auto *Info = static_cast<ModuleNameInfo *>(Tok.getAnnotationValue());
+      *Callbacks->OS << Info->getFlatName();
+      PP.Lex(Tok);
+      continue;
     } else if (Tok.is(tok::annot_header_unit)) {
       // This is a header-name that has been (effectively) converted into a
       // module-name.
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index 8221db46e06ac..c3a903917e9ce 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -122,7 +122,8 @@ void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
   CurPPLexer = TheLexer;
   CurDirLookup = CurDir;
   CurLexerSubmodule = nullptr;
-  if (CurLexerCallback != CLK_LexAfterModuleImport)
+  if (CurLexerCallback != CLK_LexAfterModuleImport &&
+      CurLexerCallback != CLK_LexAfterModuleDecl)
     CurLexerCallback = TheLexer->isDependencyDirectivesLexer()
                            ? CLK_DependencyDirectivesLexer
                            : CLK_Lexer;
@@ -161,8 +162,7 @@ void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
   PushIncludeMacroStack();
   CurDirLookup = nullptr;
   CurTokenLexer = std::move(TokLexer);
-  if (CurLexerCallback != CLK_LexAfterModuleImport)
-    CurLexerCallback = CLK_TokenLexer;
+  CurLexerCallback = CLK_TokenLexer;
 }
 
 /// EnterTokenStream - Add a "macro" context to the top of the include stack,
@@ -216,7 +216,8 @@ void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
   PushIncludeMacroStack();
   CurDirLookup = nullptr;
   CurTokenLexer = std::move(TokLexer);
-  if (CurLexerCallback != CLK_LexAfterModuleImport)
+  if (CurLexerCallback != CLK_LexAfterModuleImport &&
+      CurLexerCallback != CLK_LexAfterModuleDecl)
     CurLexerCallback = CLK_TokenLexer;
 }
 
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 63e27e62cffc8..2726fae344337 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -860,9 +860,15 @@ bool Preprocessor::HandleIdentifier(Token &Identifier) {
     ModuleImportLoc = Identifier.getLocation();
     NamedModuleImportPath.clear();
     IsAtImport = true;
-    ModuleImportExpectsIdentifier = true;
     CurLexerCallback = CLK_LexAfterModuleImport;
   }
+
+  if ((II.isModulesDeclaration() || Identifier.is(tok::kw_module)) &&
+      !InMacroArgs && !DisableMacroExpansion &&
+      (getLangOpts().CPlusPlusModules || getLangOpts().DebuggerSupport) &&
+      CurLexerCallback != CLK_CachingLexer) {
+    CurLexerCallback = CLK_LexAfterModuleDecl;
+  }
   return true;
 }
 
@@ -905,6 +911,7 @@ void Preprocessor::Lex(Token &Result) {
     // This token is injected to represent the translation of '#include "a.h"'
     // into "import a.h;". Mimic the notional ';'.
     case tok::annot_module_include:
+    case tok::annot_repl_input_end:
     case tok::semi:
       TrackGMFState.handleSemi();
       StdCXXImportSeqState.handleSemi();
@@ -919,12 +926,30 @@ void Preprocessor::Lex(Token &Result) {
       StdCXXImportSeqState.handleExport();
       ModuleDeclState.handleExport();
       break;
-    case tok::colon:
-      ModuleDeclState.handleColon();
-      break;
-    case tok::period:
-      ModuleDeclState.handlePeriod();
+    case tok::annot_module_name: {
+      auto *Info = static_cast<ModuleNameInfo *>(Result.getAnnotationValue());
+      for (const auto &Tok : Info->getTokens()) {
+        switch (Tok.getKind()) {
+        case tok::identifier:
+          ModuleDeclState.handleIdentifier(Tok.getIdentifierInfo());
+          break;
+        case tok::period:
+          ModuleDeclState.handlePeriod();
+          break;
+        case tok::colon:
+          ModuleDeclState.handleColon();
+          break;
+        default:
+          llvm_unreachable("Unexpected token in module name");
+        }
+      }
+      if (ModuleDeclState.isModuleCandidate())
+        break;
+      TrackGMFState.handleMisc();
+      StdCXXImportSeqState.handleMisc();
+      ModuleDeclState.handleMisc();
       break;
+    }
     case tok::identifier:
       // Check "import" and "module" when there is no open bracket. The two
       // identifiers are not meaningful with open brackets.
@@ -936,17 +961,17 @@ void Preprocessor::Lex(Token &Result) {
             ModuleImportLoc = Result.getLocation();
             NamedModuleImportPath.clear();
             IsAtImport = false;
-            ModuleImportExpectsIdentifier = true;
             CurLexerCallback = CLK_LexAfterModuleImport;
           }
           break;
-        } else if (Result.getIdentifierInfo() == getIdentifierInfo("module")) {
+        }
+        if (Result.getIdentifierInfo()->isModulesDeclaration()) {
           TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq());
           ModuleDeclState.handleModule();
+          CurLexerCallback = CLK_LexAfterModuleDecl;
           break;
         }
       }
-      ModuleDeclState.handleIdentifier(Result.getIdentifierInfo());
       if (ModuleDeclState.isModuleCandidate())
         break;
       [[fallthrough]];
@@ -1121,6 +1146,151 @@ void Preprocessor::CollectPpImportSuffix(SmallVectorImpl<Token> &Toks) {
   }
 }
 
+ModuleNameInfo::ModuleNameInfo(ArrayRef<Token> AnnotToks,
+                               std::optional<unsigned> ColonIndex) {
+  assert(!AnnotToks.empty() && "Named module token cannot be empty.");
+  if (!ColonIndex.has_value())
+    ColonIndex = AnnotToks.size();
+  ModuleName = ArrayRef(AnnotToks.begin(), AnnotToks.begin() + *ColonIndex);
+  PartitionName = ArrayRef(AnnotToks.begin() + *ColonIndex, AnnotToks.end());
+  assert(ModuleName.end() == PartitionName.begin());
+}
+
+std::string ModuleNameInfo::getFlatName() const {
+  std::string FlatModuleName;
+  for (auto &Tok : getTokens()) {
+    switch (Tok.getKind()) {
+    case tok::identifier:
+      FlatModuleName += Tok.getIdentifierInfo()->getName();
+      break;
+    case tok::period:
+      FlatModuleName += '.';
+      break;
+    case tok::colon:
+      FlatModuleName += ':';
+      break;
+    default:
+      llvm_unreachable("Unexpected token in module name");
+    }
+  }
+  return FlatModuleName;
+}
+
+void ModuleNameInfo::getModuleIdPath(
+    SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path) const {
+  return getModuleIdPath(getTokens(), Path);
+}
+
+void ModuleNameInfo::getModuleIdPath(
+    ArrayRef<Token> ModuleName,
+    SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path) {
+  for (const auto &Tok : ModuleName) {
+    if (Tok.is(tok::identifier))
+      Path.push_back(
+          std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation()));
+  }
+}
+
+/// Lex a module name or a partition name.
+///
+///     module-name:
+///           module-name-qualifier[opt] identifier
+///
+///     partition-name: [C++20]
+///           : module-name-qualifier[opt] identifier
+///
+///     module-name-qualifier
+///           module-name-qualifier[opt] identifier .
+bool Preprocessor::LexModuleName(Token &Result, bool IsImport) {
+  bool ExpectsIdentifier = true, IsLexingPartition = false;
+  SmallVector<Token, 8> ModuleName;
+  std::optional<unsigned> ColonTokIndex;
+  auto LexNextToken = [&](Token &Tok) {
+    if (IsImport)
+      Lex(Tok);
+    else
+      LexUnexpandedToken(Tok);
+  };
+
+  while (true) {
+    LexNextToken(Result);
+    if (ExpectsIdentifier && Result.is(tok::identifier)) {
+      auto *MI = getMacroInfo(Result.getIdentifierInfo());
+      if (getLangOpts().CPlusPlusModules && !IsImport && MI &&
+          MI->isObjectLike()) {
+        Diag(Result, diag::err_module_decl_cannot_be_macros)
+            << Result.getLocation() << IsLexingPartition
+            << Result.getIdentifierInfo();
+      }
+      ModuleName.push_back(Result);
+      ExpectsIdentifier = false;
+      continue;
+    }
+
+    if (!ExpectsIdentifier && Result.is(tok::period)) {
+      ModuleName.push_back(Result);
+      ExpectsIdentifier = true;
+      continue;
+    }
+
+    // Module partition only allowed in C++20 Modules.
+    if (getLangOpts().CPlusPlusModules && Result.is(tok::colon)) {
+      // Handle the form like: import :P;
+      // If the token after ':' is not an identifier, this is a invalid module
+      // name.
+      if (ModuleName.empty()) {
+        Token Tmp;
+        LexNextToken(Tmp);
+        EnterToken(Tmp, /*IsReiject=*/false);
+        // A private-module-fragment:
+        // export module :private;
+        if (!IsImport && Tmp.is(tok::kw_private))
+          return true;
+        // import :N;
+        if (IsImport && Tmp.isNot(tok::identifier))
+          return false;
+      } else if (!ExpectsIdentifier) {
+        ExpectsIdentifier = true;
+      }
+      IsLexingPartition = true;
+      ColonTokIndex = ModuleName.size();
+      ModuleName.push_back(Result);
+      continue;
+    }
+
+    // [cpp.module]/p2: where the pp-tokens (if any) shall not begin with a (
+    // preprocessing token [...]
+    //
+    // We only emit diagnostic in the preprocessor, and in the parser we skip
+    // invalid tokens and recover from errors.
+    if (getLangOpts().CPlusPlusModules && !ExpectsIdentifier &&
+        Result.is(tok::l_paren))
+      Diag(Result, diag::err_unxepected_paren_in_module_decl)
+          << IsLexingPartition;
+    break;
+  }
+
+  // Put the last token back to stream, it's not a valid part of module name.
+  // We lexed it unexpanded but it might be a valid macro expansion
+  Result.clearFlag(Token::DisableExpand);
+  auto ToksCopy = std::make_unique<Token[]>(1);
+  *ToksCopy.get() = Result;
+  EnterTokenStream(std::move(ToksCopy), 1,
+                   /*DisableMacroExpansion=*/false,
+                   /*IsReinject=*/false);
+
+  if (ModuleName.empty())
+    return false;
+  Result.startToken();
+  Result.setKind(tok::annot_module_name);
+  Result.setLocation(ModuleName.front().getLocation());
+  Result.setAnnotationEndLoc(ModuleName.back().getLocation());
+  auto AnnotToks = ArrayRef(ModuleName).copy(getPreprocessorAllocator());
+  ModuleNameInfo *Info =
+      new (getPreprocessorAllocator()) ModuleNameInfo(AnnotToks, ColonTokIndex);
+  Result.setAnnotationValue(static_cast<void *>(Info));
+  return true;
+}
 
 /// Lex a token following the 'import' contextual keyword.
 ///
@@ -1145,6 +1315,17 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
   // Figure out what kind of lexer we actually have.
   recomputeCurLexerKind();
 
+  // Allocate a holding buffer for a sequence of tokens and introduce it into
+  // the token stream.
+  auto EnterTokens = [this](ArrayRef<Token> Toks) {
+    auto ToksCopy = std::make_unique<Token[]>(Toks.size());
+    std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
+    EnterTokenStream(std::move(ToksCopy), Toks.size(),
+                     /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
+  };
+
+  SmallVector<Token, 32> Suffix;
+
   // Lex the next token. The header-name lexing rules are used at the start of
   // a pp-import.
   //
@@ -1155,122 +1336,108 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
     if (LexHeaderName(Result))
       return true;
 
-    if (Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
-      std::string Name = ModuleDeclState.getPrimaryName().str();
-      Name += ":";
-      NamedModuleImportPath.push_back(
-          {getIdentifierInfo(Name), Result.getLocation()});
-      CurLexerCallback = CLK_LexAfterModuleImport;
-      return true;
-    }
-  } else {
-    Lex(Result);
-  }
+    // Check for a header-name.
+    if (Result.is(tok::header_name)) {
+      // Enter the header-name token into the token stream; a Lex action cannot
+      // both return a token and cache tokens (doing so would corrupt the token
+      // cache if the call to Lex comes from CachingLex / PeekAhead).
+      Suffix.push_back(Result);
+
+      // Consume the pp-import-suffix and expand any macros in it now. We'll add
+      // it back into the token stream later.
+      CollectPpImportSuffix(Suffix);
+      if (Suffix.back().isNot(tok::semi)) {
+        // This is not a pp-import after all.
+        EnterTokens(Suffix);
+        return false;
+      }
 
-  // Allocate a holding buffer for a sequence of tokens and introduce it into
-  // the token stream.
-  auto EnterTokens = [this](ArrayRef<Token> Toks) {
-    auto ToksCopy = std::make_unique<Token[]>(Toks.size());
-    std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
-    EnterTokenStream(std::move(ToksCopy), Toks.size(),
-                     /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
-  };
+      // C++2a [cpp.module]p1:
+      //   The ';' preprocessing-token terminating a pp-import shall not have
+      //   been produced by macro replacement.
+      SourceLocation SemiLoc = Suffix.back().getLocation();
+      if (SemiLoc.isMacroID())
+        Diag(SemiLoc, diag::err_header_import_semi_in_macro);
+
+      // Reconstitute the import token.
+      Token ImportTok;
+      ImportTok.startToken();
+      ImportTok.setKind(tok::kw_import);
+      ImportTok.setLocation(ModuleImportLoc);
+      ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
+      ImportTok.setLength(6);
+
+      auto Action = HandleHeaderIncludeOrImport(
+          /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
+      switch (Action.Kind) {
+      case ImportAction::None:
+        break;
 
-  bool ImportingHeader = Result.is(tok::header_name);
-  // Check for a header-name.
-  SmallVector<Token, 32> Suffix;
-  if (ImportingHeader) {
-    // Enter the header-name token into the token stream; a Lex action cannot
-    // both return a token and cache tokens (doing so would corrupt the token
-    // cache if the call to Lex comes from CachingLex / PeekAhead).
-    Suffix.push_back(Result);
+      case ImportAction::ModuleBegin:
+        // Let the parser know we're textually entering the module.
+        Suffix.emplace_back();
+        Suffix.back().startToken();
+        Suffix.back().setKind(tok::annot_module_begin);
+        Suffix.back().setLocation(SemiLoc);
+        Suffix.back().setAnnotationEndLoc(SemiLoc);
+        Suffix.back().setAnnotationValue(Action.ModuleForHeader);
+        [[fallthrough]];
+
+      case ImportAction::ModuleImport:
+      case ImportAction::HeaderUnitImport:
+      case ImportAction::SkippedModuleImport:
+        // We chose to import (or textually enter) the file. Convert the
+        // header-name token into a header unit annotation token.
+        Suffix[0].setKind(tok::annot_header_unit);
+        Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
+        Suffix[0].setAnnotationValue(Action.ModuleForHeader);
+        // FIXME: Call the moduleImport callback?
+        break;
+      case ImportAction::Failure:
+        assert(TheModuleLoader.HadFatalFailure &&
+               "This should be an early exit only to a fatal error");
+        Result.setKind(tok::eof);
+        CurLexer->cutOffLexing();
+        EnterTokens(Suffix);
+        return true;
+      }
 
-    // Consume the pp-import-suffix and expand any macros in it now. We'll add
-    // it back into the token stream later.
-    CollectPpImportSuffix(Suffix);
-    if (Suffix.back().isNot(tok::semi)) {
-      // This is not a pp-import after all.
       EnterTokens(Suffix);
       return false;
     }
+  } else {
+    Lex(Result);
+  }
 
-    // C++2a [cpp.module]p1:
-    //   The ';' preprocessing-token terminating a pp-import shall not have
-    //   been produced by macro replacement.
-    SourceLocation SemiLoc = Suffix.back().getLocation();
-    if (SemiLoc.isMacroID())
-      Diag(SemiLoc, diag::err_header_import_semi_in_macro);
-
-    // Reconstitute the import token.
-    Token ImportTok;
-    ImportTok.startToken();
-    ImportTok.setKind(tok::kw_import);
-    ImportTok.setLocation(ModuleImportLoc);
-    ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
-    ImportTok.setLength(6);
-
-    auto Action = HandleHeaderIncludeOrImport(
-        /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
-    switch (Action.Kind) {
-    case ImportAction::None:
-      break;
-
-    case ImportAction::ModuleBegin:
-      // Let the parser know we're textually entering the module.
-      Suffix.emplace_back();
-      Suffix.back().startToken();
-      Suffix.back().setKind(tok::annot_module_begin);
-      Suffix.back().setLocation(SemiLoc);
-      Suffix.back().setAnnotationEndLoc(SemiLoc);
-      Suffix.back().setAnnotationValue(Action.ModuleForHeader);
-      [[fallthrough]];
-
-    case ImportAction::ModuleImport:
-    case ImportAction::HeaderUnitImport:
-    case ImportAction::SkippedModuleImport:
-      // We chose to import (or textually enter) the file. Convert the
-      // header-name token into a header unit annotation token.
-      Suffix[0].setKind(tok::annot_header_unit);
-      Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
-      Suffix[0].setAnnotationValue(Action.ModuleForHeader);
-      // FIXME: Call the moduleImport callback?
-      break;
-    case ImportAction::Failure:
-      assert(TheModuleLoader.HadFatalFailure &&
-             "This should be an early exit only to a fatal error");
-      Result.setKind(tok::eof);
-      CurLexer->cutOffLexing();
-      EnterTokens(Suffix);
+  if (Result.isOneOf(tok::identifier, tok::colon)) {
+    EnterToken(Result, /*IsReinject=*/false);
+    if (!LexModuleName(Result, /*IsImport=*/true))
       return true;
+    auto *Info = Result.getAnnotationValueAs<ModuleNameInfo *>();
+    if (getLangOpts().CPlusPlusModules) {
+      // Under the standard C++ Modules, the dot is just part of the module
+      // name, and not a real hierarchy separator. Flatten such module names
+      // now.
+      //
+      // FIXME: Is this the right level to be performing this transformation?
+      std::string FlatModuleName;
+      if (Info->getTokens().front().is(tok::colon)) {
+        // Import a module partition allowed in C++20 Modules.
+        // We can import a partition in named module TU.
+        if (NamedModuleImportPath.empty() && ModuleDeclState.isNamedModule())
+          FlatModuleName = llvm::Twine(ModuleDeclState.getPrimaryName())
+                               .concat(Info->getFlatName())
+                               .str();
+        else
+          return true;
+      } else {
+        FlatModuleName = Info->getFlatName();
+      }
+      NamedModuleImportPath.emplace_back(getIdentifierInfo(FlatModuleName),
+                                         Result.getLocation());
+    } else {
+      Info->getModuleIdPath(NamedModuleImportPath);
     }
-
-    EnterTokens(Suffix);
-    return false;
-  }
-
-  // The token sequence
-  //
-  //   import identifier (. identifier)*
-  //
-  // indicates a module import directive. We already saw the 'import'
-  // contextual keyword, so now we're looking for the identifiers.
-  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
-    // We expected to see an identifier here, and we did; continue handling
-    // identifiers.
-    NamedModuleImportPath.push_back(
-        std::make_pair(Result.getIdentifierInfo(), Result.getLocation()));
-    ModuleImportExpectsIdentifier = false;
-    CurLexerCallback = CLK_LexAfterModuleImport;
-    return true;
-  }
-
-  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
-  // see the next identifier. (We can also see a '[[' that begins an
-  // attribute-specifier-seq here under the Standard C++ Modules.)
-  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
-    ModuleImportExpectsIdentifier = true;
-    CurLexerCallback = CLK_LexAfterModuleImport;
-    return true;
   }
 
   // If we didn't recognize a module name at all, this is not a (valid) import.
@@ -1291,24 +1458,6 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
     SemiLoc = Suffix.back().getLocation();
   }
 
-  // Under the standard C++ Modules, the dot is just part of the module name,
-  // and not a real hierarchy separator. Flatten such module names now.
-  //
-  // FIXME: Is this the right level to be performing this transformation?
-  std::string FlatModuleName;
-  if (getLangOpts().CPlusPlusModules) {
-    for (auto &Piece : NamedModuleImportPath) {
-      // If the FlatModuleName ends with colon, it implies it is a partition.
-      if (!FlatModuleName.empty() && FlatModuleName.back() != ':')
-        FlatModuleName += ".";
-      FlatModuleName += Piece.first->getName();
-    }
-    SourceLocation FirstPathLoc = NamedModuleImportPath[0].second;
-    NamedModuleImportPath.clear();
-    NamedModuleImportPath.push_back(
-        std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
-  }
-
   Module *Imported = nullptr;
   // We don't/shouldn't load the standard c++20 modules when preprocessing.
   if (getLangOpts().Modules && !isInImportingCXXNamedModules()) {
@@ -1330,6 +1479,33 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
   return true;
 }
 
+/// Lex a token following the 'module' contextual keyword.
+///
+/// [cpp.module]/p2:
+/// The pp-tokens, if any, of a pp-module shall be of the form:
+///       pp-module-name pp-module-partition[opt] pp-tokens[opt]
+///
+/// where the pp-tokens (if any) shall not begin with a ( preprocessing token
+/// and the grammar non-terminals are defined as:
+///       pp-module-name:
+///             pp-module-name-qualifierp[opt] identifier
+///       pp-module-partition:
+///             : pp-module-name-qualifier[opt] identifier
+///       pp-module-name-qualifier:
+///             identifier .
+///             pp-module-name-qualifier identifier .
+/// No identifier in the pp-module-name or pp-module-partition shall currently
+/// be defined as an object-like macro.
+///
+/// [cpp.module]/p3:
+/// Any preprocessing tokens after the module preprocessing token in the module
+/// directive are processed just as in normal text.
+bool Preprocessor::LexAfterModuleDecl(Token &Result) {
+  // Figure out what kind of lexer we actually have.
+  recomputeCurLexerKind();
+  return LexModuleName(Result, /*IsImport=*/false);
+}
+
 void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc) {
   CurSubmoduleState->VisibleModules.setVisible(
       M, Loc, [](Module *) {},
diff --git a/clang/lib/Lex/TokenConcatenation.cpp b/clang/lib/Lex/TokenConcatenation.cpp
index 865879d180533..cdb636923b9e9 100644
--- a/clang/lib/Lex/TokenConcatenation.cpp
+++ b/clang/lib/Lex/TokenConcatenation.cpp
@@ -160,6 +160,13 @@ static char GetFirstChar(const Preprocessor &PP, const Token &Tok) {
 bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
                                      const Token &PrevTok,
                                      const Token &Tok) const {
+  // If previous token is a module name, we need avoid concat it with current
+  // token, otherwise, there will has an extra space between 'M' and ';' for the
+  // following code:
+  //
+  // import M;
+  if (PrevTok.is(tok::annot_module_name))
+    return false;
   // Conservatively assume that every annotation token that has a printable
   // form requires whitespace.
   if (PrevTok.isAnnotation())
@@ -190,6 +197,9 @@ bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
       return true;
     ConcatInfo &= ~aci_avoid_equal;
   }
+
+  if (Tok.is(tok::annot_module_name))
+    return true;
   if (Tok.isAnnotation()) {
     // Modules annotation can show up when generated automatically for includes.
     assert(Tok.isOneOf(tok::annot_module_include, tok::annot_module_begin,
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 7ce9a9cea1c7a..577527d0318f2 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -3958,7 +3958,13 @@ void Parser::ParseDeclarationSpecifiers(
 
       // We're done with the declaration-specifiers.
       goto DoneWithDeclSpec;
-
+    case tok::annot_module_name: {
+      PP.EnterTokenStream(
+          Tok.getAnnotationValueAs<ModuleNameInfo *>()->getTokens(),
+          /*DisableMacroExpansion=*/true, /*IsReinject=*/false);
+      ConsumeAnyToken();
+      [[fallthrough]];
+    }
       // typedef-name
     case tok::kw___super:
     case tok::kw_decltype:
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index 5ebe71e496a2e..afb2e1e416168 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -2511,18 +2511,28 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) {
   }
 
   SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
-  if (ParseModuleName(ModuleLoc, Path, /*IsImport*/ false))
+  if (Tok.isNot(tok::annot_module_name)) {
+    Diag(Tok, diag::err_module_expected_ident) << /*IsImport=*/false;
+    SkipUntil(tok::semi, StopBeforeMatch);
+    return nullptr;
+  }
+
+  auto *Info = Tok.getAnnotationValueAs<ModuleNameInfo *>();
+  ConsumeAnnotationToken();
+  if (ParseModuleName(ModuleLoc, Info->getModuleName(), Path,
+                      /*IsImport=*/false))
     return nullptr;
 
   // Parse the optional module-partition.
   SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Partition;
-  if (Tok.is(tok::colon)) {
-    SourceLocation ColonLoc = ConsumeToken();
+  if (Info->hasPartitionName()) {
+    SourceLocation ColonLoc = Info->getColonToken().getLocation();
     if (!getLangOpts().CPlusPlusModules)
       Diag(ColonLoc, diag::err_unsupported_module_partition)
           << SourceRange(ColonLoc, Partition.back().second);
     // Recover by ignoring the partition name.
-    else if (ParseModuleName(ModuleLoc, Partition, /*IsImport*/ false))
+    else if (ParseModuleName(ModuleLoc, Info->getPartitionName(), Partition,
+                             /*IsImport=*/false))
       return nullptr;
   }
 
@@ -2581,18 +2591,32 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
     // This is a header import that the preprocessor mapped to a module import.
     HeaderUnit = reinterpret_cast<Module *>(Tok.getAnnotationValue());
     ConsumeAnnotationToken();
-  } else if (Tok.is(tok::colon)) {
-    SourceLocation ColonLoc = ConsumeToken();
-    if (!getLangOpts().CPlusPlusModules)
-      Diag(ColonLoc, diag::err_unsupported_module_partition)
-          << SourceRange(ColonLoc, Path.back().second);
-    // Recover by leaving partition empty.
-    else if (ParseModuleName(ColonLoc, Path, /*IsImport*/ true))
-      return nullptr;
-    else
-      IsPartition = true;
   } else {
-    if (ParseModuleName(ImportLoc, Path, /*IsImport*/ true))
+    if (Tok.isNot(tok::annot_module_name)) {
+      if (Tok.is(tok::code_completion)) {
+        cutOffParsing();
+        Actions.CodeCompletion().CodeCompleteModuleImport(ImportLoc, Path);
+        return nullptr;
+      }
+      Diag(Tok, diag::err_module_expected_ident) << /*IsImport=*/true;
+      SkipUntil(tok::semi, StopBeforeMatch);
+      return nullptr;
+    }
+    auto *Info = Tok.getAnnotationValueAs<ModuleNameInfo *>();
+    ConsumeAnnotationToken();
+    if (Info->hasPartitionName()) {
+      SourceLocation ColonLoc = Info->getColonToken().getLocation();
+      if (!getLangOpts().CPlusPlusModules)
+        Diag(ColonLoc, diag::err_unsupported_module_partition)
+            << SourceRange(ColonLoc, Path.back().second);
+      // Recover by leaving partition empty.
+      else if (ParseModuleName(ColonLoc, Info->getPartitionName(), Path,
+                               /*IsImport=*/true))
+        return nullptr;
+      else
+        IsPartition = true;
+    } else if (ParseModuleName(ImportLoc, Info->getModuleName(), Path,
+                               /*IsImport=*/true))
       return nullptr;
   }
 
@@ -2689,32 +2713,31 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
 ///         module-name-qualifier:
 ///           module-name-qualifier[opt] identifier '.'
 bool Parser::ParseModuleName(
-    SourceLocation UseLoc,
+    SourceLocation UseLoc, ArrayRef<Token> ModuleName,
     SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path,
     bool IsImport) {
-  // Parse the module path.
-  while (true) {
-    if (!Tok.is(tok::identifier)) {
-      if (Tok.is(tok::code_completion)) {
-        cutOffParsing();
-        Actions.CodeCompletion().CodeCompleteModuleImport(UseLoc, Path);
-        return true;
-      }
-
-      Diag(Tok, diag::err_module_expected_ident) << IsImport;
-      SkipUntil(tok::semi);
+  ModuleNameInfo::getModuleIdPath(ModuleName, Path);
+  // Eg. import A.B.
+  if (ModuleName.back().isNot(tok::identifier)) {
+    if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
+      Actions.CodeCompletion().CodeCompleteModuleImport(UseLoc, Path);
       return true;
     }
+    Diag(ModuleName.back(), diag::err_module_expected_ident) << IsImport;
+    SkipUntil(tok::semi, StopBeforeMatch);
+    return true;
+  }
 
-    // Record this part of the module path.
-    Path.push_back(std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation()));
-    ConsumeToken();
-
-    if (Tok.isNot(tok::period))
-      return false;
-
-    ConsumeToken();
+  // [cpp.module]/p2: where the pp-tokens (if any) shall not begin with a (
+  // preprocessing token [...]
+  //
+  // Skip unitl ';' to recovery.
+  if (getLangOpts().CPlusPlusModules && Tok.is(tok::l_paren)) {
+    SkipUntil(tok::semi, StopBeforeMatch);
+    return true;
   }
+  return false;
 }
 
 /// Try recover parser when module annotation appears where it must not
diff --git a/clang/test/CXX/cpp/cpp.module/p2.cppm b/clang/test/CXX/cpp/cpp.module/p2.cppm
new file mode 100644
index 0000000000000..966a88ccfa972
--- /dev/null
+++ b/clang/test/CXX/cpp/cpp.module/p2.cppm
@@ -0,0 +1,88 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -std=c++20 %t/A.cppm -triple x86_64-linux-gnu -verify
+// RUN: %clang_cc1 -std=c++20 %t/B.cppm -triple x86_64-linux-gnu -verify
+// RUN: %clang_cc1 -std=c++20 %t/C.cppm -triple x86_64-linux-gnu -verify
+// RUN: %clang_cc1 -std=c++20 %t/D.cppm -triple x86_64-linux-gnu -verify
+// RUN: %clang_cc1 -std=c++20 %t/E.cppm -triple x86_64-linux-gnu -verify
+// RUN: %clang_cc1 -std=c++20 %t/F.cppm -triple x86_64-linux-gnu -verify
+// RUN: %clang_cc1 -std=c++20 %t/G.cppm -triple x86_64-linux-gnu -verify
+// RUN: %clang_cc1 -std=c++20 %t/H.cppm -triple x86_64-linux-gnu -verify
+// RUN: %clang_cc1 -std=c++20 %t/I.cppm -triple x86_64-linux-gnu -verify
+// RUN: %clang_cc1 -std=c++20 %t/J.cppm -triple x86_64-linux-gnu -verify
+
+//--- version.h
+#ifndef VERSION_H
+#define VERSION_H
+
+#define VERSION libv5
+#define A a
+#define B b
+#define C c
+#define FUNC_LIKE(X) function_like_##X
+#define ATTRS [[]]
+#define SEMICOLON ;
+
+#endif // VERSION_H
+
+//--- A.cppm
+module;
+#include "version.h"
+export module VERSION;  // expected-error {{the module name in a module declaration cannot contain an object-like macro 'VERSION'}}
+
+//--- B.cppm
+module;
+#include "version.h"
+export module A.B;      // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
+                        // expected-error {{the module name in a module declaration cannot contain an object-like macro 'B'}}
+
+//--- C.cppm
+module;                             // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
+#include "version.h"
+export module A.FUNC_LIKE(foo):C;   // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
+                                    // expected-error {{unexpected '(' after the module name in a module declaration}}
+
+//--- D.cppm
+module;                               // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
+#include "version.h"
+export module B.A.FUNC_LIKE(bar):C;   // expected-error {{the module name in a module declaration cannot contain an object-like macro 'B'}} \
+                                      // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
+                                      // expected-error {{unexpected '(' after the module name in a module declaration}}
+
+//--- E.cppm
+module;
+#include "version.h"
+export module a.FUNC_LIKE:c; // OK, FUNC_LIKE would not be treated as a macro name.
+// expected-no-diagnostics
+
+//--- F.cppm
+module;
+#include "version.h"
+export module a.FUNC_LIKE:c ATTRS; // OK, FUNC_LIKE would not be treated as a macro name.
+// expected-no-diagnostics
+
+//--- G.cppm
+module;                               // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
+#include "version.h"
+export module A.FUNC_LIKE(B c:C ATTRS // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
+                                      // expected-error {{unexpected '(' after the module name in a module declaration}}
+
+//--- H.cppm
+module;                                   // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
+#include "version.h"
+export module A.FUNC_LIKE(B,). c:C ATTRS  // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
+                                          // expected-error {{unexpected '(' after the module name in a module declaration}}
+
+//--- I.cppm
+module;                                   // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
+#include "version.h"
+export module A.FUNC_LIKE(B,) c:C ATTRS   // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
+                                          // expected-error {{unexpected '(' after the module name in a module declaration}}
+
+//--- J.cppm
+module;
+#include "version.h"
+export module unexpanded : unexpanded ATTRS SEMICOLON // OK, ATTRS and SEMICOLON can be expanded.
+// expected-no-diagnostics
diff --git a/clang/test/CXX/module/basic/basic.link/module-declaration.cpp b/clang/test/CXX/module/basic/basic.link/module-declaration.cpp
index d71358cc7a571..14bbc911febfc 100644
--- a/clang/test/CXX/module/basic/basic.link/module-declaration.cpp
+++ b/clang/test/CXX/module/basic/basic.link/module-declaration.cpp
@@ -8,27 +8,19 @@
 // RUN: %clang_cc1 -std=c++20 -emit-module-interface -fmodule-file=x=%t/x.pcm %t/x.y.cppm -o %t/x.y.pcm
 //
 // Module implementation for unknown and known module. (The former is ill-formed.)
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify -x c++ %t/M.cpp \
-// RUN:            -DTEST=1 -DEXPORT= -DMODULE_NAME=z
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x=%t/x.pcm -fmodule-file=x.y=%t/x.y.pcm -verify -x c++ %t/M.cpp \
-// RUN:            -DTEST=2 -DEXPORT= -DMODULE_NAME=x
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify -x c++ %t/M1.cpp
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x=%t/x.pcm -fmodule-file=x.y=%t/x.y.pcm -verify -x c++ %t/M2.cpp
 //
 // Module interface for unknown and known module. (The latter is ill-formed due to
 // redefinition.)
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
-// RUN:            -DTEST=3 -DEXPORT=export -DMODULE_NAME=z
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
-// RUN:            -DTEST=4 -DEXPORT=export -DMODULE_NAME=x
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M3.cpp
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M4.cpp
 //
 // Miscellaneous syntax.
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
-// RUN:            -DTEST=7 -DEXPORT=export -DMODULE_NAME='z elderberry'
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
-// RUN:            -DTEST=8 -DEXPORT=export -DMODULE_NAME='z [[]]'
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
-// RUN:            -DTEST=9 -DEXPORT=export -DMODULE_NAME='z [[fancy]]'
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
-// RUN:            -DTEST=10 -DEXPORT=export -DMODULE_NAME='z [[maybe_unused]]'
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M5.cpp
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M6.cpp
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M7.cpp
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M8.cpp
 
 //--- x.cppm
 export module x;
@@ -38,17 +30,26 @@ int a, b;
 export module x.y;
 int c;
 
-//--- M.cpp
-
-EXPORT module MODULE_NAME;
-#if TEST == 7
-// expected-error@-2 {{expected ';'}} expected-error@-2 {{a type specifier is required}}
-#elif TEST == 9
-// expected-warning@-4 {{unknown attribute 'fancy' ignored}}
-#elif TEST == 10
-// expected-error-re@-6 {{'maybe_unused' attribute cannot be applied to a module{{$}}}}
-#elif TEST == 1
-// expected-error@-8 {{module 'z' not found}}
-#else
-// expected-no-diagnostics
-#endif
+//--- M1.cpp
+module z; // expected-error {{module 'z' not found}}
+
+//--- M2.cpp
+module x; // expected-no-diagnostics
+
+//--- M3.cpp
+export module z; // expected-no-diagnostics
+
+//--- M4.cpp
+export module x; // expected-no-diagnostics
+
+//--- M5.cpp
+export module z elderberry; // expected-error {{expected ';'}} expected-error {{a type specifier is required}}
+
+//--- M6.cpp
+export module z [[]]; // expected-no-diagnostics
+
+//--- M7.cpp
+export module z [[fancy]]; // expected-warning {{unknown attribute 'fancy' ignored}}
+
+//--- M8.cpp
+export module z [[maybe_unused]]; // expected-error-re {{'maybe_unused' attribute cannot be applied to a module{{$}}}}
diff --git a/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm b/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm
index 873e4c0edeac2..ecad4db32a7e9 100644
--- a/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm
+++ b/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm
@@ -6,10 +6,12 @@
 // RUN: %clang_cc1 -std=c++20 -emit-module-interface -fmodule-file=x=%t/x.pcm %t/x.y.cppm -o %t/x.y.pcm
 // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/a.b.cppm -o %t/a.b.pcm
 //
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -fmodule-file=x=%t/x.pcm -verify %t/test.cpp \
-// RUN:            -DMODULE_NAME=z -DINTERFACE
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -fmodule-file=x=%t/x.pcm -verify %t/test-interface.cpp \
+// RUN:            -DINTERFACE
 // RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -fmodule-file=x=%t/x.pcm \
-// RUN:            -fmodule-file=a.b=%t/a.b.pcm -verify %t/test.cpp -DMODULE_NAME=a.b
+// RUN:            -fmodule-file=a.b=%t/a.b.pcm -verify %t/test.cpp
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -fmodule-file=x=%t/x.pcm \
+// RUN:            -verify %t/test-module-not-found.cpp
 // RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -fmodule-file=x=%t/x.pcm -verify %t/test.x.cpp
 
 //--- x.cppm
@@ -34,11 +36,8 @@ int use_2 = b; // ok
 int use_3 = c; // expected-error {{use of undeclared identifier 'c'}}
 
 //--- test.cpp
-#ifdef INTERFACE
-export module MODULE_NAME;
-#else
-module MODULE_NAME;
-#endif
+module;
+module a.b;
 
 import x;
 
@@ -51,6 +50,28 @@ import x.y;
 import x.; // expected-error {{expected a module name after 'import'}}
 import .x; // expected-error {{expected a module name after 'import'}}
 
-import blarg; // expected-error {{module 'blarg' not found}}
+int use_4 = c; // ok
+
+
+//--- test-interface.cpp
+module;
+export module z;
+
+import x;
+
+import x [[]];
+import x [[foo]]; // expected-warning {{unknown attribute 'foo' ignored}}
+import x [[noreturn]]; // expected-error {{'noreturn' attribute cannot be applied to a module import}}
+import x [[blarg::noreturn]]; // expected-warning {{unknown attribute 'noreturn' ignored}}
+
+import x.y;
+import x.; // expected-error {{expected a module name after 'import'}}
+import .x; // expected-error {{expected a module name after 'import'}}
 
 int use_4 = c; // ok
+
+//--- test-module-not-found.cpp
+module;
+
+import blarg; // expected-error {{module 'blarg' not found}}
+
diff --git a/clang/test/SemaCXX/modules.cppm b/clang/test/SemaCXX/modules.cppm
index 41204be76eafa..267417bf5da2c 100644
--- a/clang/test/SemaCXX/modules.cppm
+++ b/clang/test/SemaCXX/modules.cppm
@@ -1,19 +1,17 @@
-// RUN:     %clang_cc1 -std=c++20 -emit-module-interface %s -o %t.0.pcm -verify -DTEST=0
-// RUN:     %clang_cc1 -std=c++20 -emit-module-interface %s -o %t.1.pcm -verify -DTEST=1
-// RUN:     %clang_cc1 -std=c++20 -emit-module-interface %s -fmodule-file=foo=%t.0.pcm -o %t.2.pcm -verify -DTEST=2
-// RUN:     %clang_cc1 -std=c++20 -emit-module-interface %s -fmodule-file=foo=%t.0.pcm -o %t.3.pcm -verify -Dfoo=bar -DTEST=3
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
 
-#if TEST == 0 || TEST == 2
-// expected-no-diagnostics
-#endif
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/A.cppm -o %t.0.pcm -verify
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/B.cppm -o %t.1.pcm -verify
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/C.cppm -fmodule-file=foo=%t.0.pcm -o %t.2.pcm -verify
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/D.cppm -fmodule-file=foo=%t.0.pcm -o %t.3.pcm -verify
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/E.cppm -fmodule-file=foo=%t.0.pcm -o %t.3.pcm -verify -Dfoo=bar
 
+//--- A.cppm
 export module foo;
-
 static int m;
-
 int n;
-
-#if TEST == 0
 export {
   int a;
   int b;
@@ -27,7 +25,43 @@ export void f() {}
 
 export struct T {
 } t;
-#elif TEST == 3
+// expected-no-diagnostics
+
+//--- B.cppm
+export module foo;
+static int m;
+int n;
+struct S {
+  export int n;        // expected-error {{expected member name or ';'}}
+  export static int n; // expected-error {{expected member name or ';'}}
+};
+
+// FIXME: Exports of declarations without external linkage are disallowed.
+// Exports of declarations with non-external-linkage types are disallowed.
+
+// Cannot export within another export. This isn't precisely covered by the
+// language rules right now, but (per personal correspondence between zygoloid
+// and gdr) is the intent.
+export { // expected-note {{export block begins here}}
+  extern "C++" {
+    namespace NestedExport {
+      export { // expected-error {{export declaration appears within another export declaration}}
+        int q;
+      }
+    } // namespace NestedExport
+  }
+}
+
+//--- C.cppm
+export module foo;
+static int m;
+int n;
+// expected-no-diagnostics
+
+//--- D.cppm
+export module foo;
+static int m;
+int n;
 int use_a = a; // expected-error {{use of undeclared identifier 'a'}}
 
 #undef foo
@@ -46,29 +80,12 @@ int use_n = n; // FIXME: this should not be visible, because it is not exported
 
 extern int n;
 static_assert(&n != p); // expected-error{{use of undeclared identifier 'p'}}
-#endif
 
-#if TEST == 1
-struct S {
-  export int n;        // expected-error {{expected member name or ';'}}
-  export static int n; // expected-error {{expected member name or ';'}}
-};
-#endif
-
-// FIXME: Exports of declarations without external linkage are disallowed.
-// Exports of declarations with non-external-linkage types are disallowed.
+//--- E.cppm
+export module foo; // expected-error {{the module name in a module declaration cannot contain an object-like macro 'foo'}}
+static int m;
+int n;
+int use_a = a; // expected-error {{use of undeclared identifier 'a'}}
 
-// Cannot export within another export. This isn't precisely covered by the
-// language rules right now, but (per personal correspondence between zygoloid
-// and gdr) is the intent.
-#if TEST == 1
-export { // expected-note {{export block begins here}}
-  extern "C++" {
-  namespace NestedExport {
-  export { // expected-error {{export declaration appears within another export declaration}}
-    int q;
-  }
-  } // namespace NestedExport
-  }
-}
-#endif
+#undef foo
+import foo; // expected-error {{imports must immediately follow the module declaration}}
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index a6ded8be3ae9e..1f69a4e8a5620 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -182,7 +182,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Module Declarations Shouldn’t be Macros</td>
   <td><a href="https://wg21.link/P3034R1">P3034R1</a> (<a href="#dr">DR</a>)</td>
-  <td class="none" align="center">No</td>
+  <td class="unreleased" align="center">Clang 19</td>
  </tr>
  <tr>
   <td>Trivial infinite loops are not Undefined Behavior</td>

From 7d80ee5bdabbcb25b15fe54297d3f13793e4d8c2 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 20 Jul 2024 17:15:46 +0400
Subject: [PATCH 285/486] CodeGen: Use IR function to query Module instead of
 MachineModuleInfo (#99755)

---
 llvm/lib/CodeGen/CFGuardLongjmp.cpp               | 2 +-
 llvm/lib/CodeGen/EHContGuardCatchret.cpp          | 2 +-
 llvm/lib/CodeGen/KCFI.cpp                         | 2 +-
 llvm/lib/Target/X86/X86AsmPrinter.cpp             | 7 +++----
 llvm/lib/Target/X86/X86FrameLowering.cpp          | 7 +++----
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp           | 3 ++-
 llvm/lib/Target/X86/X86ISelLowering.cpp           | 6 +++---
 llvm/lib/Target/X86/X86IndirectBranchTracking.cpp | 2 +-
 llvm/lib/Target/X86/X86ReturnThunks.cpp           | 2 +-
 9 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/CodeGen/CFGuardLongjmp.cpp b/llvm/lib/CodeGen/CFGuardLongjmp.cpp
index b5d88a7432b17..04de011400568 100644
--- a/llvm/lib/CodeGen/CFGuardLongjmp.cpp
+++ b/llvm/lib/CodeGen/CFGuardLongjmp.cpp
@@ -62,7 +62,7 @@ FunctionPass *llvm::createCFGuardLongjmpPass() { return new CFGuardLongjmp(); }
 bool CFGuardLongjmp::runOnMachineFunction(MachineFunction &MF) {
 
   // Skip modules for which the cfguard flag is not set.
-  if (!MF.getMMI().getModule()->getModuleFlag("cfguard"))
+  if (!MF.getFunction().getParent()->getModuleFlag("cfguard"))
     return false;
 
   // Skip functions that do not have calls to _setjmp.
diff --git a/llvm/lib/CodeGen/EHContGuardCatchret.cpp b/llvm/lib/CodeGen/EHContGuardCatchret.cpp
index f7c6580a73da5..cd1cdb0653618 100644
--- a/llvm/lib/CodeGen/EHContGuardCatchret.cpp
+++ b/llvm/lib/CodeGen/EHContGuardCatchret.cpp
@@ -62,7 +62,7 @@ FunctionPass *llvm::createEHContGuardCatchretPass() {
 bool EHContGuardCatchret::runOnMachineFunction(MachineFunction &MF) {
 
   // Skip modules for which the ehcontguard flag is not set.
-  if (!MF.getMMI().getModule()->getModuleFlag("ehcontguard"))
+  if (!MF.getFunction().getParent()->getModuleFlag("ehcontguard"))
     return false;
 
   // Skip functions that do not have catchret
diff --git a/llvm/lib/CodeGen/KCFI.cpp b/llvm/lib/CodeGen/KCFI.cpp
index 91c6ac2618279..af19319bc1bb8 100644
--- a/llvm/lib/CodeGen/KCFI.cpp
+++ b/llvm/lib/CodeGen/KCFI.cpp
@@ -89,7 +89,7 @@ bool KCFI::emitCheck(MachineBasicBlock &MBB,
 }
 
 bool KCFI::runOnMachineFunction(MachineFunction &MF) {
-  const Module *M = MF.getMMI().getModule();
+  const Module *M = MF.getFunction().getParent();
   if (!M->getModuleFlag("kcfi"))
     return false;
 
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index 3395a13545e45..0c2c6bf7f8b70 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -66,11 +66,10 @@ bool X86AsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   CodeEmitter.reset(TM.getTarget().createMCCodeEmitter(
       *Subtarget->getInstrInfo(), MF.getContext()));
 
-  EmitFPOData =
-      Subtarget->isTargetWin32() && MF.getMMI().getModule()->getCodeViewFlag();
+  const Module *M = MF.getFunction().getParent();
+  EmitFPOData = Subtarget->isTargetWin32() && M->getCodeViewFlag();
 
-  IndCSPrefix =
-      MF.getMMI().getModule()->getModuleFlag("indirect_branch_cs_prefix");
+  IndCSPrefix = M->getModuleFlag("indirect_branch_cs_prefix");
 
   SetupMachineFunction(MF);
 
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 89801783e9280..0ff50d8ef678e 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -1530,7 +1530,6 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF,
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   const Function &Fn = MF.getFunction();
-  MachineModuleInfo &MMI = MF.getMMI();
   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
   uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment.
   uint64_t StackSize = MFI.getStackSize(); // Number of bytes to allocate.
@@ -1545,8 +1544,8 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF,
   bool IsWin64Prologue = isWin64Prologue(MF);
   bool NeedsWin64CFI = IsWin64Prologue && Fn.needsUnwindTableEntry();
   // FIXME: Emit FPO data for EH funclets.
-  bool NeedsWinFPO =
-      !IsFunclet && STI.isTargetWin32() && MMI.getModule()->getCodeViewFlag();
+  bool NeedsWinFPO = !IsFunclet && STI.isTargetWin32() &&
+                     MF.getFunction().getParent()->getCodeViewFlag();
   bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO;
   bool NeedsDwarfCFI = needsDwarfCFI(MF);
   Register FramePtr = TRI->getFrameRegister(MF);
@@ -3521,7 +3520,7 @@ void X86FrameLowering::adjustForHiPEPrologue(
 
   // HiPE-specific values
   NamedMDNode *HiPELiteralsMD =
-      MF.getMMI().getModule()->getNamedMetadata("hipe.literals");
+      MF.getFunction().getParent()->getNamedMetadata("hipe.literals");
   if (!HiPELiteralsMD)
     report_fatal_error(
         "Can't generate HiPE prologue without runtime parameters");
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index c91bd576dc9f6..74804e5c9783d 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -922,7 +922,8 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
       if (Imm == EndbrImm || isEndbrImm64(Imm)) {
         // Check that the cf-protection-branch is enabled.
         Metadata *CFProtectionBranch =
-          MF->getMMI().getModule()->getModuleFlag("cf-protection-branch");
+            MF->getFunction().getParent()->getModuleFlag(
+                "cf-protection-branch");
         if (CFProtectionBranch || IndirectBranchTracking) {
           SDLoc dl(N);
           SDValue Complement = CurDAG->getConstant(~Imm, dl, VT, false, true);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 890728ba088dc..2959902c78675 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -35882,7 +35882,7 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
     MIB.addMBB(restoreMBB);
   MIB.setMemRefs(MMOs);
 
-  if (MF->getMMI().getModule()->getModuleFlag("cf-protection-return")) {
+  if (MF->getFunction().getParent()->getModuleFlag("cf-protection-return")) {
     emitSetJmpShadowStackFix(MI, thisMBB);
   }
 
@@ -36158,7 +36158,7 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
   MachineBasicBlock *thisMBB = MBB;
 
   // When CET and shadow stack is enabled, we need to fix the Shadow Stack.
-  if (MF->getMMI().getModule()->getModuleFlag("cf-protection-return")) {
+  if (MF->getFunction().getParent()->getModuleFlag("cf-protection-return")) {
     thisMBB = emitLongJmpShadowStackFix(MI, thisMBB);
   }
 
@@ -57981,7 +57981,7 @@ SDValue X86TargetLowering::expandIndirectJTBranch(const SDLoc &dl,
                                                   SDValue Value, SDValue Addr,
                                                   int JTI,
                                                   SelectionDAG &DAG) const {
-  const Module *M = DAG.getMachineFunction().getMMI().getModule();
+  const Module *M = DAG.getMachineFunction().getFunction().getParent();
   Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch");
   if (IsCFProtectionSupported) {
     // In case control-flow branch protection is enabled, we need to add
diff --git a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
index 381286a3bbfc0..7740a174af4f3 100644
--- a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
+++ b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
@@ -116,7 +116,7 @@ static bool needsPrologueENDBR(MachineFunction &MF, const Module *M) {
 bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) {
   const X86Subtarget &SubTarget = MF.getSubtarget<X86Subtarget>();
 
-  const Module *M = MF.getMMI().getModule();
+  const Module *M = MF.getFunction().getParent();
   // Check that the cf-protection-branch is enabled.
   Metadata *isCFProtectionSupported = M->getModuleFlag("cf-protection-branch");
 
diff --git a/llvm/lib/Target/X86/X86ReturnThunks.cpp b/llvm/lib/Target/X86/X86ReturnThunks.cpp
index fe89238f26f97..c40b4f371fb31 100644
--- a/llvm/lib/Target/X86/X86ReturnThunks.cpp
+++ b/llvm/lib/Target/X86/X86ReturnThunks.cpp
@@ -78,7 +78,7 @@ bool X86ReturnThunks::runOnMachineFunction(MachineFunction &MF) {
         Rets.push_back(&Term);
 
   bool IndCS =
-      MF.getMMI().getModule()->getModuleFlag("indirect_branch_cs_prefix");
+      MF.getFunction().getParent()->getModuleFlag("indirect_branch_cs_prefix");
   const MCInstrDesc &CS = ST.getInstrInfo()->get(X86::CS_PREFIX);
   const MCInstrDesc &JMP = ST.getInstrInfo()->get(X86::TAILJMPd);
 

From 56a9f7ce611ba21f51043d91c965b59e116013f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sat, 20 Jul 2024 15:25:00 +0200
Subject: [PATCH 286/486] [clang][Interp] Pass ASTContext to toAPValue()

Not yet needed, but we need to ASTContext in a later patch when we start
computing proper values for the APValue offset.
---
 clang/lib/AST/Interp/Boolean.h            |  2 +-
 clang/lib/AST/Interp/Disasm.cpp           |  4 ++--
 clang/lib/AST/Interp/EvalEmitter.cpp      |  7 ++++---
 clang/lib/AST/Interp/EvaluationResult.cpp |  6 +++---
 clang/lib/AST/Interp/Floating.h           |  2 +-
 clang/lib/AST/Interp/FunctionPointer.h    |  4 ++--
 clang/lib/AST/Interp/Integral.h           |  2 +-
 clang/lib/AST/Interp/IntegralAP.h         |  2 +-
 clang/lib/AST/Interp/Interp.h             |  9 +++++----
 clang/lib/AST/Interp/MemberPointer.cpp    |  4 ++--
 clang/lib/AST/Interp/MemberPointer.h      |  2 +-
 clang/lib/AST/Interp/Pointer.cpp          | 22 ++++++++++++----------
 clang/lib/AST/Interp/Pointer.h            |  2 +-
 clang/unittests/AST/Interp/toAPValue.cpp  | 20 ++++++++++++--------
 14 files changed, 48 insertions(+), 40 deletions(-)

diff --git a/clang/lib/AST/Interp/Boolean.h b/clang/lib/AST/Interp/Boolean.h
index 1bfb26b1b669f..23f7286036764 100644
--- a/clang/lib/AST/Interp/Boolean.h
+++ b/clang/lib/AST/Interp/Boolean.h
@@ -56,7 +56,7 @@ class Boolean final {
   APSInt toAPSInt(unsigned NumBits) const {
     return APSInt(toAPSInt().zextOrTrunc(NumBits), true);
   }
-  APValue toAPValue() const { return APValue(toAPSInt()); }
+  APValue toAPValue(const ASTContext &) const { return APValue(toAPSInt()); }
 
   Boolean toUnsigned() const { return *this; }
 
diff --git a/clang/lib/AST/Interp/Disasm.cpp b/clang/lib/AST/Interp/Disasm.cpp
index c6c6275593007..867284ecf7f4b 100644
--- a/clang/lib/AST/Interp/Disasm.cpp
+++ b/clang/lib/AST/Interp/Disasm.cpp
@@ -366,9 +366,9 @@ LLVM_DUMP_METHOD void EvaluationResult::dump() const {
 
     OS << "LValue: ";
     if (const auto *P = std::get_if<Pointer>(&Value))
-      P->toAPValue().printPretty(OS, ASTCtx, SourceType);
+      P->toAPValue(ASTCtx).printPretty(OS, ASTCtx, SourceType);
     else if (const auto *FP = std::get_if<FunctionPointer>(&Value)) // Nope
-      FP->toAPValue().printPretty(OS, ASTCtx, SourceType);
+      FP->toAPValue(ASTCtx).printPretty(OS, ASTCtx, SourceType);
     OS << "\n";
     break;
   }
diff --git a/clang/lib/AST/Interp/EvalEmitter.cpp b/clang/lib/AST/Interp/EvalEmitter.cpp
index 221bbfdc542ff..08536536ac3c2 100644
--- a/clang/lib/AST/Interp/EvalEmitter.cpp
+++ b/clang/lib/AST/Interp/EvalEmitter.cpp
@@ -145,7 +145,7 @@ template <PrimType OpType> bool EvalEmitter::emitRet(const SourceInfo &Info) {
     return false;
 
   using T = typename PrimConv<OpType>::T;
-  EvalResult.setValue(S.Stk.pop<T>().toAPValue());
+  EvalResult.setValue(S.Stk.pop<T>().toAPValue(Ctx.getASTContext()));
   return true;
 }
 
@@ -181,7 +181,7 @@ template <> bool EvalEmitter::emitRet<PT_Ptr>(const SourceInfo &Info) {
       return false;
     }
   } else {
-    EvalResult.setValue(Ptr.toAPValue());
+    EvalResult.setValue(Ptr.toAPValue(Ctx.getASTContext()));
   }
 
   return true;
@@ -285,7 +285,8 @@ void EvalEmitter::updateGlobalTemporaries() {
       APValue *Cached = Temp->getOrCreateValue(true);
 
       if (std::optional<PrimType> T = Ctx.classify(E->getType())) {
-        TYPE_SWITCH(*T, { *Cached = Ptr.deref<T>().toAPValue(); });
+        TYPE_SWITCH(
+            *T, { *Cached = Ptr.deref<T>().toAPValue(Ctx.getASTContext()); });
       } else {
         if (std::optional<APValue> APV =
                 Ptr.toRValue(Ctx, Temp->getTemporaryExpr()->getType()))
diff --git a/clang/lib/AST/Interp/EvaluationResult.cpp b/clang/lib/AST/Interp/EvaluationResult.cpp
index 0bebfd4ad984e..1b255711c7b36 100644
--- a/clang/lib/AST/Interp/EvaluationResult.cpp
+++ b/clang/lib/AST/Interp/EvaluationResult.cpp
@@ -21,9 +21,9 @@ APValue EvaluationResult::toAPValue() const {
   case LValue:
     // Either a pointer or a function pointer.
     if (const auto *P = std::get_if<Pointer>(&Value))
-      return P->toAPValue();
+      return P->toAPValue(Ctx->getASTContext());
     else if (const auto *FP = std::get_if<FunctionPointer>(&Value))
-      return FP->toAPValue();
+      return FP->toAPValue(Ctx->getASTContext());
     else
       llvm_unreachable("Unhandled LValue type");
     break;
@@ -46,7 +46,7 @@ std::optional<APValue> EvaluationResult::toRValue() const {
   if (const auto *P = std::get_if<Pointer>(&Value))
     return P->toRValue(*Ctx, getSourceType());
   else if (const auto *FP = std::get_if<FunctionPointer>(&Value)) // Nope
-    return FP->toAPValue();
+    return FP->toAPValue(Ctx->getASTContext());
   llvm_unreachable("Unhandled lvalue kind");
 }
 
diff --git a/clang/lib/AST/Interp/Floating.h b/clang/lib/AST/Interp/Floating.h
index e4ac76d8509fb..114487821880f 100644
--- a/clang/lib/AST/Interp/Floating.h
+++ b/clang/lib/AST/Interp/Floating.h
@@ -69,7 +69,7 @@ class Floating final {
   APSInt toAPSInt(unsigned NumBits = 0) const {
     return APSInt(F.bitcastToAPInt());
   }
-  APValue toAPValue() const { return APValue(F); }
+  APValue toAPValue(const ASTContext &) const { return APValue(F); }
   void print(llvm::raw_ostream &OS) const {
     // Can't use APFloat::print() since it appends a newline.
     SmallVector<char, 16> Buffer;
diff --git a/clang/lib/AST/Interp/FunctionPointer.h b/clang/lib/AST/Interp/FunctionPointer.h
index fc3d7a4214a72..0f2c6e571a1d8 100644
--- a/clang/lib/AST/Interp/FunctionPointer.h
+++ b/clang/lib/AST/Interp/FunctionPointer.h
@@ -40,7 +40,7 @@ class FunctionPointer final {
     return Func->getDecl()->isWeak();
   }
 
-  APValue toAPValue() const {
+  APValue toAPValue(const ASTContext &) const {
     if (!Func)
       return APValue(static_cast<Expr *>(nullptr), CharUnits::Zero(), {},
                      /*OnePastTheEnd=*/false, /*IsNull=*/true);
@@ -69,7 +69,7 @@ class FunctionPointer final {
     if (!Func)
       return "nullptr";
 
-    return toAPValue().getAsString(Ctx, Func->getDecl()->getType());
+    return toAPValue(Ctx).getAsString(Ctx, Func->getDecl()->getType());
   }
 
   uint64_t getIntegerRepresentation() const {
diff --git a/clang/lib/AST/Interp/Integral.h b/clang/lib/AST/Interp/Integral.h
index db4cc9ae45b49..aafdd02676c96 100644
--- a/clang/lib/AST/Interp/Integral.h
+++ b/clang/lib/AST/Interp/Integral.h
@@ -112,7 +112,7 @@ template <unsigned Bits, bool Signed> class Integral final {
     else
       return APSInt(toAPSInt().zextOrTrunc(NumBits), !Signed);
   }
-  APValue toAPValue() const { return APValue(toAPSInt()); }
+  APValue toAPValue(const ASTContext &) const { return APValue(toAPSInt()); }
 
   Integral<Bits, false> toUnsigned() const {
     return Integral<Bits, false>(*this);
diff --git a/clang/lib/AST/Interp/IntegralAP.h b/clang/lib/AST/Interp/IntegralAP.h
index 7464f15cdb03b..b8aa21038256c 100644
--- a/clang/lib/AST/Interp/IntegralAP.h
+++ b/clang/lib/AST/Interp/IntegralAP.h
@@ -133,7 +133,7 @@ template <bool Signed> class IntegralAP final {
     else
       return APSInt(V.zext(Bits), !Signed);
   }
-  APValue toAPValue() const { return APValue(toAPSInt()); }
+  APValue toAPValue(const ASTContext &) const { return APValue(toAPSInt()); }
 
   bool isZero() const { return V.isZero(); }
   bool isPositive() const { return V.isNonNegative(); }
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index f86b787fb034e..b2581b5f7b5d0 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -39,8 +39,9 @@ namespace interp {
 using APSInt = llvm::APSInt;
 
 /// Convert a value to an APValue.
-template <typename T> bool ReturnValue(const T &V, APValue &R) {
-  R = V.toAPValue();
+template <typename T>
+bool ReturnValue(const InterpState &S, const T &V, APValue &R) {
+  R = V.toAPValue(S.getCtx());
   return true;
 }
 
@@ -286,7 +287,7 @@ bool Ret(InterpState &S, CodePtr &PC, APValue &Result) {
   } else {
     delete S.Current;
     S.Current = nullptr;
-    if (!ReturnValue<T>(Ret, Result))
+    if (!ReturnValue<T>(S, Ret, Result))
       return false;
   }
   return true;
@@ -1318,7 +1319,7 @@ bool InitGlobalTemp(InterpState &S, CodePtr OpPC, uint32_t I,
   const Pointer &Ptr = S.P.getGlobal(I);
 
   const T Value = S.Stk.peek<T>();
-  APValue APV = Value.toAPValue();
+  APValue APV = Value.toAPValue(S.getCtx());
   APValue *Cached = Temp->getOrCreateValue(true);
   *Cached = APV;
 
diff --git a/clang/lib/AST/Interp/MemberPointer.cpp b/clang/lib/AST/Interp/MemberPointer.cpp
index 96f63643e83c9..0c1b6edc5f7e1 100644
--- a/clang/lib/AST/Interp/MemberPointer.cpp
+++ b/clang/lib/AST/Interp/MemberPointer.cpp
@@ -60,13 +60,13 @@ FunctionPointer MemberPointer::toFunctionPointer(const Context &Ctx) const {
   return FunctionPointer(Ctx.getProgram().getFunction(cast<FunctionDecl>(Dcl)));
 }
 
-APValue MemberPointer::toAPValue() const {
+APValue MemberPointer::toAPValue(const ASTContext &ASTCtx) const {
   if (isZero())
     return APValue(static_cast<ValueDecl *>(nullptr), /*IsDerivedMember=*/false,
                    /*Path=*/{});
 
   if (hasBase())
-    return Base.toAPValue();
+    return Base.toAPValue(ASTCtx);
 
   return APValue(cast<ValueDecl>(getDecl()), /*IsDerivedMember=*/false,
                  /*Path=*/{});
diff --git a/clang/lib/AST/Interp/MemberPointer.h b/clang/lib/AST/Interp/MemberPointer.h
index f56dc530431e4..2b3be124db426 100644
--- a/clang/lib/AST/Interp/MemberPointer.h
+++ b/clang/lib/AST/Interp/MemberPointer.h
@@ -80,7 +80,7 @@ class MemberPointer final {
     return MemberPointer(Instance, this->Dcl, this->PtrOffset);
   }
 
-  APValue toAPValue() const;
+  APValue toAPValue(const ASTContext &) const;
 
   bool isZero() const { return Base.isZero() && !Dcl; }
   bool hasBase() const { return !Base.isZero(); }
diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp
index b22b4b1918ba5..f7bd76b260584 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/Interp/Pointer.cpp
@@ -119,7 +119,7 @@ void Pointer::operator=(Pointer &&P) {
   }
 }
 
-APValue Pointer::toAPValue() const {
+APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
   llvm::SmallVector<APValue::LValuePathEntry, 5> Path;
 
   if (isZero())
@@ -220,7 +220,7 @@ std::string Pointer::toDiagnosticString(const ASTContext &Ctx) const {
   if (isIntegralPointer())
     return (Twine("&(") + Twine(asIntPointer().Value + Offset) + ")").str();
 
-  return toAPValue().getAsString(Ctx, getType());
+  return toAPValue(Ctx).getAsString(Ctx, getType());
 }
 
 bool Pointer::isInitialized() const {
@@ -344,10 +344,12 @@ bool Pointer::hasSameArray(const Pointer &A, const Pointer &B) {
 
 std::optional<APValue> Pointer::toRValue(const Context &Ctx,
                                          QualType ResultType) const {
+  const ASTContext &ASTCtx = Ctx.getASTContext();
   assert(!ResultType.isNull());
   // Method to recursively traverse composites.
   std::function<bool(QualType, const Pointer &, APValue &)> Composite;
-  Composite = [&Composite, &Ctx](QualType Ty, const Pointer &Ptr, APValue &R) {
+  Composite = [&Composite, &Ctx, &ASTCtx](QualType Ty, const Pointer &Ptr,
+                                          APValue &R) {
     if (const auto *AT = Ty->getAs<AtomicType>())
       Ty = AT->getValueType();
 
@@ -358,7 +360,7 @@ std::optional<APValue> Pointer::toRValue(const Context &Ctx,
 
     // Primitive values.
     if (std::optional<PrimType> T = Ctx.classify(Ty)) {
-      TYPE_SWITCH(*T, R = Ptr.deref<T>().toAPValue());
+      TYPE_SWITCH(*T, R = Ptr.deref<T>().toAPValue(ASTCtx));
       return true;
     }
 
@@ -375,7 +377,7 @@ std::optional<APValue> Pointer::toRValue(const Context &Ctx,
           QualType FieldTy = F.Decl->getType();
           if (FP.isActive()) {
             if (std::optional<PrimType> T = Ctx.classify(FieldTy)) {
-              TYPE_SWITCH(*T, Value = FP.deref<T>().toAPValue());
+              TYPE_SWITCH(*T, Value = FP.deref<T>().toAPValue(ASTCtx));
             } else {
               Ok &= Composite(FieldTy, FP, Value);
             }
@@ -398,7 +400,7 @@ std::optional<APValue> Pointer::toRValue(const Context &Ctx,
           APValue &Value = R.getStructField(I);
 
           if (std::optional<PrimType> T = Ctx.classify(FieldTy)) {
-            TYPE_SWITCH(*T, Value = FP.deref<T>().toAPValue());
+            TYPE_SWITCH(*T, Value = FP.deref<T>().toAPValue(ASTCtx));
           } else {
             Ok &= Composite(FieldTy, FP, Value);
           }
@@ -436,7 +438,7 @@ std::optional<APValue> Pointer::toRValue(const Context &Ctx,
         APValue &Slot = R.getArrayInitializedElt(I);
         const Pointer &EP = Ptr.atIndex(I);
         if (std::optional<PrimType> T = Ctx.classify(ElemTy)) {
-          TYPE_SWITCH(*T, Slot = EP.deref<T>().toAPValue());
+          TYPE_SWITCH(*T, Slot = EP.deref<T>().toAPValue(ASTCtx));
         } else {
           Ok &= Composite(ElemTy, EP.narrow(), Slot);
         }
@@ -475,7 +477,7 @@ std::optional<APValue> Pointer::toRValue(const Context &Ctx,
       Values.reserve(VT->getNumElements());
       for (unsigned I = 0; I != VT->getNumElements(); ++I) {
         TYPE_SWITCH(ElemT, {
-          Values.push_back(Ptr.atIndex(I).deref<T>().toAPValue());
+          Values.push_back(Ptr.atIndex(I).deref<T>().toAPValue(ASTCtx));
         });
       }
 
@@ -493,11 +495,11 @@ std::optional<APValue> Pointer::toRValue(const Context &Ctx,
 
   // We can return these as rvalues, but we can't deref() them.
   if (isZero() || isIntegralPointer())
-    return toAPValue();
+    return toAPValue(ASTCtx);
 
   // Just load primitive types.
   if (std::optional<PrimType> T = Ctx.classify(ResultType)) {
-    TYPE_SWITCH(*T, return this->deref<T>().toAPValue());
+    TYPE_SWITCH(*T, return this->deref<T>().toAPValue(ASTCtx));
   }
 
   // Return the composite type.
diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/Interp/Pointer.h
index 972f55a553f6e..7fa6a3230a4f9 100644
--- a/clang/lib/AST/Interp/Pointer.h
+++ b/clang/lib/AST/Interp/Pointer.h
@@ -118,7 +118,7 @@ class Pointer {
   bool operator!=(const Pointer &P) const { return !(P == *this); }
 
   /// Converts the pointer to an APValue.
-  APValue toAPValue() const;
+  APValue toAPValue(const ASTContext &ASTCtx) const;
 
   /// Converts the pointer to a string usable in diagnostics.
   std::string toDiagnosticString(const ASTContext &Ctx) const;
diff --git a/clang/unittests/AST/Interp/toAPValue.cpp b/clang/unittests/AST/Interp/toAPValue.cpp
index d6879d6e0bca3..5ec607a824349 100644
--- a/clang/unittests/AST/Interp/toAPValue.cpp
+++ b/clang/unittests/AST/Interp/toAPValue.cpp
@@ -27,6 +27,7 @@ TEST(ToAPValue, Pointers) {
   auto AST = tooling::buildASTFromCodeWithArgs(
       Code, {"-fexperimental-new-constant-interpreter"});
 
+  auto &ASTCtx = AST->getASTContext();
   auto &Ctx = AST->getASTContext().getInterpContext();
   Program &Prog = Ctx.getProgram();
 
@@ -47,7 +48,7 @@ TEST(ToAPValue, Pointers) {
     const Pointer &GP = getGlobalPtr("b");
     const Pointer &P = GP.deref<Pointer>();
     ASSERT_TRUE(P.isLive());
-    APValue A = P.toAPValue();
+    APValue A = P.toAPValue(ASTCtx);
     ASSERT_TRUE(A.isLValue());
     ASSERT_TRUE(A.hasLValuePath());
     const auto &Path = A.getLValuePath();
@@ -62,7 +63,7 @@ TEST(ToAPValue, Pointers) {
     const Pointer &GP = getGlobalPtr("p");
     const Pointer &P = GP.deref<Pointer>();
     ASSERT_TRUE(P.isIntegralPointer());
-    APValue A = P.toAPValue();
+    APValue A = P.toAPValue(ASTCtx);
     ASSERT_TRUE(A.isLValue());
     ASSERT_TRUE(A.getLValueBase().isNull());
     APSInt I;
@@ -77,7 +78,7 @@ TEST(ToAPValue, Pointers) {
     const Pointer &GP = getGlobalPtr("nullp");
     const Pointer &P = GP.deref<Pointer>();
     ASSERT_TRUE(P.isIntegralPointer());
-    APValue A = P.toAPValue();
+    APValue A = P.toAPValue(ASTCtx);
     ASSERT_TRUE(A.isLValue());
     ASSERT_TRUE(A.getLValueBase().isNull());
     ASSERT_TRUE(A.isNullPointer());
@@ -96,6 +97,7 @@ TEST(ToAPValue, FunctionPointers) {
   auto AST = tooling::buildASTFromCodeWithArgs(
       Code, {"-fexperimental-new-constant-interpreter"});
 
+  auto &ASTCtx = AST->getASTContext();
   auto &Ctx = AST->getASTContext().getInterpContext();
   Program &Prog = Ctx.getProgram();
 
@@ -117,7 +119,7 @@ TEST(ToAPValue, FunctionPointers) {
     const Pointer &GP = getGlobalPtr("func");
     const FunctionPointer &FP = GP.deref<FunctionPointer>();
     ASSERT_FALSE(FP.isZero());
-    APValue A = FP.toAPValue();
+    APValue A = FP.toAPValue(ASTCtx);
     ASSERT_TRUE(A.hasValue());
     ASSERT_TRUE(A.isLValue());
     ASSERT_TRUE(A.hasLValuePath());
@@ -132,7 +134,7 @@ TEST(ToAPValue, FunctionPointers) {
     ASSERT_NE(D, nullptr);
     const Pointer &GP = getGlobalPtr("nullp");
     const auto &P = GP.deref<FunctionPointer>();
-    APValue A = P.toAPValue();
+    APValue A = P.toAPValue(ASTCtx);
     ASSERT_TRUE(A.isLValue());
     ASSERT_TRUE(A.getLValueBase().isNull());
     ASSERT_TRUE(A.isNullPointer());
@@ -151,6 +153,7 @@ TEST(ToAPValue, FunctionPointersC) {
   auto AST = tooling::buildASTFromCodeWithArgs(
       Code, {"-x", "c", "-fexperimental-new-constant-interpreter"});
 
+  auto &ASTCtx = AST->getASTContext();
   auto &Ctx = AST->getASTContext().getInterpContext();
   Program &Prog = Ctx.getProgram();
 
@@ -174,7 +177,7 @@ TEST(ToAPValue, FunctionPointersC) {
     ASSERT_TRUE(GP.isLive());
     const FunctionPointer &FP = GP.deref<FunctionPointer>();
     ASSERT_FALSE(FP.isZero());
-    APValue A = FP.toAPValue();
+    APValue A = FP.toAPValue(ASTCtx);
     ASSERT_TRUE(A.hasValue());
     ASSERT_TRUE(A.isLValue());
     const auto &Path = A.getLValuePath();
@@ -197,6 +200,7 @@ TEST(ToAPValue, MemberPointers) {
   auto AST = tooling::buildASTFromCodeWithArgs(
       Code, {"-fexperimental-new-constant-interpreter"});
 
+  auto &ASTCtx = AST->getASTContext();
   auto &Ctx = AST->getASTContext().getInterpContext();
   Program &Prog = Ctx.getProgram();
 
@@ -218,7 +222,7 @@ TEST(ToAPValue, MemberPointers) {
     const Pointer &GP = getGlobalPtr("pm");
     ASSERT_TRUE(GP.isLive());
     const MemberPointer &FP = GP.deref<MemberPointer>();
-    APValue A = FP.toAPValue();
+    APValue A = FP.toAPValue(ASTCtx);
     ASSERT_EQ(A.getMemberPointerDecl(), getDecl("m"));
     ASSERT_EQ(A.getKind(), APValue::MemberPointer);
   }
@@ -228,7 +232,7 @@ TEST(ToAPValue, MemberPointers) {
     ASSERT_TRUE(GP.isLive());
     const MemberPointer &NP = GP.deref<MemberPointer>();
     ASSERT_TRUE(NP.isZero());
-    APValue A = NP.toAPValue();
+    APValue A = NP.toAPValue(ASTCtx);
     ASSERT_EQ(A.getKind(), APValue::MemberPointer);
   }
 }

From 639560353408f82ca101e12cf4edafa9a492116b Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Sat, 20 Jul 2024 21:59:13 +0800
Subject: [PATCH 287/486] [X86][MC] Check AdSize16 for 16-bit addressing
 (#99761)

Fixes: #99735
---
 llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp |  5 ++++-
 llvm/test/MC/X86/x86-32-coverage.s                    | 10 +++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 3cd0af0c7f546..6553e1cc4a930 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -691,9 +691,12 @@ void X86MCCodeEmitter::emitMemModRMByte(
 
   unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;
 
+  bool IsAdSize16 = STI.hasFeature(X86::Is32Bit) &&
+                    (TSFlags & X86II::AdSizeMask) == X86II::AdSize16;
+
   // 16-bit addressing forms of the ModR/M byte have a different encoding for
   // the R/M field and are far more limited in which registers can be used.
-  if (X86_MC::is16BitMemOperand(MI, Op, STI)) {
+  if (IsAdSize16 || X86_MC::is16BitMemOperand(MI, Op, STI)) {
     if (BaseReg) {
       // For 32-bit addressing, the row and column values in Table 2-2 are
       // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
diff --git a/llvm/test/MC/X86/x86-32-coverage.s b/llvm/test/MC/X86/x86-32-coverage.s
index fbe2714aed263..5475946a9d216 100644
--- a/llvm/test/MC/X86/x86-32-coverage.s
+++ b/llvm/test/MC/X86/x86-32-coverage.s
@@ -10790,7 +10790,7 @@ btcl $4, (%eax)
           movdir64b 485498096, %ecx
 
 // CHECK: movdir64b 485498096, %cx
-// CHECK: # encoding: [0x67,0x66,0x0f,0x38,0xf8,0x0d,0xf0,0x1c,0xf0,0x1c]
+// CHECK: # encoding: [0x67,0x66,0x0f,0x38,0xf8,0x0e,0xf0,0x1c]
           movdir64b 485498096, %cx
 
 // CHECK: movdir64b (%edx), %eax
@@ -10877,6 +10877,10 @@ enqcmd  (%bx,%di), %di
 // CHECK: encoding: [0x67,0xf2,0x0f,0x38,0xf8,0x81,0xc0,0x1f]
 enqcmd  8128(%bx,%di), %ax
 
+// CHECK: enqcmd 485498096, %cx
+// CHECK: encoding: [0x67,0xf2,0x0f,0x38,0xf8,0x0e,0xf0,0x1c]
+enqcmd 485498096, %cx
+
 // CHECK: enqcmds (%bx,%di), %di
 // CHECK: encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x39]
 enqcmds (%bx,%di), %di
@@ -10885,6 +10889,10 @@ enqcmds (%bx,%di), %di
 // CHECK: encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x81,0xc0,0x1f]
 enqcmds 8128(%bx,%di), %ax
 
+// CHECK: enqcmds 485498096, %cx
+// CHECK: encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x0e,0xf0,0x1c]
+enqcmds 485498096, %cx
+
 // CHECK: serialize
 // CHECK: encoding: [0x0f,0x01,0xe8]
 serialize

From 615b7eeaa94d7c2d2c782fcdc21de5b62f3c168e Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Sat, 20 Jul 2024 07:29:04 -0500
Subject: [PATCH 288/486] Reapply "[LLVM][LTO] Factor out RTLib calls and allow
 them to be dropped (#98512)"

This reverts commit 740161a9b98c9920dedf1852b5f1c94d0a683af5.

I moved the `ISD` dependencies into the CodeGen portion of the handling,
it's a little awkward but it's the easiest solution I can think of for
now.
---
 lld/COFF/Driver.cpp                           |   7 +-
 lld/ELF/Driver.cpp                            |   6 +-
 lld/wasm/Driver.cpp                           |   6 +-
 .../llvm/CodeGen/GlobalISel/LegalizerHelper.h |   2 +-
 ...RuntimeLibcalls.h => RuntimeLibcallUtil.h} |  23 +-
 llvm/include/llvm/CodeGen/TargetLowering.h    |  33 +-
 llvm/include/llvm/IR/RuntimeLibcalls.h        | 105 +++++
 llvm/include/llvm/LTO/LTO.h                   |   2 +-
 llvm/lib/CodeGen/AtomicExpandPass.cpp         |   2 +-
 llvm/lib/CodeGen/DwarfEHPrepare.cpp           |   2 +-
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |   2 +-
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |   2 +-
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |   2 +-
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |   2 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   2 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      |   2 +-
 .../SelectionDAG/StatepointLowering.cpp       |   2 +-
 llvm/lib/CodeGen/TargetLoweringBase.cpp       | 416 ++----------------
 llvm/lib/IR/CMakeLists.txt                    |   1 +
 llvm/lib/IR/RuntimeLibcalls.cpp               | 346 +++++++++++++++
 llvm/lib/LTO/LTO.cpp                          |  15 +-
 llvm/lib/Object/IRSymtab.cpp                  |  20 +-
 llvm/lib/Target/AArch64/AArch64FastISel.cpp   |   2 +-
 .../Target/AArch64/AArch64ISelLowering.cpp    |   2 +-
 llvm/lib/Target/ARM/ARMFastISel.cpp           |   2 +-
 llvm/lib/Target/ARM/ARMISelLowering.cpp       |   2 +-
 llvm/lib/Target/ARM/ARMLegalizerInfo.h        |   2 +-
 llvm/lib/Target/ARM/ARMSelectionDAGInfo.h     |   2 +-
 .../Target/Hexagon/HexagonISelLowering.cpp    |   2 +-
 llvm/lib/Target/Lanai/LanaiISelLowering.cpp   |   2 +-
 .../LoongArch/LoongArchISelLowering.cpp       |   2 +-
 llvm/lib/Target/Mips/MipsISelLowering.cpp     |   2 +-
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |   2 +-
 .../WebAssemblyRuntimeLibcallSignatures.cpp   |   2 +-
 .../WebAssemblyRuntimeLibcallSignatures.h     |   2 +-
 llvm/tools/lto/lto.cpp                        |   2 +-
 36 files changed, 566 insertions(+), 462 deletions(-)
 rename llvm/include/llvm/CodeGen/{RuntimeLibcalls.h => RuntimeLibcallUtil.h} (82%)
 create mode 100644 llvm/include/llvm/IR/RuntimeLibcalls.h
 create mode 100644 llvm/lib/IR/RuntimeLibcalls.cpp

diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index cef6271e4c8f8..9e28b1c50be50 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -2428,9 +2428,12 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       // file's symbol table. If any of those library functions are defined in a
       // bitcode file in an archive member, we need to arrange to use LTO to
       // compile those archive members by adding them to the link beforehand.
-      if (!ctx.bitcodeFileInstances.empty())
-        for (auto *s : lto::LTO::getRuntimeLibcallSymbols())
+      if (!ctx.bitcodeFileInstances.empty()) {
+        llvm::Triple TT(
+            ctx.bitcodeFileInstances.front()->obj->getTargetTriple());
+        for (auto *s : lto::LTO::getRuntimeLibcallSymbols(TT))
           ctx.symtab.addLibcall(s);
+      }
 
       // Windows specific -- if __load_config_used can be resolved, resolve it.
       if (ctx.symtab.findUnderscore("_load_config_used"))
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 3a3070aea94f5..40e095a133d95 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -2884,9 +2884,11 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // to, i.e. if the symbol's definition is in bitcode. Any other required
   // libcall symbols will be added to the link after LTO when we add the LTO
   // object file to the link.
-  if (!ctx.bitcodeFiles.empty())
-    for (auto *s : lto::LTO::getRuntimeLibcallSymbols())
+  if (!ctx.bitcodeFiles.empty()) {
+    llvm::Triple TT(ctx.bitcodeFiles.front()->obj->getTargetTriple());
+    for (auto *s : lto::LTO::getRuntimeLibcallSymbols(TT))
       handleLibcall(s);
+  }
 
   // Archive members defining __wrap symbols may be extracted.
   std::vector<WrappedSymbol> wrapped = addWrappedSymbols(args);
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index b66b988005d58..8c83d17db02f5 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -1320,9 +1320,11 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   // We only need to add libcall symbols to the link before LTO if the symbol's
   // definition is in bitcode. Any other required libcall symbols will be added
   // to the link after LTO when we add the LTO object file to the link.
-  if (!ctx.bitcodeFiles.empty())
-    for (auto *s : lto::LTO::getRuntimeLibcallSymbols())
+  if (!ctx.bitcodeFiles.empty()) {
+    llvm::Triple TT(ctx.bitcodeFiles.front()->obj->getTargetTriple());
+    for (auto *s : lto::LTO::getRuntimeLibcallSymbols(TT))
       handleLibcall(s);
+  }
   if (errorCount())
     return;
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index b599b0368b9aa..b17bc9aa2a44e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -22,7 +22,7 @@
 
 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/CodeGen/RuntimeLibcalls.h b/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
similarity index 82%
rename from llvm/include/llvm/CodeGen/RuntimeLibcalls.h
rename to llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
index f3170418f3364..7a13164589392 100644
--- a/llvm/include/llvm/CodeGen/RuntimeLibcalls.h
+++ b/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
@@ -1,4 +1,4 @@
-//===-- CodeGen/RuntimeLibcalls.h - Runtime Library Calls -------*- C++ -*-===//
+//===-- CodeGen/RuntimeLibcallUtil.h - Runtime Library Calls ----*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,32 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file defines the enum representing the list of runtime library calls
-// the backend may emit during code generation, and also some helper functions.
+// This file defines some helper functions for runtime library calls.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CODEGEN_RUNTIMELIBCALLS_H
 #define LLVM_CODEGEN_RUNTIMELIBCALLS_H
 
+#include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/ValueTypes.h"
+#include "llvm/IR/RuntimeLibcalls.h"
 #include "llvm/Support/AtomicOrdering.h"
 
 namespace llvm {
 namespace RTLIB {
-/// RTLIB::Libcall enum - This enum defines all of the runtime library calls
-/// the backend can emit.  The various long double types cannot be merged,
-/// because 80-bit library functions use "xf" and 128-bit use "tf".
-///
-/// When adding PPCF128 functions here, note that their names generally need
-/// to be overridden for Darwin with the xxx$LDBL128 form.  See
-/// PPCISelLowering.cpp.
-///
-enum Libcall {
-#define HANDLE_LIBCALL(code, name) code,
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-};
 
 /// GetFPLibCall - Helper to return the right libcall for the given floating
 /// point type, or UNKNOWN_LIBCALL if there is none.
@@ -103,6 +91,9 @@ Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
 /// UNKNOW_LIBCALL if there is none.
 Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
 
+/// Initialize the default condition code on the libcalls.
+void initCmpLibcallCCs(ISD::CondCode *CmpLibcallCCs);
+
 } // namespace RTLIB
 } // namespace llvm
 
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 64c73820add64..9d9886f4920a2 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -31,7 +31,7 @@
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
@@ -45,6 +45,7 @@
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/RuntimeLibcalls.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/AtomicOrdering.h"
@@ -3414,44 +3415,44 @@ class TargetLoweringBase {
     return nullptr;
   }
 
-  //===--------------------------------------------------------------------===//
-  // Runtime Library hooks
-  //
-
   /// Rename the default libcall routine name for the specified libcall.
   void setLibcallName(RTLIB::Libcall Call, const char *Name) {
-    LibcallRoutineNames[Call] = Name;
+    Libcalls.setLibcallName(Call, Name);
   }
+
   void setLibcallName(ArrayRef<RTLIB::Libcall> Calls, const char *Name) {
-    for (auto Call : Calls)
-      setLibcallName(Call, Name);
+    Libcalls.setLibcallName(Calls, Name);
   }
 
   /// Get the libcall routine name for the specified libcall.
   const char *getLibcallName(RTLIB::Libcall Call) const {
-    return LibcallRoutineNames[Call];
+    return Libcalls.getLibcallName(Call);
   }
 
   /// Override the default CondCode to be used to test the result of the
   /// comparison libcall against zero.
+  /// FIXME: This can't be merged with 'RuntimeLibcallsInfo' because of the ISD.
   void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC) {
     CmpLibcallCCs[Call] = CC;
   }
 
+
   /// Get the CondCode that's to be used to test the result of the comparison
   /// libcall against zero.
+  /// FIXME: This can't be merged with 'RuntimeLibcallsInfo' because of the ISD.
   ISD::CondCode getCmpLibcallCC(RTLIB::Libcall Call) const {
     return CmpLibcallCCs[Call];
   }
 
+
   /// Set the CallingConv that should be used for the specified libcall.
   void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC) {
-    LibcallCallingConvs[Call] = CC;
+    Libcalls.setLibcallCallingConv(Call, CC);
   }
 
   /// Get the CallingConv that should be used for the specified libcall.
   CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const {
-    return LibcallCallingConvs[Call];
+    return Libcalls.getLibcallCallingConv(Call);
   }
 
   /// Execute target specific actions to finalize target lowering.
@@ -3630,19 +3631,13 @@ class TargetLoweringBase {
   std::map<std::pair<unsigned, MVT::SimpleValueType>, MVT::SimpleValueType>
     PromoteToType;
 
-  /// Stores the name each libcall.
-  const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1];
+  /// The list of libcalls that the target will use.
+  RTLIB::RuntimeLibcallsInfo Libcalls;
 
   /// The ISD::CondCode that should be used to test the result of each of the
   /// comparison libcall against zero.
   ISD::CondCode CmpLibcallCCs[RTLIB::UNKNOWN_LIBCALL];
 
-  /// Stores the CallingConv that should be used for each libcall.
-  CallingConv::ID LibcallCallingConvs[RTLIB::UNKNOWN_LIBCALL];
-
-  /// Set default libcall names and calling conventions.
-  void InitLibcalls(const Triple &TT);
-
   /// The bits of IndexedModeActions used to store the legalisation actions
   /// We store the data as   | ML | MS |  L |  S | each taking 4 bits.
   enum IndexedModeActionsBits {
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.h b/llvm/include/llvm/IR/RuntimeLibcalls.h
new file mode 100644
index 0000000000000..b3648f5a31e2a
--- /dev/null
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.h
@@ -0,0 +1,105 @@
+//===- RuntimeLibcalls.h - Interface for runtime libcalls -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a common interface to work with library calls into a
+// runtime that may be emitted by a given backend.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_IR_RUNTIME_LIBCALLS_H
+#define LLVM_IR_RUNTIME_LIBCALLS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/IR/CallingConv.h"
+#include "llvm/Support/AtomicOrdering.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+namespace RTLIB {
+
+/// RTLIB::Libcall enum - This enum defines all of the runtime library calls
+/// the backend can emit.  The various long double types cannot be merged,
+/// because 80-bit library functions use "xf" and 128-bit use "tf".
+///
+/// When adding PPCF128 functions here, note that their names generally need
+/// to be overridden for Darwin with the xxx$LDBL128 form.  See
+/// PPCISelLowering.cpp.
+///
+enum Libcall {
+#define HANDLE_LIBCALL(code, name) code,
+#include "llvm/IR/RuntimeLibcalls.def"
+#undef HANDLE_LIBCALL
+};
+
+/// A simple container for information about the supported runtime calls.
+struct RuntimeLibcallsInfo {
+  explicit RuntimeLibcallsInfo(const Triple &TT) {
+    initLibcalls(TT);
+  }
+
+  /// Rename the default libcall routine name for the specified libcall.
+  void setLibcallName(RTLIB::Libcall Call, const char *Name) {
+    LibcallRoutineNames[Call] = Name;
+  }
+
+  void setLibcallName(ArrayRef<RTLIB::Libcall> Calls, const char *Name) {
+    for (auto Call : Calls)
+      setLibcallName(Call, Name);
+  }
+
+  /// Get the libcall routine name for the specified libcall.
+  const char *getLibcallName(RTLIB::Libcall Call) const {
+    return LibcallRoutineNames[Call];
+  }
+
+  /// Set the CallingConv that should be used for the specified libcall.
+  void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC) {
+    LibcallCallingConvs[Call] = CC;
+  }
+
+  /// Get the CallingConv that should be used for the specified libcall.
+  CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const {
+    return LibcallCallingConvs[Call];
+  }
+
+  iterator_range<const char **> getLibcallNames() {
+    return llvm::make_range(LibcallRoutineNames,
+                            LibcallRoutineNames + RTLIB::UNKNOWN_LIBCALL);
+  }
+
+private:
+  /// Stores the name each libcall.
+  const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1];
+
+  /// Stores the CallingConv that should be used for each libcall.
+  CallingConv::ID LibcallCallingConvs[RTLIB::UNKNOWN_LIBCALL];
+
+  static bool darwinHasSinCos(const Triple &TT) {
+    assert(TT.isOSDarwin() && "should be called with darwin triple");
+    // Don't bother with 32 bit x86.
+    if (TT.getArch() == Triple::x86)
+      return false;
+    // Macos < 10.9 has no sincos_stret.
+    if (TT.isMacOSX())
+      return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit();
+    // iOS < 7.0 has no sincos_stret.
+    if (TT.isiOS())
+      return !TT.isOSVersionLT(7, 0);
+    // Any other darwin such as WatchOS/TvOS is new enough.
+    return true;
+  }
+
+  /// Set default libcall names. If a target wants to opt-out of a libcall it
+  /// should be placed here.
+  void initLibcalls(const Triple &TT);
+};
+
+} // namespace RTLIB
+} // namespace llvm
+
+#endif // LLVM_IR_RUNTIME_LIBCALLS_H
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index 63c8435838f27..30eda34cd7ba5 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -301,7 +301,7 @@ class LTO {
 
   /// Static method that returns a list of libcall symbols that can be generated
   /// by LTO but might not be visible from bitcode symbol table.
-  static ArrayRef<const char *> getRuntimeLibcallSymbols();
+  static SmallVector<const char *> getRuntimeLibcallSymbols(const Triple &TT);
 
 private:
   Config Conf;
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index cea09bcb45386..ebcf76175a36b 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -21,7 +21,7 @@
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/AtomicExpand.h"
 #include "llvm/CodeGen/AtomicExpandUtils.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
diff --git a/llvm/lib/CodeGen/DwarfEHPrepare.cpp b/llvm/lib/CodeGen/DwarfEHPrepare.cpp
index 09e7cfb12bdba..324329ce989e7 100644
--- a/llvm/lib/CodeGen/DwarfEHPrepare.cpp
+++ b/llvm/lib/CodeGen/DwarfEHPrepare.cpp
@@ -18,7 +18,7 @@
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index b10d51adb69ae..68a8a273a1b47 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -38,7 +38,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/StackProtector.h"
 #include "llvm/CodeGen/SwitchLoweringUtils.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index b8653aff690ef..640a425ffa735 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -25,7 +25,7 @@
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index b584aa5923ed6..9b2153c68ccae 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -37,7 +37,7 @@
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 9f515739ee048..7f5b46af01c62 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -25,7 +25,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index e4d08228ee903..a1376aaaf12a9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -36,7 +36,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index cf133e162c65f..02eb1305b24ae 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -44,7 +44,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
 #include "llvm/CodeGen/StackMaps.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 671ec84fb9416..4268da8670d50 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -26,7 +26,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/StackMaps.h"
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 914c67abdd617..2be7fc90a0e75 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -28,7 +28,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/StackMaps.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
@@ -98,349 +98,6 @@ static cl::opt<bool> DisableStrictNodeMutation("disable-strictnode-mutation",
        cl::desc("Don't mutate strict-float node to a legalize node"),
        cl::init(false), cl::Hidden);
 
-static bool darwinHasSinCos(const Triple &TT) {
-  assert(TT.isOSDarwin() && "should be called with darwin triple");
-  // Don't bother with 32 bit x86.
-  if (TT.getArch() == Triple::x86)
-    return false;
-  // Macos < 10.9 has no sincos_stret.
-  if (TT.isMacOSX())
-    return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit();
-  // iOS < 7.0 has no sincos_stret.
-  if (TT.isiOS())
-    return !TT.isOSVersionLT(7, 0);
-  // Any other darwin such as WatchOS/TvOS is new enough.
-  return true;
-}
-
-void TargetLoweringBase::InitLibcalls(const Triple &TT) {
-#define HANDLE_LIBCALL(code, name) setLibcallName(RTLIB::code, name);
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-  // Initialize calling conventions to their default.
-  for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC)
-    setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C);
-
-  // Use the f128 variants of math functions on x86_64
-  if (TT.getArch() == Triple::ArchType::x86_64 && TT.isGNUEnvironment()) {
-    setLibcallName(RTLIB::REM_F128, "fmodf128");
-    setLibcallName(RTLIB::FMA_F128, "fmaf128");
-    setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
-    setLibcallName(RTLIB::CBRT_F128, "cbrtf128");
-    setLibcallName(RTLIB::LOG_F128, "logf128");
-    setLibcallName(RTLIB::LOG_FINITE_F128, "__logf128_finite");
-    setLibcallName(RTLIB::LOG2_F128, "log2f128");
-    setLibcallName(RTLIB::LOG2_FINITE_F128, "__log2f128_finite");
-    setLibcallName(RTLIB::LOG10_F128, "log10f128");
-    setLibcallName(RTLIB::LOG10_FINITE_F128, "__log10f128_finite");
-    setLibcallName(RTLIB::EXP_F128, "expf128");
-    setLibcallName(RTLIB::EXP_FINITE_F128, "__expf128_finite");
-    setLibcallName(RTLIB::EXP2_F128, "exp2f128");
-    setLibcallName(RTLIB::EXP2_FINITE_F128, "__exp2f128_finite");
-    setLibcallName(RTLIB::EXP10_F128, "exp10f128");
-    setLibcallName(RTLIB::SIN_F128, "sinf128");
-    setLibcallName(RTLIB::COS_F128, "cosf128");
-    setLibcallName(RTLIB::TAN_F128, "tanf128");
-    setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
-    setLibcallName(RTLIB::ASIN_F128, "asinf128");
-    setLibcallName(RTLIB::ACOS_F128, "acosf128");
-    setLibcallName(RTLIB::ATAN_F128, "atanf128");
-    setLibcallName(RTLIB::SINH_F128, "sinhf128");
-    setLibcallName(RTLIB::COSH_F128, "coshf128");
-    setLibcallName(RTLIB::TANH_F128, "tanhf128");
-    setLibcallName(RTLIB::POW_F128, "powf128");
-    setLibcallName(RTLIB::POW_FINITE_F128, "__powf128_finite");
-    setLibcallName(RTLIB::CEIL_F128, "ceilf128");
-    setLibcallName(RTLIB::TRUNC_F128, "truncf128");
-    setLibcallName(RTLIB::RINT_F128, "rintf128");
-    setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
-    setLibcallName(RTLIB::ROUND_F128, "roundf128");
-    setLibcallName(RTLIB::ROUNDEVEN_F128, "roundevenf128");
-    setLibcallName(RTLIB::FLOOR_F128, "floorf128");
-    setLibcallName(RTLIB::COPYSIGN_F128, "copysignf128");
-    setLibcallName(RTLIB::FMIN_F128, "fminf128");
-    setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
-    setLibcallName(RTLIB::LROUND_F128, "lroundf128");
-    setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
-    setLibcallName(RTLIB::LRINT_F128, "lrintf128");
-    setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
-    setLibcallName(RTLIB::LDEXP_F128, "ldexpf128");
-    setLibcallName(RTLIB::FREXP_F128, "frexpf128");
-  }
-
-  // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
-  if (TT.isPPC()) {
-    setLibcallName(RTLIB::ADD_F128, "__addkf3");
-    setLibcallName(RTLIB::SUB_F128, "__subkf3");
-    setLibcallName(RTLIB::MUL_F128, "__mulkf3");
-    setLibcallName(RTLIB::DIV_F128, "__divkf3");
-    setLibcallName(RTLIB::POWI_F128, "__powikf2");
-    setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
-    setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
-    setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
-    setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
-    setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
-    setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
-    setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti");
-    setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
-    setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
-    setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti");
-    setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
-    setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
-    setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf");
-    setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
-    setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
-    setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf");
-    setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
-    setLibcallName(RTLIB::UNE_F128, "__nekf2");
-    setLibcallName(RTLIB::OGE_F128, "__gekf2");
-    setLibcallName(RTLIB::OLT_F128, "__ltkf2");
-    setLibcallName(RTLIB::OLE_F128, "__lekf2");
-    setLibcallName(RTLIB::OGT_F128, "__gtkf2");
-    setLibcallName(RTLIB::UO_F128, "__unordkf2");
-  }
-
-  // A few names are different on particular architectures or environments.
-  if (TT.isOSDarwin()) {
-    // For f16/f32 conversions, Darwin uses the standard naming scheme, instead
-    // of the gnueabi-style __gnu_*_ieee.
-    // FIXME: What about other targets?
-    setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
-    setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
-
-    // Some darwins have an optimized __bzero/bzero function.
-    switch (TT.getArch()) {
-    case Triple::x86:
-    case Triple::x86_64:
-      if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6))
-        setLibcallName(RTLIB::BZERO, "__bzero");
-      break;
-    case Triple::aarch64:
-    case Triple::aarch64_32:
-      setLibcallName(RTLIB::BZERO, "bzero");
-      break;
-    default:
-      break;
-    }
-
-    if (darwinHasSinCos(TT)) {
-      setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret");
-      setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret");
-      if (TT.isWatchABI()) {
-        setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
-                              CallingConv::ARM_AAPCS_VFP);
-        setLibcallCallingConv(RTLIB::SINCOS_STRET_F64,
-                              CallingConv::ARM_AAPCS_VFP);
-      }
-    }
-
-    switch (TT.getOS()) {
-    case Triple::MacOSX:
-      if (TT.isMacOSXVersionLT(10, 9)) {
-        setLibcallName(RTLIB::EXP10_F32, nullptr);
-        setLibcallName(RTLIB::EXP10_F64, nullptr);
-      } else {
-        setLibcallName(RTLIB::EXP10_F32, "__exp10f");
-        setLibcallName(RTLIB::EXP10_F64, "__exp10");
-      }
-      break;
-    case Triple::IOS:
-      if (TT.isOSVersionLT(7, 0)) {
-        setLibcallName(RTLIB::EXP10_F32, nullptr);
-        setLibcallName(RTLIB::EXP10_F64, nullptr);
-        break;
-      }
-      [[fallthrough]];
-    case Triple::TvOS:
-    case Triple::WatchOS:
-    case Triple::XROS:
-      setLibcallName(RTLIB::EXP10_F32, "__exp10f");
-      setLibcallName(RTLIB::EXP10_F64, "__exp10");
-      break;
-    default:
-      break;
-    }
-  } else {
-    setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
-    setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
-  }
-
-  if (TT.isGNUEnvironment() || TT.isOSFuchsia() ||
-      (TT.isAndroid() && !TT.isAndroidVersionLT(9))) {
-    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
-    setLibcallName(RTLIB::SINCOS_F64, "sincos");
-    setLibcallName(RTLIB::SINCOS_F80, "sincosl");
-    setLibcallName(RTLIB::SINCOS_F128, "sincosl");
-    setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl");
-  }
-
-  if (TT.isPS()) {
-    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
-    setLibcallName(RTLIB::SINCOS_F64, "sincos");
-  }
-
-  if (TT.isOSOpenBSD()) {
-    setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr);
-  }
-
-  if (TT.isOSWindows() && !TT.isOSCygMing()) {
-    setLibcallName(RTLIB::LDEXP_F32, nullptr);
-    setLibcallName(RTLIB::LDEXP_F80, nullptr);
-    setLibcallName(RTLIB::LDEXP_F128, nullptr);
-    setLibcallName(RTLIB::LDEXP_PPCF128, nullptr);
-
-    setLibcallName(RTLIB::FREXP_F32, nullptr);
-    setLibcallName(RTLIB::FREXP_F80, nullptr);
-    setLibcallName(RTLIB::FREXP_F128, nullptr);
-    setLibcallName(RTLIB::FREXP_PPCF128, nullptr);
-  }
-
-  if (TT.isAArch64()) {
-    if (TT.isOSMSVCRT()) {
-      // MSVCRT doesn't have powi; fall back to pow
-      setLibcallName(RTLIB::POWI_F32, nullptr);
-      setLibcallName(RTLIB::POWI_F64, nullptr);
-    }
-  }
-
-  // Disable most libcalls on AMDGPU.
-  if (TT.isAMDGPU()) {
-    for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I) {
-      if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
-        setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
-    }
-  }
-
-  // Disable most libcalls on NVPTX.
-  if (TT.isNVPTX()) {
-    for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I)
-      if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
-        setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
-  }
-
-  if (TT.isARM() || TT.isThumb()) {
-    // These libcalls are not available in 32-bit.
-    setLibcallName(RTLIB::SHL_I128, nullptr);
-    setLibcallName(RTLIB::SRL_I128, nullptr);
-    setLibcallName(RTLIB::SRA_I128, nullptr);
-    setLibcallName(RTLIB::MUL_I128, nullptr);
-    setLibcallName(RTLIB::MULO_I64, nullptr);
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-
-    if (TT.isOSMSVCRT()) {
-      // MSVCRT doesn't have powi; fall back to pow
-      setLibcallName(RTLIB::POWI_F32, nullptr);
-      setLibcallName(RTLIB::POWI_F64, nullptr);
-    }
-  }
-
-  if (TT.getArch() == Triple::ArchType::avr) {
-    // Division rtlib functions (not supported), use divmod functions instead
-    setLibcallName(RTLIB::SDIV_I8, nullptr);
-    setLibcallName(RTLIB::SDIV_I16, nullptr);
-    setLibcallName(RTLIB::SDIV_I32, nullptr);
-    setLibcallName(RTLIB::UDIV_I8, nullptr);
-    setLibcallName(RTLIB::UDIV_I16, nullptr);
-    setLibcallName(RTLIB::UDIV_I32, nullptr);
-
-    // Modulus rtlib functions (not supported), use divmod functions instead
-    setLibcallName(RTLIB::SREM_I8, nullptr);
-    setLibcallName(RTLIB::SREM_I16, nullptr);
-    setLibcallName(RTLIB::SREM_I32, nullptr);
-    setLibcallName(RTLIB::UREM_I8, nullptr);
-    setLibcallName(RTLIB::UREM_I16, nullptr);
-    setLibcallName(RTLIB::UREM_I32, nullptr);
-  }
-
-  if (TT.getArch() == Triple::ArchType::hexagon) {
-    // These cause problems when the shift amount is non-constant.
-    setLibcallName(RTLIB::SHL_I128, nullptr);
-    setLibcallName(RTLIB::SRL_I128, nullptr);
-    setLibcallName(RTLIB::SRA_I128, nullptr);
-  }
-
-  if (TT.isLoongArch()) {
-    if (!TT.isLoongArch64()) {
-      // Set libcalls.
-      setLibcallName(RTLIB::MUL_I128, nullptr);
-      // The MULO libcall is not part of libgcc, only compiler-rt.
-      setLibcallName(RTLIB::MULO_I64, nullptr);
-    }
-    // The MULO libcall is not part of libgcc, only compiler-rt.
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-  }
-
-  if (TT.isMIPS32()) {
-    // These libcalls are not available in 32-bit.
-    setLibcallName(RTLIB::SHL_I128, nullptr);
-    setLibcallName(RTLIB::SRL_I128, nullptr);
-    setLibcallName(RTLIB::SRA_I128, nullptr);
-    setLibcallName(RTLIB::MUL_I128, nullptr);
-    setLibcallName(RTLIB::MULO_I64, nullptr);
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-  }
-
-  if (TT.isPPC()) {
-    if (!TT.isPPC64()) {
-      // These libcalls are not available in 32-bit.
-      setLibcallName(RTLIB::SHL_I128, nullptr);
-      setLibcallName(RTLIB::SRL_I128, nullptr);
-      setLibcallName(RTLIB::SRA_I128, nullptr);
-      setLibcallName(RTLIB::MUL_I128, nullptr);
-      setLibcallName(RTLIB::MULO_I64, nullptr);
-    }
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-  }
-
-  if (TT.isRISCV32()) {
-    // These libcalls are not available in 32-bit.
-    setLibcallName(RTLIB::SHL_I128, nullptr);
-    setLibcallName(RTLIB::SRL_I128, nullptr);
-    setLibcallName(RTLIB::SRA_I128, nullptr);
-    setLibcallName(RTLIB::MUL_I128, nullptr);
-    setLibcallName(RTLIB::MULO_I64, nullptr);
-  }
-
-  if (TT.isSPARC()) {
-    if (!TT.isSPARC64()) {
-      // These libcalls are not available in 32-bit.
-      setLibcallName(RTLIB::MULO_I64, nullptr);
-      setLibcallName(RTLIB::MUL_I128, nullptr);
-      setLibcallName(RTLIB::SHL_I128, nullptr);
-      setLibcallName(RTLIB::SRL_I128, nullptr);
-      setLibcallName(RTLIB::SRA_I128, nullptr);
-    }
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-  }
-
-  if (TT.isSystemZ()) {
-    setLibcallName(RTLIB::SRL_I128, nullptr);
-    setLibcallName(RTLIB::SHL_I128, nullptr);
-    setLibcallName(RTLIB::SRA_I128, nullptr);
-  }
-
-  if (TT.isX86()) {
-    if (TT.getArch() == Triple::ArchType::x86) {
-      // These libcalls are not available in 32-bit.
-      setLibcallName(RTLIB::SHL_I128, nullptr);
-      setLibcallName(RTLIB::SRL_I128, nullptr);
-      setLibcallName(RTLIB::SRA_I128, nullptr);
-      setLibcallName(RTLIB::MUL_I128, nullptr);
-      // The MULO libcall is not part of libgcc, only compiler-rt.
-      setLibcallName(RTLIB::MULO_I64, nullptr);
-    }
-
-    // The MULO libcall is not part of libgcc, only compiler-rt.
-    setLibcallName(RTLIB::MULO_I128, nullptr);
-
-    if (TT.isOSMSVCRT()) {
-      // MSVCRT doesn't have powi; fall back to pow
-      setLibcallName(RTLIB::POWI_F32, nullptr);
-      setLibcallName(RTLIB::POWI_F64, nullptr);
-    }
-  }
-}
-
 /// GetFPLibCall - Helper to return the right libcall for the given floating
 /// point type, or UNKNOWN_LIBCALL if there is none.
 RTLIB::Libcall RTLIB::getFPLibCall(EVT VT,
@@ -917,41 +574,42 @@ RTLIB::Libcall RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
   }
 }
 
-/// InitCmpLibcallCCs - Set default comparison libcall CC.
-static void InitCmpLibcallCCs(ISD::CondCode *CCs) {
-  std::fill(CCs, CCs + RTLIB::UNKNOWN_LIBCALL, ISD::SETCC_INVALID);
-  CCs[RTLIB::OEQ_F32] = ISD::SETEQ;
-  CCs[RTLIB::OEQ_F64] = ISD::SETEQ;
-  CCs[RTLIB::OEQ_F128] = ISD::SETEQ;
-  CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
-  CCs[RTLIB::UNE_F32] = ISD::SETNE;
-  CCs[RTLIB::UNE_F64] = ISD::SETNE;
-  CCs[RTLIB::UNE_F128] = ISD::SETNE;
-  CCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
-  CCs[RTLIB::OGE_F32] = ISD::SETGE;
-  CCs[RTLIB::OGE_F64] = ISD::SETGE;
-  CCs[RTLIB::OGE_F128] = ISD::SETGE;
-  CCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
-  CCs[RTLIB::OLT_F32] = ISD::SETLT;
-  CCs[RTLIB::OLT_F64] = ISD::SETLT;
-  CCs[RTLIB::OLT_F128] = ISD::SETLT;
-  CCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
-  CCs[RTLIB::OLE_F32] = ISD::SETLE;
-  CCs[RTLIB::OLE_F64] = ISD::SETLE;
-  CCs[RTLIB::OLE_F128] = ISD::SETLE;
-  CCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
-  CCs[RTLIB::OGT_F32] = ISD::SETGT;
-  CCs[RTLIB::OGT_F64] = ISD::SETGT;
-  CCs[RTLIB::OGT_F128] = ISD::SETGT;
-  CCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
-  CCs[RTLIB::UO_F32] = ISD::SETNE;
-  CCs[RTLIB::UO_F64] = ISD::SETNE;
-  CCs[RTLIB::UO_F128] = ISD::SETNE;
-  CCs[RTLIB::UO_PPCF128] = ISD::SETNE;
+void RTLIB::initCmpLibcallCCs(ISD::CondCode *CmpLibcallCCs) {
+  std::fill(CmpLibcallCCs, CmpLibcallCCs + RTLIB::UNKNOWN_LIBCALL,
+            ISD::SETCC_INVALID);
+  CmpLibcallCCs[RTLIB::OEQ_F32] = ISD::SETEQ;
+  CmpLibcallCCs[RTLIB::OEQ_F64] = ISD::SETEQ;
+  CmpLibcallCCs[RTLIB::OEQ_F128] = ISD::SETEQ;
+  CmpLibcallCCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
+  CmpLibcallCCs[RTLIB::UNE_F32] = ISD::SETNE;
+  CmpLibcallCCs[RTLIB::UNE_F64] = ISD::SETNE;
+  CmpLibcallCCs[RTLIB::UNE_F128] = ISD::SETNE;
+  CmpLibcallCCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
+  CmpLibcallCCs[RTLIB::OGE_F32] = ISD::SETGE;
+  CmpLibcallCCs[RTLIB::OGE_F64] = ISD::SETGE;
+  CmpLibcallCCs[RTLIB::OGE_F128] = ISD::SETGE;
+  CmpLibcallCCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
+  CmpLibcallCCs[RTLIB::OLT_F32] = ISD::SETLT;
+  CmpLibcallCCs[RTLIB::OLT_F64] = ISD::SETLT;
+  CmpLibcallCCs[RTLIB::OLT_F128] = ISD::SETLT;
+  CmpLibcallCCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
+  CmpLibcallCCs[RTLIB::OLE_F32] = ISD::SETLE;
+  CmpLibcallCCs[RTLIB::OLE_F64] = ISD::SETLE;
+  CmpLibcallCCs[RTLIB::OLE_F128] = ISD::SETLE;
+  CmpLibcallCCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
+  CmpLibcallCCs[RTLIB::OGT_F32] = ISD::SETGT;
+  CmpLibcallCCs[RTLIB::OGT_F64] = ISD::SETGT;
+  CmpLibcallCCs[RTLIB::OGT_F128] = ISD::SETGT;
+  CmpLibcallCCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
+  CmpLibcallCCs[RTLIB::UO_F32] = ISD::SETNE;
+  CmpLibcallCCs[RTLIB::UO_F64] = ISD::SETNE;
+  CmpLibcallCCs[RTLIB::UO_F128] = ISD::SETNE;
+  CmpLibcallCCs[RTLIB::UO_PPCF128] = ISD::SETNE;
 }
 
 /// NOTE: The TargetMachine owns TLOF.
-TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
+TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm)
+    : TM(tm), Libcalls(TM.getTargetTriple()) {
   initActions();
 
   // Perform these initializations only once.
@@ -984,11 +642,7 @@ TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
   MinCmpXchgSizeInBits = 0;
   SupportsUnalignedAtomics = false;
 
-  std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames),
-            nullptr);
-
-  InitLibcalls(TM.getTargetTriple());
-  InitCmpLibcallCCs(CmpLibcallCCs);
+  RTLIB::initCmpLibcallCCs(CmpLibcallCCs);
 }
 
 void TargetLoweringBase::initActions() {
diff --git a/llvm/lib/IR/CMakeLists.txt b/llvm/lib/IR/CMakeLists.txt
index 6b3224e22ffb6..91e0e0cc65f36 100644
--- a/llvm/lib/IR/CMakeLists.txt
+++ b/llvm/lib/IR/CMakeLists.txt
@@ -73,6 +73,7 @@ add_llvm_component_library(LLVMCore
   VectorBuilder.cpp
   Verifier.cpp
   VFABIDemangler.cpp
+  RuntimeLibcalls.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/IR
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
new file mode 100644
index 0000000000000..8ce0caadbafd1
--- /dev/null
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -0,0 +1,346 @@
+//===- RuntimeLibcalls.cpp - Interface for runtime libcalls -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/RuntimeLibcalls.h"
+
+using namespace llvm;
+using namespace RTLIB;
+
+/// Set default libcall names. If a target wants to opt-out of a libcall it
+/// should be placed here.
+void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
+  std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames),
+            nullptr);
+
+#define HANDLE_LIBCALL(code, name) setLibcallName(RTLIB::code, name);
+#include "llvm/IR/RuntimeLibcalls.def"
+#undef HANDLE_LIBCALL
+
+  // Initialize calling conventions to their default.
+  for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC)
+    setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C);
+
+  // Use the f128 variants of math functions on x86_64
+  if (TT.getArch() == Triple::ArchType::x86_64 && TT.isGNUEnvironment()) {
+    setLibcallName(RTLIB::REM_F128, "fmodf128");
+    setLibcallName(RTLIB::FMA_F128, "fmaf128");
+    setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
+    setLibcallName(RTLIB::CBRT_F128, "cbrtf128");
+    setLibcallName(RTLIB::LOG_F128, "logf128");
+    setLibcallName(RTLIB::LOG_FINITE_F128, "__logf128_finite");
+    setLibcallName(RTLIB::LOG2_F128, "log2f128");
+    setLibcallName(RTLIB::LOG2_FINITE_F128, "__log2f128_finite");
+    setLibcallName(RTLIB::LOG10_F128, "log10f128");
+    setLibcallName(RTLIB::LOG10_FINITE_F128, "__log10f128_finite");
+    setLibcallName(RTLIB::EXP_F128, "expf128");
+    setLibcallName(RTLIB::EXP_FINITE_F128, "__expf128_finite");
+    setLibcallName(RTLIB::EXP2_F128, "exp2f128");
+    setLibcallName(RTLIB::EXP2_FINITE_F128, "__exp2f128_finite");
+    setLibcallName(RTLIB::EXP10_F128, "exp10f128");
+    setLibcallName(RTLIB::SIN_F128, "sinf128");
+    setLibcallName(RTLIB::COS_F128, "cosf128");
+    setLibcallName(RTLIB::TAN_F128, "tanf128");
+    setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
+    setLibcallName(RTLIB::ASIN_F128, "asinf128");
+    setLibcallName(RTLIB::ACOS_F128, "acosf128");
+    setLibcallName(RTLIB::ATAN_F128, "atanf128");
+    setLibcallName(RTLIB::SINH_F128, "sinhf128");
+    setLibcallName(RTLIB::COSH_F128, "coshf128");
+    setLibcallName(RTLIB::TANH_F128, "tanhf128");
+    setLibcallName(RTLIB::POW_F128, "powf128");
+    setLibcallName(RTLIB::POW_FINITE_F128, "__powf128_finite");
+    setLibcallName(RTLIB::CEIL_F128, "ceilf128");
+    setLibcallName(RTLIB::TRUNC_F128, "truncf128");
+    setLibcallName(RTLIB::RINT_F128, "rintf128");
+    setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
+    setLibcallName(RTLIB::ROUND_F128, "roundf128");
+    setLibcallName(RTLIB::ROUNDEVEN_F128, "roundevenf128");
+    setLibcallName(RTLIB::FLOOR_F128, "floorf128");
+    setLibcallName(RTLIB::COPYSIGN_F128, "copysignf128");
+    setLibcallName(RTLIB::FMIN_F128, "fminf128");
+    setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
+    setLibcallName(RTLIB::LROUND_F128, "lroundf128");
+    setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
+    setLibcallName(RTLIB::LRINT_F128, "lrintf128");
+    setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
+    setLibcallName(RTLIB::LDEXP_F128, "ldexpf128");
+    setLibcallName(RTLIB::FREXP_F128, "frexpf128");
+  }
+
+  // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
+  if (TT.isPPC()) {
+    setLibcallName(RTLIB::ADD_F128, "__addkf3");
+    setLibcallName(RTLIB::SUB_F128, "__subkf3");
+    setLibcallName(RTLIB::MUL_F128, "__mulkf3");
+    setLibcallName(RTLIB::DIV_F128, "__divkf3");
+    setLibcallName(RTLIB::POWI_F128, "__powikf2");
+    setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
+    setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
+    setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
+    setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
+    setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
+    setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
+    setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti");
+    setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
+    setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
+    setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti");
+    setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
+    setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
+    setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf");
+    setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
+    setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
+    setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf");
+    setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
+    setLibcallName(RTLIB::UNE_F128, "__nekf2");
+    setLibcallName(RTLIB::OGE_F128, "__gekf2");
+    setLibcallName(RTLIB::OLT_F128, "__ltkf2");
+    setLibcallName(RTLIB::OLE_F128, "__lekf2");
+    setLibcallName(RTLIB::OGT_F128, "__gtkf2");
+    setLibcallName(RTLIB::UO_F128, "__unordkf2");
+  }
+
+  // A few names are different on particular architectures or environments.
+  if (TT.isOSDarwin()) {
+    // For f16/f32 conversions, Darwin uses the standard naming scheme,
+    // instead of the gnueabi-style __gnu_*_ieee.
+    // FIXME: What about other targets?
+    setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
+    setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
+
+    // Some darwins have an optimized __bzero/bzero function.
+    switch (TT.getArch()) {
+    case Triple::x86:
+    case Triple::x86_64:
+      if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6))
+        setLibcallName(RTLIB::BZERO, "__bzero");
+      break;
+    case Triple::aarch64:
+    case Triple::aarch64_32:
+      setLibcallName(RTLIB::BZERO, "bzero");
+      break;
+    default:
+      break;
+    }
+
+    if (darwinHasSinCos(TT)) {
+      setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret");
+      setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret");
+      if (TT.isWatchABI()) {
+        setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
+                              CallingConv::ARM_AAPCS_VFP);
+        setLibcallCallingConv(RTLIB::SINCOS_STRET_F64,
+                              CallingConv::ARM_AAPCS_VFP);
+      }
+    }
+
+    switch (TT.getOS()) {
+    case Triple::MacOSX:
+      if (TT.isMacOSXVersionLT(10, 9)) {
+        setLibcallName(RTLIB::EXP10_F32, nullptr);
+        setLibcallName(RTLIB::EXP10_F64, nullptr);
+      } else {
+        setLibcallName(RTLIB::EXP10_F32, "__exp10f");
+        setLibcallName(RTLIB::EXP10_F64, "__exp10");
+      }
+      break;
+    case Triple::IOS:
+      if (TT.isOSVersionLT(7, 0)) {
+        setLibcallName(RTLIB::EXP10_F32, nullptr);
+        setLibcallName(RTLIB::EXP10_F64, nullptr);
+        break;
+      }
+      [[fallthrough]];
+    case Triple::TvOS:
+    case Triple::WatchOS:
+    case Triple::XROS:
+      setLibcallName(RTLIB::EXP10_F32, "__exp10f");
+      setLibcallName(RTLIB::EXP10_F64, "__exp10");
+      break;
+    default:
+      break;
+    }
+  } else {
+    setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
+    setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
+  }
+
+  if (TT.isGNUEnvironment() || TT.isOSFuchsia() ||
+      (TT.isAndroid() && !TT.isAndroidVersionLT(9))) {
+    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
+    setLibcallName(RTLIB::SINCOS_F64, "sincos");
+    setLibcallName(RTLIB::SINCOS_F80, "sincosl");
+    setLibcallName(RTLIB::SINCOS_F128, "sincosl");
+    setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl");
+  }
+
+  if (TT.isPS()) {
+    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
+    setLibcallName(RTLIB::SINCOS_F64, "sincos");
+  }
+
+  if (TT.isOSOpenBSD()) {
+    setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr);
+  }
+
+  if (TT.isOSWindows() && !TT.isOSCygMing()) {
+    setLibcallName(RTLIB::LDEXP_F32, nullptr);
+    setLibcallName(RTLIB::LDEXP_F80, nullptr);
+    setLibcallName(RTLIB::LDEXP_F128, nullptr);
+    setLibcallName(RTLIB::LDEXP_PPCF128, nullptr);
+
+    setLibcallName(RTLIB::FREXP_F32, nullptr);
+    setLibcallName(RTLIB::FREXP_F80, nullptr);
+    setLibcallName(RTLIB::FREXP_F128, nullptr);
+    setLibcallName(RTLIB::FREXP_PPCF128, nullptr);
+  }
+
+  if (TT.isAArch64()) {
+    if (TT.isOSMSVCRT()) {
+      // MSVCRT doesn't have powi; fall back to pow
+      setLibcallName(RTLIB::POWI_F32, nullptr);
+      setLibcallName(RTLIB::POWI_F64, nullptr);
+    }
+  }
+
+  // Disable most libcalls on AMDGPU.
+  if (TT.isAMDGPU()) {
+    for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I) {
+      if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
+        setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
+    }
+  }
+
+  // Disable most libcalls on NVPTX.
+  if (TT.isNVPTX()) {
+    for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I)
+      if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
+        setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
+  }
+
+  if (TT.isARM() || TT.isThumb()) {
+    // These libcalls are not available in 32-bit.
+    setLibcallName(RTLIB::SHL_I128, nullptr);
+    setLibcallName(RTLIB::SRL_I128, nullptr);
+    setLibcallName(RTLIB::SRA_I128, nullptr);
+    setLibcallName(RTLIB::MUL_I128, nullptr);
+    setLibcallName(RTLIB::MULO_I64, nullptr);
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+
+    if (TT.isOSMSVCRT()) {
+      // MSVCRT doesn't have powi; fall back to pow
+      setLibcallName(RTLIB::POWI_F32, nullptr);
+      setLibcallName(RTLIB::POWI_F64, nullptr);
+    }
+  }
+
+  if (TT.getArch() == Triple::ArchType::avr) {
+    // Division rtlib functions (not supported), use divmod functions instead
+    setLibcallName(RTLIB::SDIV_I8, nullptr);
+    setLibcallName(RTLIB::SDIV_I16, nullptr);
+    setLibcallName(RTLIB::SDIV_I32, nullptr);
+    setLibcallName(RTLIB::UDIV_I8, nullptr);
+    setLibcallName(RTLIB::UDIV_I16, nullptr);
+    setLibcallName(RTLIB::UDIV_I32, nullptr);
+
+    // Modulus rtlib functions (not supported), use divmod functions instead
+    setLibcallName(RTLIB::SREM_I8, nullptr);
+    setLibcallName(RTLIB::SREM_I16, nullptr);
+    setLibcallName(RTLIB::SREM_I32, nullptr);
+    setLibcallName(RTLIB::UREM_I8, nullptr);
+    setLibcallName(RTLIB::UREM_I16, nullptr);
+    setLibcallName(RTLIB::UREM_I32, nullptr);
+  }
+
+  if (TT.getArch() == Triple::ArchType::hexagon) {
+    // These cause problems when the shift amount is non-constant.
+    setLibcallName(RTLIB::SHL_I128, nullptr);
+    setLibcallName(RTLIB::SRL_I128, nullptr);
+    setLibcallName(RTLIB::SRA_I128, nullptr);
+  }
+
+  if (TT.isLoongArch()) {
+    if (!TT.isLoongArch64()) {
+      // Set libcalls.
+      setLibcallName(RTLIB::MUL_I128, nullptr);
+      // The MULO libcall is not part of libgcc, only compiler-rt.
+      setLibcallName(RTLIB::MULO_I64, nullptr);
+    }
+    // The MULO libcall is not part of libgcc, only compiler-rt.
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+  }
+
+  if (TT.isMIPS32()) {
+    // These libcalls are not available in 32-bit.
+    setLibcallName(RTLIB::SHL_I128, nullptr);
+    setLibcallName(RTLIB::SRL_I128, nullptr);
+    setLibcallName(RTLIB::SRA_I128, nullptr);
+    setLibcallName(RTLIB::MUL_I128, nullptr);
+    setLibcallName(RTLIB::MULO_I64, nullptr);
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+  }
+
+  if (TT.isPPC()) {
+    if (!TT.isPPC64()) {
+      // These libcalls are not available in 32-bit.
+      setLibcallName(RTLIB::SHL_I128, nullptr);
+      setLibcallName(RTLIB::SRL_I128, nullptr);
+      setLibcallName(RTLIB::SRA_I128, nullptr);
+      setLibcallName(RTLIB::MUL_I128, nullptr);
+      setLibcallName(RTLIB::MULO_I64, nullptr);
+    }
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+  }
+
+  if (TT.isRISCV32()) {
+    // These libcalls are not available in 32-bit.
+    setLibcallName(RTLIB::SHL_I128, nullptr);
+    setLibcallName(RTLIB::SRL_I128, nullptr);
+    setLibcallName(RTLIB::SRA_I128, nullptr);
+    setLibcallName(RTLIB::MUL_I128, nullptr);
+    setLibcallName(RTLIB::MULO_I64, nullptr);
+  }
+
+  if (TT.isSPARC()) {
+    if (!TT.isSPARC64()) {
+      // These libcalls are not available in 32-bit.
+      setLibcallName(RTLIB::MULO_I64, nullptr);
+      setLibcallName(RTLIB::MUL_I128, nullptr);
+      setLibcallName(RTLIB::SHL_I128, nullptr);
+      setLibcallName(RTLIB::SRL_I128, nullptr);
+      setLibcallName(RTLIB::SRA_I128, nullptr);
+    }
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+  }
+
+  if (TT.isSystemZ()) {
+    setLibcallName(RTLIB::SRL_I128, nullptr);
+    setLibcallName(RTLIB::SHL_I128, nullptr);
+    setLibcallName(RTLIB::SRA_I128, nullptr);
+  }
+
+  if (TT.isX86()) {
+    if (TT.getArch() == Triple::ArchType::x86) {
+      // These libcalls are not available in 32-bit.
+      setLibcallName(RTLIB::SHL_I128, nullptr);
+      setLibcallName(RTLIB::SRL_I128, nullptr);
+      setLibcallName(RTLIB::SRA_I128, nullptr);
+      setLibcallName(RTLIB::MUL_I128, nullptr);
+      // The MULO libcall is not part of libgcc, only compiler-rt.
+      setLibcallName(RTLIB::MULO_I64, nullptr);
+    }
+
+    // The MULO libcall is not part of libgcc, only compiler-rt.
+    setLibcallName(RTLIB::MULO_I128, nullptr);
+
+    if (TT.isOSMSVCRT()) {
+      // MSVCRT doesn't have powi; fall back to pow
+      setLibcallName(RTLIB::POWI_F32, nullptr);
+      setLibcallName(RTLIB::POWI_F64, nullptr);
+    }
+  }
+}
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 94db6660e0ba5..bb3c9f7acdb8e 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -30,6 +30,7 @@
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Metadata.h"
+#include "llvm/IR/RuntimeLibcalls.h"
 #include "llvm/LTO/LTOBackend.h"
 #include "llvm/LTO/SummaryBasedOptimizations.h"
 #include "llvm/Linker/IRMover.h"
@@ -1357,14 +1358,12 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) {
   return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 }
 
-static const char *libcallRoutineNames[] = {
-#define HANDLE_LIBCALL(code, name) name,
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-};
-
-ArrayRef<const char *> LTO::getRuntimeLibcallSymbols() {
-  return ArrayRef(libcallRoutineNames);
+SmallVector<const char *> LTO::getRuntimeLibcallSymbols(const Triple &TT) {
+  RTLIB::RuntimeLibcallsInfo Libcalls(TT);
+  SmallVector<const char *> LibcallSymbols;
+  copy_if(Libcalls.getLibcallNames(), std::back_inserter(LibcallSymbols),
+          [](const char *Name) { return Name; });
+  return LibcallSymbols;
 }
 
 /// This class defines the interface to the ThinLTO backend.
diff --git a/llvm/lib/Object/IRSymtab.cpp b/llvm/lib/Object/IRSymtab.cpp
index 7e29da4d5e948..2a2b235461a55 100644
--- a/llvm/lib/Object/IRSymtab.cpp
+++ b/llvm/lib/Object/IRSymtab.cpp
@@ -22,6 +22,7 @@
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/RuntimeLibcalls.h"
 #include "llvm/MC/StringTableBuilder.h"
 #include "llvm/Object/ModuleSymbolTable.h"
 #include "llvm/Object/SymbolicFile.h"
@@ -46,9 +47,6 @@ static cl::opt<bool> DisableBitcodeVersionUpgrade(
     cl::desc("Disable automatic bitcode upgrade for version mismatch"));
 
 static const char *PreservedSymbols[] = {
-#define HANDLE_LIBCALL(code, name) name,
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
     // There are global variables, so put it here instead of in
     // RuntimeLibcalls.def.
     // TODO: Are there similar such variables?
@@ -215,9 +213,16 @@ Expected<int> Builder::getComdatIndex(const Comdat *C, const Module *M) {
   return P.first->second;
 }
 
-static DenseSet<StringRef> buildPreservedSymbolsSet() {
-  return DenseSet<StringRef>(std::begin(PreservedSymbols),
-                             std::end(PreservedSymbols));
+static DenseSet<StringRef> buildPreservedSymbolsSet(const Triple &TT) {
+  DenseSet<StringRef> PreservedSymbolSet(std::begin(PreservedSymbols),
+                                         std::end(PreservedSymbols));
+
+  RTLIB::RuntimeLibcallsInfo Libcalls(TT);
+  for (const char *Name : Libcalls.getLibcallNames()) {
+    if (Name)
+      PreservedSymbolSet.insert(Name);
+  }
+  return PreservedSymbolSet;
 }
 
 Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
@@ -276,7 +281,8 @@ Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
   setStr(Sym.IRName, GV->getName());
 
   static const DenseSet<StringRef> PreservedSymbolsSet =
-      buildPreservedSymbolsSet();
+      buildPreservedSymbolsSet(
+          llvm::Triple(GV->getParent()->getTargetTriple()));
   bool IsPreservedSymbol = PreservedSymbolsSet.contains(GV->getName());
 
   if (Used.count(GV) || IsPreservedSymbol)
diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index e3c5a143b2889..527496f1a6374 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -35,7 +35,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/CodeGenTypes/MachineValueType.h"
 #include "llvm/IR/Argument.h"
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index dbaf31166229b..84de1ee8f8923 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -48,7 +48,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
diff --git a/llvm/lib/Target/ARM/ARMFastISel.cpp b/llvm/lib/Target/ARM/ARMFastISel.cpp
index 61d2928fe6d41..1cc21da51d1e6 100644
--- a/llvm/lib/Target/ARM/ARMFastISel.cpp
+++ b/llvm/lib/Target/ARM/ARMFastISel.cpp
@@ -40,7 +40,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 2683b5741d459..75d16a42d0205 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -53,7 +53,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
diff --git a/llvm/lib/Target/ARM/ARMLegalizerInfo.h b/llvm/lib/Target/ARM/ARMLegalizerInfo.h
index d6ce4eb1055b4..9e10638233e6d 100644
--- a/llvm/lib/Target/ARM/ARMLegalizerInfo.h
+++ b/llvm/lib/Target/ARM/ARMLegalizerInfo.h
@@ -16,7 +16,7 @@
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/IR/Instructions.h"
 
 namespace llvm {
diff --git a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
index ffa8b50493510..275b1c0f8dc01 100644
--- a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
+++ b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
@@ -14,7 +14,7 @@
 #define LLVM_LIB_TARGET_ARM_ARMSELECTIONDAGINFO_H
 
 #include "MCTargetDesc/ARMAddressingModes.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
 
 namespace llvm {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 84defa8d7b91b..7aeaebc584c64 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -27,7 +27,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
 #include "llvm/CodeGen/ValueTypes.h"
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index c7848356682d0..f6763a35cc0d5 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -27,7 +27,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index d5d047b3ee1a5..6072e5e244263 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -22,7 +22,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicsLoongArch.h"
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 8cb64a0e6f4bf..0f2047fcac640 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -40,7 +40,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 564c0f74d975d..898d1f80d0564 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -48,7 +48,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
index 53619eaa6f562..ba3ab5164af26 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
@@ -21,7 +21,7 @@
 #include "WebAssemblyRuntimeLibcallSignatures.h"
 #include "WebAssemblySubtarget.h"
 #include "WebAssemblyUtilities.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
index ff6515d5bf4e6..966b84aa4951b 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
@@ -16,7 +16,7 @@
 
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/RuntimeLibcallUtil.h"
 
 namespace llvm {
 
diff --git a/llvm/tools/lto/lto.cpp b/llvm/tools/lto/lto.cpp
index ece6dd0f10830..d68cff839604f 100644
--- a/llvm/tools/lto/lto.cpp
+++ b/llvm/tools/lto/lto.cpp
@@ -691,7 +691,7 @@ extern const char *lto_input_get_dependent_library(lto_input_t input,
 }
 
 extern const char *const *lto_runtime_lib_symbols_list(size_t *size) {
-  auto symbols = lto::LTO::getRuntimeLibcallSymbols();
+  auto symbols = lto::LTO::getRuntimeLibcallSymbols(Triple());
   *size = symbols.size();
   return symbols.data();
 }

From 3c6ea7b7168fdb4d4396bcf67ea0d69495a558b8 Mon Sep 17 00:00:00 2001
From: James Y Knight <jyknight@google.com>
Date: Sat, 20 Jul 2024 11:28:06 -0400
Subject: [PATCH 289/486] Remove 3DNow! from X86TargetParser. (#99352)

This addresses the spurious inclusion of (now unsupported) target
features '-3dnow' and '-3dnowa' when disabling mmx (when then caused log
output from `clang -mno-mmx`).

It should've been part of PR #96246, but was missed.

Also tweaks the warning in prfchwintrin.h to not recommend the
deprecated mm3dnow.h header.
---
 clang/lib/Headers/prfchwintrin.h              |  2 +-
 clang/test/CodeGen/attr-target-x86.c          |  2 +-
 compiler-rt/lib/builtins/cpu_model/x86.c      |  2 +-
 .../llvm/TargetParser/X86TargetParser.def     |  8 ++++++--
 llvm/lib/TargetParser/X86TargetParser.cpp     | 20 +++++++++----------
 5 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/clang/lib/Headers/prfchwintrin.h b/clang/lib/Headers/prfchwintrin.h
index 8a13784543c5f..eaea5f3cf8feb 100644
--- a/clang/lib/Headers/prfchwintrin.h
+++ b/clang/lib/Headers/prfchwintrin.h
@@ -8,7 +8,7 @@
  */
 
 #if !defined(__X86INTRIN_H) && !defined(_MM3DNOW_H_INCLUDED)
-#error "Never use <prfchwintrin.h> directly; include <x86intrin.h> or <mm3dnow.h> instead."
+#error "Never use <prfchwintrin.h> directly; include <x86intrin.h> instead."
 #endif
 
 #ifndef __PRFCHWINTRIN_H
diff --git a/clang/test/CodeGen/attr-target-x86.c b/clang/test/CodeGen/attr-target-x86.c
index 3c2b511157f99..b1ae6678531b9 100644
--- a/clang/test/CodeGen/attr-target-x86.c
+++ b/clang/test/CodeGen/attr-target-x86.c
@@ -64,7 +64,7 @@ void __attribute__((target("avx10.1-512"))) avx10_1_512(void) {}
 // CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-avx,-avx10.1-256,-avx10.1-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
 // CHECK: #5 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt,-aes,-avx10.1-256,-avx10.1-512,-vaes"
 // CHECK-NOT: tune-cpu
-// CHECK: #6 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-3dnow,-3dnowa,-mmx"
+// CHECK: #6 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-mmx"
 // CHECK: #7 = {{.*}}"target-cpu"="lakemont" "target-features"="+cx8,+mmx"
 // CHECK-NOT: tune-cpu
 // CHECK: #8 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87" "tune-cpu"="sandybridge"
diff --git a/compiler-rt/lib/builtins/cpu_model/x86.c b/compiler-rt/lib/builtins/cpu_model/x86.c
index ab2b685e67ef8..867ed97e57bf2 100644
--- a/compiler-rt/lib/builtins/cpu_model/x86.c
+++ b/compiler-rt/lib/builtins/cpu_model/x86.c
@@ -141,7 +141,7 @@ enum ProcessorFeatures {
   FEATURE_AVX512VP2INTERSECT,
   // FIXME: Below Features has some missings comparing to gcc, it's because gcc
   // has some not one-to-one mapped in llvm.
-  FEATURE_3DNOW,
+  // FEATURE_3DNOW,
   // FEATURE_3DNOWP,
   FEATURE_ADX = 40,
   // FEATURE_ABM,
diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def
index 0e4ad873e3639..92798cbe4b4c1 100644
--- a/llvm/include/llvm/TargetParser/X86TargetParser.def
+++ b/llvm/include/llvm/TargetParser/X86TargetParser.def
@@ -175,8 +175,12 @@ X86_FEATURE_COMPAT(AVX512BF16,      "avx512bf16",            34)
 X86_FEATURE_COMPAT(AVX512VP2INTERSECT, "avx512vp2intersect", 35)
 // Below Features has some missings comparing to gcc, it's because gcc has some
 // not one-to-one mapped in llvm.
-X86_FEATURE_COMPAT(3DNOW,           "3dnow",                  0)
-X86_FEATURE       (3DNOWA,          "3dnowa")
+
+// FIXME: dummy features were added to keep the numeric values of later features
+// stable. Since the values need to be ABI stable, they should be changed to
+// have explicitly assigned values, and then these dummy features removed.
+X86_FEATURE       (DUMMYFEATURE1,   "__dummyfeature1")
+X86_FEATURE       (DUMMYFEATURE2,   "__dummyfeature2")
 X86_FEATURE_COMPAT(ADX,             "adx",                    0)
 X86_FEATURE       (64BIT,           "64bit")
 X86_FEATURE_COMPAT(CLDEMOTE,        "cldemote",               0)
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 141ecb936b708..dcf9130052ac1 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -171,14 +171,14 @@ constexpr FeatureBitset FeaturesClearwaterforest =
 
 // Geode Processor.
 constexpr FeatureBitset FeaturesGeode =
-    FeatureX87 | FeatureCMPXCHG8B | FeatureMMX | Feature3DNOW | Feature3DNOWA;
+    FeatureX87 | FeatureCMPXCHG8B | FeatureMMX | FeaturePRFCHW;
 
 // K6 processor.
 constexpr FeatureBitset FeaturesK6 = FeatureX87 | FeatureCMPXCHG8B | FeatureMMX;
 
 // K7 and K8 architecture processors.
 constexpr FeatureBitset FeaturesAthlon =
-    FeatureX87 | FeatureCMPXCHG8B | FeatureMMX | Feature3DNOW | Feature3DNOWA;
+    FeatureX87 | FeatureCMPXCHG8B | FeatureMMX | FeaturePRFCHW;
 constexpr FeatureBitset FeaturesAthlonXP =
     FeaturesAthlon | FeatureFXSR | FeatureSSE;
 constexpr FeatureBitset FeaturesK8 =
@@ -256,8 +256,8 @@ constexpr ProcInfo Processors[] = {
   // i486-generation processors.
   { {"i486"}, CK_i486, ~0U, FeatureX87, '\0', false },
   { {"winchip-c6"}, CK_WinChipC6, ~0U, FeaturesPentiumMMX, '\0', false },
-  { {"winchip2"}, CK_WinChip2, ~0U, FeaturesPentiumMMX | Feature3DNOW, '\0', false },
-  { {"c3"}, CK_C3, ~0U, FeaturesPentiumMMX | Feature3DNOW, '\0', false },
+  { {"winchip2"}, CK_WinChip2, ~0U, FeaturesPentiumMMX | FeaturePRFCHW, '\0', false },
+  { {"c3"}, CK_C3, ~0U, FeaturesPentiumMMX | FeaturePRFCHW, '\0', false },
   // i586-generation processors, P5 microarchitecture based.
   { {"i586"}, CK_i586, ~0U, FeatureX87 | FeatureCMPXCHG8B, '\0', false },
   { {"pentium"}, CK_Pentium, ~0U, FeatureX87 | FeatureCMPXCHG8B, 'B', false },
@@ -386,8 +386,8 @@ constexpr ProcInfo Processors[] = {
   { {"lakemont"}, CK_Lakemont, ~0U, FeatureCMPXCHG8B, '\0', false },
   // K6 architecture processors.
   { {"k6"}, CK_K6, ~0U, FeaturesK6, '\0', false },
-  { {"k6-2"}, CK_K6_2, ~0U, FeaturesK6 | Feature3DNOW, '\0', false },
-  { {"k6-3"}, CK_K6_3, ~0U, FeaturesK6 | Feature3DNOW, '\0', false },
+  { {"k6-2"}, CK_K6_2, ~0U, FeaturesK6 | FeaturePRFCHW, '\0', false },
+  { {"k6-3"}, CK_K6_3, ~0U, FeaturesK6 | FeaturePRFCHW, '\0', false },
   // K7 architecture processors.
   { {"athlon"}, CK_Athlon, ~0U, FeaturesAthlon, '\0', false },
   { {"athlon-tbird"}, CK_Athlon, ~0U, FeaturesAthlon, '\0', false },
@@ -493,6 +493,7 @@ constexpr FeatureBitset ImpliedFeaturesFXSR = {};
 constexpr FeatureBitset ImpliedFeaturesINVPCID = {};
 constexpr FeatureBitset ImpliedFeaturesLWP = {};
 constexpr FeatureBitset ImpliedFeaturesLZCNT = {};
+constexpr FeatureBitset ImpliedFeaturesMMX = {};
 constexpr FeatureBitset ImpliedFeaturesMWAITX = {};
 constexpr FeatureBitset ImpliedFeaturesMOVBE = {};
 constexpr FeatureBitset ImpliedFeaturesMOVDIR64B = {};
@@ -520,6 +521,8 @@ constexpr FeatureBitset ImpliedFeaturesWBNOINVD = {};
 constexpr FeatureBitset ImpliedFeaturesVZEROUPPER = {};
 constexpr FeatureBitset ImpliedFeaturesX87 = {};
 constexpr FeatureBitset ImpliedFeaturesXSAVE = {};
+constexpr FeatureBitset ImpliedFeaturesDUMMYFEATURE1 = {};
+constexpr FeatureBitset ImpliedFeaturesDUMMYFEATURE2 = {};
 
 // Not really CPU features, but need to be in the table because clang uses
 // target features to communicate them to the backend.
@@ -534,11 +537,6 @@ constexpr FeatureBitset ImpliedFeaturesXSAVEC = FeatureXSAVE;
 constexpr FeatureBitset ImpliedFeaturesXSAVEOPT = FeatureXSAVE;
 constexpr FeatureBitset ImpliedFeaturesXSAVES = FeatureXSAVE;
 
-// MMX->3DNOW->3DNOWA chain.
-constexpr FeatureBitset ImpliedFeaturesMMX = {};
-constexpr FeatureBitset ImpliedFeatures3DNOW = FeatureMMX;
-constexpr FeatureBitset ImpliedFeatures3DNOWA = Feature3DNOW;
-
 // SSE/AVX/AVX512F chain.
 constexpr FeatureBitset ImpliedFeaturesSSE = {};
 constexpr FeatureBitset ImpliedFeaturesSSE2 = FeatureSSE;

From 5303ca1496fc5f604f37c071d37821597788e83e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sat, 20 Jul 2024 16:49:59 +0200
Subject: [PATCH 290/486] [clang][Interp] Start computing APValue offsets

For array elements, arrays roots and fields.
---
 clang/lib/AST/Interp/Pointer.cpp  | 31 +++++++++++++++++++++++++++----
 clang/test/AST/Interp/codegen.cpp | 20 ++++++++++++++++++++
 2 files changed, 47 insertions(+), 4 deletions(-)
 create mode 100644 clang/test/AST/Interp/codegen.cpp

diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp
index f7bd76b260584..229007c6d720a 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/Interp/Pointer.cpp
@@ -16,6 +16,7 @@
 #include "MemberPointer.h"
 #include "PrimType.h"
 #include "Record.h"
+#include "clang/AST/RecordLayout.h"
 
 using namespace clang;
 using namespace clang::interp;
@@ -141,25 +142,38 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
   else
     llvm_unreachable("Invalid allocation type");
 
-  if (isDummy() || isUnknownSizeArray() || Desc->asExpr())
+  if (isUnknownSizeArray() || Desc->asExpr())
     return APValue(Base, CharUnits::Zero(), Path,
                    /*IsOnePastEnd=*/isOnePastEnd(), /*IsNullPtr=*/false);
 
-  // TODO: compute the offset into the object.
   CharUnits Offset = CharUnits::Zero();
 
+  auto getFieldOffset = [&](const FieldDecl *FD) -> CharUnits {
+    const ASTRecordLayout &Layout = ASTCtx.getASTRecordLayout(FD->getParent());
+    unsigned FieldIndex = FD->getFieldIndex();
+    return ASTCtx.toCharUnitsFromBits(Layout.getFieldOffset(FieldIndex));
+  };
+
   // Build the path into the object.
   Pointer Ptr = *this;
   while (Ptr.isField() || Ptr.isArrayElement()) {
     if (Ptr.isArrayRoot()) {
       Path.push_back(APValue::LValuePathEntry(
           {Ptr.getFieldDesc()->asDecl(), /*IsVirtual=*/false}));
+
+      if (const auto *FD = dyn_cast<FieldDecl>(Ptr.getFieldDesc()->asDecl()))
+        Offset += getFieldOffset(FD);
+
       Ptr = Ptr.getBase();
     } else if (Ptr.isArrayElement()) {
+      unsigned Index;
       if (Ptr.isOnePastEnd())
-        Path.push_back(APValue::LValuePathEntry::ArrayIndex(Ptr.getArray().getNumElems()));
+        Index = Ptr.getArray().getNumElems();
       else
-        Path.push_back(APValue::LValuePathEntry::ArrayIndex(Ptr.getIndex()));
+        Index = Ptr.getIndex();
+
+      Offset += (Index * ASTCtx.getTypeSizeInChars(Ptr.getType()));
+      Path.push_back(APValue::LValuePathEntry::ArrayIndex(Index));
       Ptr = Ptr.getArray();
     } else {
       // TODO: figure out if base is virtual
@@ -170,12 +184,21 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
       if (const auto *BaseOrMember = Desc->asDecl()) {
         Path.push_back(APValue::LValuePathEntry({BaseOrMember, IsVirtual}));
         Ptr = Ptr.getBase();
+
+        if (const auto *FD = dyn_cast<FieldDecl>(BaseOrMember))
+          Offset += getFieldOffset(FD);
+
         continue;
       }
       llvm_unreachable("Invalid field type");
     }
   }
 
+  // FIXME(perf): We compute the lvalue path above, but we can't supply it
+  // for dummy pointers (that causes crashes later in CheckConstantExpression).
+  if (isDummy())
+    Path.clear();
+
   // We assemble the LValuePath starting from the innermost pointer to the
   // outermost one. SO in a.b.c, the first element in Path will refer to
   // the field 'c', while later code expects it to refer to 'a'.
diff --git a/clang/test/AST/Interp/codegen.cpp b/clang/test/AST/Interp/codegen.cpp
new file mode 100644
index 0000000000000..8a0d070d19da3
--- /dev/null
+++ b/clang/test/AST/Interp/codegen.cpp
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -o - %s -fexperimental-new-constant-interpreter | FileCheck %s
+
+
+int arr[2];
+// CHECK: @pastEnd = constant ptr getelementptr (i8, ptr @arr, i64 8)
+int &pastEnd = arr[2];
+
+// CHECK: @F = constant ptr @arr, align 8
+int &F = arr[0];
+
+struct S {
+  int a;
+  float c[3];
+};
+
+// CHECK: @s = global %struct.S zeroinitializer, align 4
+S s;
+// CHECK: @sp = constant ptr getelementptr (i8, ptr @s, i64 16), align 8
+float &sp = s.c[3];

From af0d731b12983a649e07f25037ee0e16a68ca470 Mon Sep 17 00:00:00 2001
From: PaulXiCao <paul.luckner@rwth-aachen.de>
Date: Sat, 20 Jul 2024 15:50:05 +0000
Subject: [PATCH 291/486] [libc++][math] Mathematical Special Functions:
 Hermite Polynomial (#89982)

Implementing the Hermite polynomials which are part of C++17's
mathematical special functions. The goal is to get early feedback which
will make implementing the other functions easier. Integration of
functions in chunks (e.g. `std::hermite` at first, then `std::laguerre`,
etc.) might make sense as well (also see note on boost.math below).

I started out from this abandoned merge request:
https://reviews.llvm.org/D58876 .

The C++23 standard defines them in-terms of `/* floating-point type */`
arguments. I have not looked into that.

Note, there is still an ongoing discussion on discourse whether
importing boost.math is an option.
---
 libcxx/docs/ImplementationDefinedBehavior.rst |  11 +
 libcxx/docs/Status/Cxx17.rst                  |   1 +
 libcxx/docs/Status/Cxx17Papers.csv            |   2 +-
 libcxx/docs/Status/SpecialMath.rst            |  35 ++
 libcxx/docs/Status/SpecialMathProjects.csv    |  22 ++
 libcxx/docs/index.rst                         |   1 +
 libcxx/include/CMakeLists.txt                 |   1 +
 libcxx/include/__math/special_functions.h     |  84 +++++
 libcxx/include/cmath                          |   9 +
 libcxx/include/module.modulemap               |   1 +
 libcxx/modules/std/cmath.inc                  |   2 +
 .../test/std/numerics/c.math/hermite.pass.cpp | 341 ++++++++++++++++++
 libcxx/utils/libcxx/test/modules.py           |   7 +
 13 files changed, 516 insertions(+), 1 deletion(-)
 create mode 100644 libcxx/docs/Status/SpecialMath.rst
 create mode 100644 libcxx/docs/Status/SpecialMathProjects.csv
 create mode 100644 libcxx/include/__math/special_functions.h
 create mode 100644 libcxx/test/std/numerics/c.math/hermite.pass.cpp

diff --git a/libcxx/docs/ImplementationDefinedBehavior.rst b/libcxx/docs/ImplementationDefinedBehavior.rst
index 3000bb7cfa468..f0ef733fc2c55 100644
--- a/libcxx/docs/ImplementationDefinedBehavior.rst
+++ b/libcxx/docs/ImplementationDefinedBehavior.rst
@@ -51,6 +51,17 @@ Libc++ determines that a stream is Unicode-capable terminal by:
   <http://eel.is/c++draft/print.fun#7>`_. This function is used for other
   ``std::print`` overloads that don't take an ``ostream&`` argument.
 
+`[sf.cmath] <https://wg21.link/sf.cmath>`_ Mathematical Special Functions: Large indices
+----------------------------------------------------------------------------------------
+
+Most functions within the Mathematical Special Functions section contain integral indices.
+The Standard specifies the result for larger indices as implementation-defined.
+Libc++ pursuits reasonable results by choosing the same formulas as for indices below that threshold.
+E.g.
+
+- ``std::hermite(unsigned n, T x)`` for ``n >= 128``
+
+
 Listed in the index of implementation-defined behavior
 ======================================================
 
diff --git a/libcxx/docs/Status/Cxx17.rst b/libcxx/docs/Status/Cxx17.rst
index d4426afa81638..ad4f8576f03db 100644
--- a/libcxx/docs/Status/Cxx17.rst
+++ b/libcxx/docs/Status/Cxx17.rst
@@ -41,6 +41,7 @@ Paper Status
 .. note::
 
    .. [#note-P0067] P0067: ``std::(to|from)_chars`` for integrals has been available since version 7.0. ``std::to_chars`` for ``float`` and ``double`` since version 14.0 ``std::to_chars`` for ``long double`` uses the implementation for ``double``.
+   .. [#note-P0226] P0226: Progress is tracked `here <https://https://libcxx.llvm.org/Status/SpecialMath.html>`_.
    .. [#note-P0607] P0607: The parts of P0607 that are not done are the ``<regex>`` bits.
    .. [#note-P0154] P0154: The required macros are only implemented as of clang 19.
    .. [#note-P0452] P0452: The changes to ``std::transform_inclusive_scan`` and ``std::transform_exclusive_scan`` have not yet been implemented.
diff --git a/libcxx/docs/Status/Cxx17Papers.csv b/libcxx/docs/Status/Cxx17Papers.csv
index 2e560cfe0d576..6c657d51f5c7e 100644
--- a/libcxx/docs/Status/Cxx17Papers.csv
+++ b/libcxx/docs/Status/Cxx17Papers.csv
@@ -26,7 +26,7 @@
 "`P0013R1 <https://wg21.link/p0013r1>`__","LWG","Logical type traits rev 2","Kona","|Complete|","3.8"
 "","","","","",""
 "`P0024R2 <https://wg21.link/P0024R2>`__","LWG","The Parallelism TS Should be Standardized","Jacksonville","|Partial|",""
-"`P0226R1 <https://wg21.link/P0226R1>`__","LWG","Mathematical Special Functions for C++17","Jacksonville","",""
+"`P0226R1 <https://wg21.link/P0226R1>`__","LWG","Mathematical Special Functions for C++17","Jacksonville","|In Progress| [#note-P0226]_",""
 "`P0220R1 <https://wg21.link/P0220R1>`__","LWG","Adopt Library Fundamentals V1 TS Components for C++17","Jacksonville","|Complete|","16.0"
 "`P0218R1 <https://wg21.link/P0218R1>`__","LWG","Adopt the File System TS for C++17","Jacksonville","|Complete|","7.0"
 "`P0033R1 <https://wg21.link/P0033R1>`__","LWG","Re-enabling shared_from_this","Jacksonville","|Complete|","3.9"
diff --git a/libcxx/docs/Status/SpecialMath.rst b/libcxx/docs/Status/SpecialMath.rst
new file mode 100644
index 0000000000000..fcc9f03e3ae64
--- /dev/null
+++ b/libcxx/docs/Status/SpecialMath.rst
@@ -0,0 +1,35 @@
+.. special-math-status:
+
+======================================================
+libc++ Mathematical Special Functions Status (P0226R1)
+======================================================
+
+.. include:: ../Helpers/Styles.rst
+
+.. contents::
+  :local:
+
+Overview
+========
+
+This document contains the status of the C++17 mathematical special functions implementation in libc++.
+It is used to track both the status of the sub-projects of the effort and who is assigned to these sub-projects.
+This avoids duplicating effort.
+
+If you are interested in contributing to this effort, please send a message
+to the #libcxx channel in the LLVM discord. Please *do not* start working
+on any items below that has already been assigned to someone else.
+
+Sub-projects in the Implementation Effort
+=========================================
+
+.. csv-table::
+  :file: SpecialMathProjects.csv
+  :header-rows: 1
+  :widths: auto
+
+Paper and Issue Status
+======================
+
+The underlying paper is `Mathematical Special Functions for C++17 (P0226) <https://wg21.link/P0226>`_ and is included in C++17.
+Implementation is *In Progress*.
diff --git a/libcxx/docs/Status/SpecialMathProjects.csv b/libcxx/docs/Status/SpecialMathProjects.csv
new file mode 100644
index 0000000000000..f964e79de91d3
--- /dev/null
+++ b/libcxx/docs/Status/SpecialMathProjects.csv
@@ -0,0 +1,22 @@
+Section,Description,Assignee,Complete
+| `[sf.cmath.assoc.laguerre] <https://wg21.link/sf.cmath.assoc.laguerre>`_, std::assoc_laguerre, None, |Not Started|
+| `[sf.cmath.assoc.legendre] <https://wg21.link/sf.cmath.assoc.legendre>`_, std::assoc_legendre, None, |Not Started|
+| `[sf.cmath.beta] <https://wg21.link/sf.cmath.beta>`_, std::beta, None, |Not Started|
+| `[sf.cmath.comp.ellint.1] <https://wg21.link/sf.cmath.comp.ellint.1>`_, std::comp_ellint_1, None, |Not Started|
+| `[sf.cmath.comp.ellint.2] <https://wg21.link/sf.cmath.comp.ellint.2>`_, std::comp_ellint_2, None, |Not Started|
+| `[sf.cmath.comp.ellint.3] <https://wg21.link/sf.cmath.comp.ellint.3>`_, std::comp_ellint_3, None, |Not Started|
+| `[sf.cmath.cyl.bessel.i] <https://wg21.link/sf.cmath.cyl.bessel.i>`_, std::cyl_bessel_i, None, |Not Started|
+| `[sf.cmath.cyl.bessel.j] <https://wg21.link/sf.cmath.cyl.bessel.j>`_, std::cyl_bessel_j, None, |Not Started|
+| `[sf.cmath.cyl.bessel.k] <https://wg21.link/sf.cmath.cyl.bessel.k>`_, std::cyl_bessel_k, None, |Not Started|
+| `[sf.cmath.cyl.neumann] <https://wg21.link/sf.cmath.cyl.neumann>`_, std::cyl_neumann, None, |Not Started|
+| `[sf.cmath.ellint.1] <https://wg21.link/sf.cmath.ellint.1>`_, std::ellint_1, None, |Not Started|
+| `[sf.cmath.ellint.2] <https://wg21.link/sf.cmath.ellint.2>`_, std::ellint_2, None, |Not Started|
+| `[sf.cmath.ellint.3] <https://wg21.link/sf.cmath.ellint.3>`_, std::ellint_3, None, |Not Started|
+| `[sf.cmath.expint] <https://wg21.link/sf.cmath.expint>`_, std::expint, None, |Not Started|
+| `[sf.cmath.hermite] <https://wg21.link/sf.cmath.hermite>`_, std::hermite, Paul Xi Cao, |Complete|
+| `[sf.cmath.laguerre] <https://wg21.link/sf.cmath.laguerre>`_, std::laguerre, None, |Not Started|
+| `[sf.cmath.legendre] <https://wg21.link/sf.cmath.legendre>`_, std::legendre, None, |Not Started|
+| `[sf.cmath.riemann.zeta] <https://wg21.link/sf.cmath.riemann.zeta>`_, std::riemann_zeta, None, |Not Started|
+| `[sf.cmath.sph.bessel] <https://wg21.link/sf.cmath.sph.bessel>`_, std::sph_bessel, None, |Not Started|
+| `[sf.cmath.sph.legendre] <https://wg21.link/sf.cmath.sph.legendre>`_, std::sph_legendre, None, |Not Started|
+| `[sf.cmath.sph.neumann] <https://wg21.link/sf.cmath.sph.neumann>`_, std::sph_neumann, None, |Not Started|
diff --git a/libcxx/docs/index.rst b/libcxx/docs/index.rst
index 69a9e575cfe7c..4bca3ccc8fa06 100644
--- a/libcxx/docs/index.rst
+++ b/libcxx/docs/index.rst
@@ -53,6 +53,7 @@ Getting Started with libc++
    Status/PSTL
    Status/Ranges
    Status/Spaceship
+   Status/SpecialMath
    Status/Zip
 
 
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 1a4d9c7070f14..32579272858a8 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -509,6 +509,7 @@ set(files
   __math/remainder.h
   __math/roots.h
   __math/rounding_functions.h
+  __math/special_functions.h
   __math/traits.h
   __math/trigonometric_functions.h
   __mbstate_t.h
diff --git a/libcxx/include/__math/special_functions.h b/libcxx/include/__math/special_functions.h
new file mode 100644
index 0000000000000..0b1c753a659ad
--- /dev/null
+++ b/libcxx/include/__math/special_functions.h
@@ -0,0 +1,84 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_SPECIAL_FUNCTIONS_H
+#define _LIBCPP___MATH_SPECIAL_FUNCTIONS_H
+
+#include <__config>
+#include <__math/copysign.h>
+#include <__math/traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Real>
+_LIBCPP_HIDE_FROM_ABI _Real __hermite(unsigned __n, _Real __x) {
+  // The Hermite polynomial H_n(x).
+  // The implementation is based on the recurrence formula: H_{n+1}(x) = 2x H_n(x) - 2n H_{n-1}.
+  // Press, William H., et al. Numerical recipes 3rd edition: The art of scientific computing.
+  // Cambridge university press, 2007, p. 183.
+
+  // NOLINTBEGIN(readability-identifier-naming)
+  if (__math::isnan(__x))
+    return __x;
+
+  _Real __H_0{1};
+  if (__n == 0)
+    return __H_0;
+
+  _Real __H_n_prev = __H_0;
+  _Real __H_n      = 2 * __x;
+  for (unsigned __i = 1; __i < __n; ++__i) {
+    _Real __H_n_next = 2 * (__x * __H_n - __i * __H_n_prev);
+    __H_n_prev       = __H_n;
+    __H_n            = __H_n_next;
+  }
+
+  if (!__math::isfinite(__H_n)) {
+    // Overflow occured. Two possible cases:
+    //    n is odd:  return infinity of the same sign as x.
+    //    n is even: return +Inf
+    _Real __inf = std::numeric_limits<_Real>::infinity();
+    return (__n & 1) ? __math::copysign(__inf, __x) : __inf;
+  }
+  return __H_n;
+  // NOLINTEND(readability-identifier-naming)
+}
+
+inline _LIBCPP_HIDE_FROM_ABI double hermite(unsigned __n, double __x) { return std::__hermite(__n, __x); }
+
+inline _LIBCPP_HIDE_FROM_ABI float hermite(unsigned __n, float __x) {
+  // use double internally -- float is too prone to overflow!
+  return static_cast<float>(std::hermite(__n, static_cast<double>(__x)));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double hermite(unsigned __n, long double __x) { return std::__hermite(__n, __x); }
+
+inline _LIBCPP_HIDE_FROM_ABI float hermitef(unsigned __n, float __x) { return std::hermite(__n, __x); }
+
+inline _LIBCPP_HIDE_FROM_ABI long double hermitel(unsigned __n, long double __x) { return std::hermite(__n, __x); }
+
+template <class _Integer, std::enable_if_t<std::is_integral_v<_Integer>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI double hermite(unsigned __n, _Integer __x) {
+  return std::hermite(__n, static_cast<double>(__x));
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_SPECIAL_FUNCTIONS_H
diff --git a/libcxx/include/cmath b/libcxx/include/cmath
index 7a87e35c84603..3c22604a683c3 100644
--- a/libcxx/include/cmath
+++ b/libcxx/include/cmath
@@ -204,6 +204,14 @@ floating_point fmin (arithmetic x, arithmetic y);
 float          fminf(float x, float y);
 long double    fminl(long double x, long double y);
 
+double         hermite(unsigned n, double x);                    // C++17
+float          hermite(unsigned n, float x);                     // C++17
+long double    hermite(unsigned n, long double x);               // C++17
+float          hermitef(unsigned n, float x);                    // C++17
+long double    hermitel(unsigned n, long double x);              // C++17
+template <class Integer>
+double         hermite(unsigned n, Integer x);                   // C++17
+
 floating_point hypot (arithmetic x, arithmetic y);
 float          hypotf(float x, float y);
 long double    hypotl(long double x, long double y);
@@ -315,6 +323,7 @@ constexpr long double lerp(long double a, long double b, long double t) noexcept
 #include <limits>
 #include <version>
 
+#include <__math/special_functions.h>
 #include <math.h>
 
 #ifndef _LIBCPP_MATH_H
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 3443fbc3347a3..13d0dce34d97e 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1485,6 +1485,7 @@ module std_private_math_modulo                          [system] { header "__mat
 module std_private_math_remainder                       [system] { header "__math/remainder.h" }
 module std_private_math_roots                           [system] { header "__math/roots.h" }
 module std_private_math_rounding_functions              [system] { header "__math/rounding_functions.h" }
+module std_private_math_special_functions               [system] { header "__math/special_functions.h" }
 module std_private_math_traits                          [system] { header "__math/traits.h" }
 module std_private_math_trigonometric_functions         [system] { header "__math/trigonometric_functions.h" }
 
diff --git a/libcxx/modules/std/cmath.inc b/libcxx/modules/std/cmath.inc
index a463c1e3ccf86..fe8ac773c9d1c 100644
--- a/libcxx/modules/std/cmath.inc
+++ b/libcxx/modules/std/cmath.inc
@@ -334,12 +334,14 @@ export namespace std {
   using std::expint;
   using std::expintf;
   using std::expintl;
+#endif
 
   // [sf.cmath.hermite], Hermite polynomials
   using std::hermite;
   using std::hermitef;
   using std::hermitel;
 
+#if 0
   // [sf.cmath.laguerre], Laguerre polynomials
   using std::laguerre;
   using std::laguerref;
diff --git a/libcxx/test/std/numerics/c.math/hermite.pass.cpp b/libcxx/test/std/numerics/c.math/hermite.pass.cpp
new file mode 100644
index 0000000000000..08fbd5c3283c1
--- /dev/null
+++ b/libcxx/test/std/numerics/c.math/hermite.pass.cpp
@@ -0,0 +1,341 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// <cmath>
+
+// double         hermite(unsigned n, double x);
+// float          hermite(unsigned n, float x);
+// long double    hermite(unsigned n, long double x);
+// float          hermitef(unsigned n, float x);
+// long double    hermitel(unsigned n, long double x);
+// template <class Integer>
+// double         hermite(unsigned n, Integer x);
+
+#include <array>
+#include <cassert>
+#include <cmath>
+#include <limits>
+#include <vector>
+
+#include "type_algorithms.h"
+
+inline constexpr unsigned g_max_n = 128;
+
+template <class T>
+std::array<T, 11> sample_points() {
+  return {-12.34, -7.42, -1.0, -0.5, -0.1, 0.0, 0.1, 0.5, 1.0, 5.67, 15.67};
+}
+
+template <class Real>
+class CompareFloatingValues {
+private:
+  Real abs_tol;
+  Real rel_tol;
+
+public:
+  CompareFloatingValues() {
+    abs_tol = []() -> Real {
+      if (std::is_same_v<Real, float>)
+        return 1e-5f;
+      else if (std::is_same_v<Real, double>)
+        return 1e-11;
+      else
+        return 1e-12l;
+    }();
+
+    rel_tol = abs_tol;
+  }
+
+  bool operator()(Real result, Real expected) const {
+    if (std::isinf(expected) && std::isinf(result))
+      return result == expected;
+
+    if (std::isnan(expected) || std::isnan(result))
+      return false;
+
+    Real tol = abs_tol + std::abs(expected) * rel_tol;
+    return std::abs(result - expected) < tol;
+  }
+};
+
+// Roots are taken from
+// Salzer, Herbert E., Ruth Zucker, and Ruth Capuano.
+// Table of the zeros and weight factors of the first twenty Hermite
+// polynomials. US Government Printing Office, 1952.
+template <class T>
+std::vector<T> get_roots(unsigned n) {
+  switch (n) {
+  case 0:
+    return {};
+  case 1:
+    return {T(0)};
+  case 2:
+    return {T(0.707106781186548)};
+  case 3:
+    return {T(0), T(1.224744871391589)};
+  case 4:
+    return {T(0.524647623275290), T(1.650680123885785)};
+  case 5:
+    return {T(0), T(0.958572464613819), T(2.020182870456086)};
+  case 6:
+    return {T(0.436077411927617), T(1.335849074013697), T(2.350604973674492)};
+  case 7:
+    return {T(0), T(0.816287882858965), T(1.673551628767471), T(2.651961356835233)};
+  case 8:
+    return {T(0.381186990207322), T(1.157193712446780), T(1.981656756695843), T(2.930637420257244)};
+  case 9:
+    return {T(0), T(0.723551018752838), T(1.468553289216668), T(2.266580584531843), T(3.190993201781528)};
+  case 10:
+    return {
+        T(0.342901327223705), T(1.036610829789514), T(1.756683649299882), T(2.532731674232790), T(3.436159118837738)};
+  case 11:
+    return {T(0),
+            T(0.65680956682100),
+            T(1.326557084494933),
+            T(2.025948015825755),
+            T(2.783290099781652),
+            T(3.668470846559583)};
+
+  case 12:
+    return {T(0.314240376254359),
+            T(0.947788391240164),
+            T(1.597682635152605),
+            T(2.279507080501060),
+            T(3.020637025120890),
+            T(3.889724897869782)};
+
+  case 13:
+    return {T(0),
+            T(0.605763879171060),
+            T(1.220055036590748),
+            T(1.853107651601512),
+            T(2.519735685678238),
+            T(3.246608978372410),
+            T(4.101337596178640)};
+
+  case 14:
+    return {T(0.29174551067256),
+            T(0.87871378732940),
+            T(1.47668273114114),
+            T(2.09518325850772),
+            T(2.74847072498540),
+            T(3.46265693360227),
+            T(4.30444857047363)};
+
+  case 15:
+    return {T(0.00000000000000),
+            T(0.56506958325558),
+            T(1.13611558521092),
+            T(1.71999257518649),
+            T(2.32573248617386),
+            T(2.96716692790560),
+            T(3.66995037340445),
+            T(4.49999070730939)};
+
+  case 16:
+    return {T(0.27348104613815),
+            T(0.82295144914466),
+            T(1.38025853919888),
+            T(1.95178799091625),
+            T(2.54620215784748),
+            T(3.17699916197996),
+            T(3.86944790486012),
+            T(4.68873893930582)};
+
+  case 17:
+    return {T(0),
+            T(0.5316330013427),
+            T(1.0676487257435),
+            T(1.6129243142212),
+            T(2.1735028266666),
+            T(2.7577629157039),
+            T(3.3789320911415),
+            T(4.0619466758755),
+            T(4.8713451936744)};
+
+  case 18:
+    return {T(0.2582677505191),
+            T(0.7766829192674),
+            T(1.3009208583896),
+            T(1.8355316042616),
+            T(2.3862990891667),
+            T(2.9613775055316),
+            T(3.5737690684863),
+            T(4.2481178735681),
+            T(5.0483640088745)};
+
+  case 19:
+    return {T(0),
+            T(0.5035201634239),
+            T(1.0103683871343),
+            T(1.5241706193935),
+            T(2.0492317098506),
+            T(2.5911337897945),
+            T(3.1578488183476),
+            T(3.7621873519640),
+            T(4.4285328066038),
+            T(5.2202716905375)};
+
+  case 20:
+    return {T(0.2453407083009),
+            T(0.7374737285454),
+            T(1.2340762153953),
+            T(1.7385377121166),
+            T(2.2549740020893),
+            T(2.7888060584281),
+            T(3.347854567332),
+            T(3.9447640401156),
+            T(4.6036824495507),
+            T(5.3874808900112)};
+
+  default: // polynom degree n>20 is unsupported
+    assert(false);
+    return {T(-42)};
+  }
+}
+
+template <class Real>
+void test() {
+  { // checks if NaNs are reported correctly (i.e. output == input for input == NaN)
+    using nl = std::numeric_limits<Real>;
+    for (Real NaN : {nl::quiet_NaN(), nl::signaling_NaN()})
+      for (unsigned n = 0; n < g_max_n; ++n)
+        assert(std::isnan(std::hermite(n, NaN)));
+  }
+
+  { // simple sample points for n=0..127 should not produce NaNs.
+    for (Real x : sample_points<Real>())
+      for (unsigned n = 0; n < g_max_n; ++n)
+        assert(!std::isnan(std::hermite(n, x)));
+  }
+
+  { // checks std::hermite(n, x) for n=0..5 against analytic polynoms
+    const auto h0 = [](Real) -> Real { return 1; };
+    const auto h1 = [](Real y) -> Real { return 2 * y; };
+    const auto h2 = [](Real y) -> Real { return 4 * y * y - 2; };
+    const auto h3 = [](Real y) -> Real { return y * (8 * y * y - 12); };
+    const auto h4 = [](Real y) -> Real { return (16 * std::pow(y, 4) - 48 * y * y + 12); };
+    const auto h5 = [](Real y) -> Real { return y * (32 * std::pow(y, 4) - 160 * y * y + 120); };
+
+    for (Real x : sample_points<Real>()) {
+      const CompareFloatingValues<Real> compare;
+      assert(compare(std::hermite(0, x), h0(x)));
+      assert(compare(std::hermite(1, x), h1(x)));
+      assert(compare(std::hermite(2, x), h2(x)));
+      assert(compare(std::hermite(3, x), h3(x)));
+      assert(compare(std::hermite(4, x), h4(x)));
+      assert(compare(std::hermite(5, x), h5(x)));
+    }
+  }
+
+  { // checks std::hermitef for bitwise equality with std::hermite(unsigned, float)
+    if constexpr (std::is_same_v<Real, float>)
+      for (unsigned n = 0; n < g_max_n; ++n)
+        for (float x : sample_points<float>())
+          assert(std::hermite(n, x) == std::hermitef(n, x));
+  }
+
+  { // checks std::hermitel for bitwise equality with std::hermite(unsigned, long double)
+    if constexpr (std::is_same_v<Real, long double>)
+      for (unsigned n = 0; n < g_max_n; ++n)
+        for (long double x : sample_points<long double>())
+          assert(std::hermite(n, x) == std::hermitel(n, x));
+  }
+
+  { // Checks if the characteristic recurrence relation holds:    H_{n+1}(x) = 2x H_n(x) - 2n H_{n-1}(x)
+    for (Real x : sample_points<Real>()) {
+      for (unsigned n = 1; n < g_max_n - 1; ++n) {
+        Real H_next            = std::hermite(n + 1, x);
+        Real H_next_recurrence = 2 * (x * std::hermite(n, x) - n * std::hermite(n - 1, x));
+
+        if (std::isinf(H_next))
+          break;
+        const CompareFloatingValues<Real> compare;
+        assert(compare(H_next, H_next_recurrence));
+      }
+    }
+  }
+
+  { // sanity checks: hermite polynoms need to change signs at (simple) roots. checked upto order n<=20.
+
+    // root tolerance: must be smaller than the smallest difference between adjacent roots
+    Real tol = []() -> Real {
+      if (std::is_same_v<Real, float>)
+        return 1e-5f;
+      else if (std::is_same_v<Real, double>)
+        return 1e-9;
+      else
+        return 1e-10l;
+    }();
+
+    const auto is_sign_change = [tol](unsigned n, Real x) -> bool {
+      return std::hermite(n, x - tol) * std::hermite(n, x + tol) < 0;
+    };
+
+    for (unsigned n = 0; n <= 20u; ++n) {
+      for (Real x : get_roots<Real>(n)) {
+        // the roots are symmetric: if x is a root, so is -x
+        if (x > 0)
+          assert(is_sign_change(n, -x));
+        assert(is_sign_change(n, x));
+      }
+    }
+  }
+
+  { // check input infinity is handled correctly
+    Real inf = std::numeric_limits<Real>::infinity();
+    for (unsigned n = 1; n < g_max_n; ++n) {
+      assert(std::hermite(n, +inf) == inf);
+      assert(std::hermite(n, -inf) == ((n & 1) ? -inf : inf));
+    }
+  }
+
+  { // check: if overflow occurs that it is mapped to the correct infinity
+    if constexpr (std::is_same_v<Real, double>) {
+      // Q: Why only double?
+      // A: The numeric values (e.g. overflow threshold `n`) below are different for other types.
+      static_assert(sizeof(double) == 8);
+      for (unsigned n = 0; n < g_max_n; ++n) {
+        // Q: Why n=111 and x=300?
+        // A: Both are chosen s.t. the first overlow occurs for some `n<g_max_n`.
+        if (n < 111) {
+          assert(std::isfinite(std::hermite(n, +300.0)));
+          assert(std::isfinite(std::hermite(n, -300.0)));
+        } else {
+          double inf = std::numeric_limits<double>::infinity();
+          assert(std::hermite(n, +300.0) == inf);
+          assert(std::hermite(n, -300.0) == ((n & 1) ? -inf : inf));
+        }
+      }
+    }
+  }
+}
+
+struct TestFloat {
+  template <class Real>
+  void operator()() {
+    test<Real>();
+  }
+};
+
+struct TestInt {
+  template <class Integer>
+  void operator()() {
+    // checks that std::hermite(unsigned, Integer) actually wraps std::hermite(unsigned, double)
+    for (unsigned n = 0; n < g_max_n; ++n)
+      for (Integer x : {-42, -7, -5, -1, 0, 1, 5, 7, 42})
+        assert(std::hermite(n, x) == std::hermite(n, static_cast<double>(x)));
+  }
+};
+
+int main() {
+  types::for_each(types::floating_point_types(), TestFloat());
+  types::for_each(types::type_list<short, int, long, long long>(), TestInt());
+}
diff --git a/libcxx/utils/libcxx/test/modules.py b/libcxx/utils/libcxx/test/modules.py
index aab7651c7bb03..b7758dc9a41ee 100644
--- a/libcxx/utils/libcxx/test/modules.py
+++ b/libcxx/utils/libcxx/test/modules.py
@@ -76,6 +76,13 @@
 # This declaration is in the ostream header.
 ExtraDeclarations["system_error"] = ["std::operator<<"]
 
+# TODO MODULES avoid this work-around
+# This is a work-around for the special math functions. They are declared in
+# __math/special_functions.h. Adding this as an ExtraHeader works for the std
+# module. However these functions are special; they are not available in the
+# global namespace.
+ExtraDeclarations["cmath"] = ["std::hermite", "std::hermitef", "std::hermitel"]
+
 ### ExtraHeader
 
 # Adds extra headers file to scan

From 4d8e42ea6a89c73f90941fd1b6e899912e31dd34 Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Sat, 20 Jul 2024 08:54:57 -0700
Subject: [PATCH 292/486] [NVPTX] enforce signed 32 bit type for immediate
 offset (#99682)

The NVPTX ISA states that an immOff must fit in a signed 32-bit integer
(https://docs.nvidia.com/cuda/parallel-thread-execution/#addresses-as-operands):

> `[reg+immOff]`
>
> a sum of register `reg` containing a byte address plus a constant
> integer byte offset (signed, 32-bit).
>
> `[var+immOff]`
>
> a sum of address of addressable variable `var` containing a byte
> address plus a constant integer byte offset (signed, 32-bit).

Currently we do not consider this constraint, meaning that in some edge
cases we generate invalid PTX when a value is offset by a very large
immediate.
---
 llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp   | 10 ++-
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp   |  7 +-
 llvm/test/CodeGen/NVPTX/addr-mode.ll          | 85 +++++++++++++++++++
 .../NVPTX/split-gep.ll                        | 24 ++++++
 4 files changed, 122 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/addr-mode.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 11193c11ede3b..8516bc1bef83e 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -3752,8 +3752,14 @@ bool NVPTXDAGToDAGISel::SelectADDRri_imp(
         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
       else
         Base = Addr.getOperand(0);
-      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode),
-                                         mvt);
+
+      // Offset must fit in a 32-bit signed int in PTX [register+offset] address
+      // mode
+      if (!CN->getAPIntValue().isSignedIntN(32))
+        return false;
+
+      Offset = CurDAG->getTargetConstant(CN->getSExtValue(), SDLoc(OpNode),
+                                         MVT::i32);
       return true;
     }
   }
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index bc23998455a68..44c1a2e50486c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -5167,9 +5167,12 @@ bool NVPTXTargetLowering::isLegalAddressingMode(const DataLayout &DL,
   // - [areg+immoff]
   // - [immAddr]
 
-  if (AM.BaseGV) {
+  // immoff must fit in a signed 32-bit int
+  if (!APInt(64, AM.BaseOffs).isSignedIntN(32))
+    return false;
+
+  if (AM.BaseGV)
     return !AM.BaseOffs && !AM.HasBaseReg && !AM.Scale;
-  }
 
   switch (AM.Scale) {
   case 0: // "r", "r+i" or "i" is allowed
diff --git a/llvm/test/CodeGen/NVPTX/addr-mode.ll b/llvm/test/CodeGen/NVPTX/addr-mode.ll
new file mode 100644
index 0000000000000..a6a085c0e2e33
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/addr-mode.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 | FileCheck %s
+
+target triple = "nvptx64-nvidia-cuda"
+
+define i32 @test_addr_mode_i64(ptr %x) {
+; CHECK-LABEL: test_addr_mode_i64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_addr_mode_i64_param_0];
+; CHECK-NEXT:    ld.u32 %r1, [%rd1+-4];
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; CHECK-NEXT:    ret;
+  %addr = getelementptr i32, ptr %x, i64 -1
+  %res = load i32, ptr %addr
+  ret i32 %res
+}
+
+define i32 @test_addr_mode_i32(ptr %x) {
+; CHECK-LABEL: test_addr_mode_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_addr_mode_i32_param_0];
+; CHECK-NEXT:    ld.u32 %r1, [%rd1+-4];
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; CHECK-NEXT:    ret;
+  %addr = getelementptr i32, ptr %x, i32 -1
+  %res = load i32, ptr %addr
+  ret i32 %res
+}
+
+define i32 @test_addr_mode_i16(ptr %x) {
+; CHECK-LABEL: test_addr_mode_i16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_addr_mode_i16_param_0];
+; CHECK-NEXT:    ld.u32 %r1, [%rd1+-4];
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; CHECK-NEXT:    ret;
+  %addr = getelementptr i32, ptr %x, i16 -1
+  %res = load i32, ptr %addr
+  ret i32 %res
+}
+
+define i32 @test_addr_mode_i8(ptr %x) {
+; CHECK-LABEL: test_addr_mode_i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_addr_mode_i8_param_0];
+; CHECK-NEXT:    ld.u32 %r1, [%rd1+-4];
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; CHECK-NEXT:    ret;
+  %addr = getelementptr i32, ptr %x, i8 -1
+  %res = load i32, ptr %addr
+  ret i32 %res
+}
+
+define i32 @test_addr_mode_i64_large(ptr %x) {
+; CHECK-LABEL: test_addr_mode_i64_large(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_addr_mode_i64_large_param_0];
+; CHECK-NEXT:    add.s64 %rd2, %rd1, 17179869172;
+; CHECK-NEXT:    ld.u32 %r1, [%rd2];
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; CHECK-NEXT:    ret;
+  %addr = getelementptr i32, ptr %x, i64 4294967293
+  %res = load i32, ptr %addr
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll
index c915b9a5e59ac..16e9e5eeb6143 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll
@@ -397,3 +397,27 @@ entry:
   %ptr2 = getelementptr inbounds %struct0, ptr %ptr, i65 1, i32 3, i64 %idx, i32 1
   ret ptr %ptr2
 }
+
+; Do not extract large constant offset that cannot be folded in to PTX
+; addressing mode
+define void @large_offset(ptr %out, i32 %in) {
+; CHECK-LABEL: define void @large_offset(
+; CHECK-SAME: ptr [[OUT:%.*]], i32 [[IN:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 536870912
+; CHECK-NEXT:    [[IDX:%.*]] = zext nneg i32 [[ADD]] to i64
+; CHECK-NEXT:    [[GETELEM:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[IDX]]
+; CHECK-NEXT:    store i32 [[IN]], ptr [[GETELEM]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = tail call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+  %add = add nuw nsw i32 %0, 536870912
+  %idx = zext nneg i32 %add to i64
+  %getElem = getelementptr inbounds i32, ptr %out, i64 %idx
+  store i32 %in, ptr %getElem, align 4
+  ret void
+}
+
+declare i32 @llvm.nvvm.read.ptx.sreg.tid.x()

From afbfb16d294af2fd4f6fb3c37bb07b274adef8bc Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 20 Jul 2024 19:24:41 +0200
Subject: [PATCH 293/486] [libc++][TZDB] Implements zoned_time formatters.
 (#98347)

Implements parts of:
- P0355 Extending to chrono Calendars and Time Zones
- P1361 Integration of chrono with text formatting
- P2372 Fixing locale handling in chrono formatters
---
 libcxx/docs/Status/FormatPaper.csv            |   4 +-
 libcxx/include/__chrono/convert_to_tm.h       |  10 +-
 libcxx/include/__chrono/formatter.h           |  69 +-
 libcxx/include/__chrono/ostream.h             |   9 +
 libcxx/include/chrono                         |   7 +
 .../time.syn/formatter.zoned_time.pass.cpp    | 974 ++++++++++++++++++
 .../test_offset_time_zone.h                   |  19 +
 .../ostream.pass.cpp                          | 351 +++++++
 8 files changed, 1434 insertions(+), 9 deletions(-)
 create mode 100644 libcxx/test/std/time/time.syn/formatter.zoned_time.pass.cpp
 create mode 100644 libcxx/test/std/time/time.zone/time.zone.zonedtime/time.zone.zonedtime.nonmembers/ostream.pass.cpp

diff --git a/libcxx/docs/Status/FormatPaper.csv b/libcxx/docs/Status/FormatPaper.csv
index f29f1f7ca7487..fb96b1fff30ad 100644
--- a/libcxx/docs/Status/FormatPaper.csv
+++ b/libcxx/docs/Status/FormatPaper.csv
@@ -7,7 +7,7 @@ Section,Description,Dependencies,Assignee,Status,First released version
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::gps_time<Duration>``",A ``<chrono>`` implementation,Mark de Wever,,,
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::file_time<Duration>``",,Mark de Wever,|Complete|,17.0
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::local_time<Duration>``",,Mark de Wever,|Complete|,17.0
-`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::local-time-format-t<Duration>``",A ``<chrono>`` implementation,Mark de Wever,,,
+`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::local-time-format-t<Duration>``",,,|Nothing To Do|,
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::day``",,Mark de Wever,|Complete|,16.0
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::month``",,Mark de Wever,|Complete|,16.0
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::year``",,Mark de Wever,|Complete|,16.0
@@ -26,7 +26,7 @@ Section,Description,Dependencies,Assignee,Status,First released version
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::hh_mm_ss<duration<Rep, Period>>``",,Mark de Wever,|Complete|,17.0
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::sys_info``",,Mark de Wever,|Complete|,19.0
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::local_info``",,Mark de Wever,|Complete|,19.0
-`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::zoned_time<Duration, TimeZonePtr>``",A ``<chrono>`` implementation,Mark de Wever,,
+`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::zoned_time<Duration, TimeZonePtr>``",,Mark de Wever,|Complete|,19.0
 
 "`P2693R1 <https://wg21.link/P2693R1>`__","Formatting ``thread::id`` and ``stacktrace``"
 `[thread.thread.id] <https://wg21.link/thread.thread.id>`_,"Formatting ``thread::id``",,Mark de Wever,|Complete|,17.0
diff --git a/libcxx/include/__chrono/convert_to_tm.h b/libcxx/include/__chrono/convert_to_tm.h
index 881a4970822d8..3a51019b80784 100644
--- a/libcxx/include/__chrono/convert_to_tm.h
+++ b/libcxx/include/__chrono/convert_to_tm.h
@@ -29,11 +29,13 @@
 #include <__chrono/year_month.h>
 #include <__chrono/year_month_day.h>
 #include <__chrono/year_month_weekday.h>
+#include <__chrono/zoned_time.h>
 #include <__concepts/same_as.h>
 #include <__config>
 #include <__format/format_error.h>
 #include <__memory/addressof.h>
 #include <__type_traits/is_convertible.h>
+#include <__type_traits/is_specialization.h>
 #include <cstdint>
 #include <ctime>
 #include <limits>
@@ -178,7 +180,13 @@ _LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(const _ChronoT& __value) {
     // Has no time information.
   } else if constexpr (same_as<_ChronoT, chrono::local_info>) {
     // Has no time information.
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  } else if constexpr (__is_specialization_v<_ChronoT, chrono::zoned_time>) {
+    return std::__convert_to_tm<_Tm>(
+        chrono::sys_time<typename _ChronoT::duration>{__value.get_local_time().time_since_epoch()});
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
   } else
     static_assert(sizeof(_ChronoT) == 0, "Add the missing type specialization");
 
diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h
index 9a77316385abd..449c415e95760 100644
--- a/libcxx/include/__chrono/formatter.h
+++ b/libcxx/include/__chrono/formatter.h
@@ -33,6 +33,7 @@
 #include <__chrono/year_month.h>
 #include <__chrono/year_month_day.h>
 #include <__chrono/year_month_weekday.h>
+#include <__chrono/zoned_time.h>
 #include <__concepts/arithmetic.h>
 #include <__concepts/same_as.h>
 #include <__config>
@@ -44,6 +45,7 @@
 #include <__format/parser_std_format_spec.h>
 #include <__format/write_escaped.h>
 #include <__memory/addressof.h>
+#include <__type_traits/is_specialization.h>
 #include <cmath>
 #include <ctime>
 #include <limits>
@@ -137,10 +139,24 @@ __format_sub_seconds(basic_stringstream<_CharT>& __sstr, const chrono::hh_mm_ss<
                    __value.fractional_width);
 }
 
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB) && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) &&                     \
+      !defined(_LIBCPP_HAS_NO_FILESYSTEM) && !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+template <class _CharT, class _Duration, class _TimeZonePtr>
+_LIBCPP_HIDE_FROM_ABI void
+__format_sub_seconds(basic_stringstream<_CharT>& __sstr, const chrono::zoned_time<_Duration, _TimeZonePtr>& __value) {
+  __formatter::__format_sub_seconds(__sstr, __value.get_local_time().time_since_epoch());
+}
+#  endif
+
 template <class _Tp>
 consteval bool __use_fraction() {
   if constexpr (__is_time_point<_Tp>)
     return chrono::hh_mm_ss<typename _Tp::duration>::fractional_width;
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB) && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) &&                     \
+      !defined(_LIBCPP_HAS_NO_FILESYSTEM) && !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return chrono::hh_mm_ss<typename _Tp::duration>::fractional_width;
+#  endif
   else if constexpr (chrono::__is_duration<_Tp>::value)
     return chrono::hh_mm_ss<_Tp>::fractional_width;
   else if constexpr (__is_hh_mm_ss<_Tp>)
@@ -212,8 +228,13 @@ _LIBCPP_HIDE_FROM_ABI __time_zone __convert_to_time_zone([[maybe_unused]] const
 #  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
   if constexpr (same_as<_Tp, chrono::sys_info>)
     return {__value.abbrev, __value.offset};
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return __formatter::__convert_to_time_zone(__value.get_info());
+#    endif
   else
-#  endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
     return {"UTC", chrono::seconds{0}};
 }
 
@@ -426,7 +447,12 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __weekday_ok(const _Tp& __value) {
     return true;
   else if constexpr (same_as<_Tp, chrono::local_info>)
     return true;
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return true;
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
   else
     static_assert(sizeof(_Tp) == 0, "Add the missing type specialization");
 }
@@ -472,7 +498,12 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __weekday_name_ok(const _Tp& __value) {
     return true;
   else if constexpr (same_as<_Tp, chrono::local_info>)
     return true;
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return true;
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
   else
     static_assert(sizeof(_Tp) == 0, "Add the missing type specialization");
 }
@@ -518,7 +549,12 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __date_ok(const _Tp& __value) {
     return true;
   else if constexpr (same_as<_Tp, chrono::local_info>)
     return true;
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return true;
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
   else
     static_assert(sizeof(_Tp) == 0, "Add the missing type specialization");
 }
@@ -564,7 +600,12 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __month_name_ok(const _Tp& __value) {
     return true;
   else if constexpr (same_as<_Tp, chrono::local_info>)
     return true;
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return true;
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
   else
     static_assert(sizeof(_Tp) == 0, "Add the missing type specialization");
 }
@@ -924,7 +965,23 @@ struct formatter<chrono::local_info, _CharT> : public __formatter_chrono<_CharT>
     return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags{});
   }
 };
-#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+// Note due to how libc++'s formatters are implemented there is no need to add
+// the exposition only local-time-format-t abstraction.
+template <class _Duration, class _TimeZonePtr, __fmt_char_type _CharT>
+struct formatter<chrono::zoned_time<_Duration, _TimeZonePtr>, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__clock);
+  }
+};
+#    endif // !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&
+           // !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  endif   // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
 #endif // if _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__chrono/ostream.h b/libcxx/include/__chrono/ostream.h
index bb0341bc3ec63..e6c43254eea15 100644
--- a/libcxx/include/__chrono/ostream.h
+++ b/libcxx/include/__chrono/ostream.h
@@ -27,6 +27,7 @@
 #include <__chrono/year_month.h>
 #include <__chrono/year_month_day.h>
 #include <__chrono/year_month_weekday.h>
+#include <__chrono/zoned_time.h>
 #include <__concepts/same_as.h>
 #include <__config>
 #include <__format/format_functions.h>
@@ -302,6 +303,14 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const local_info& __info) {
              _LIBCPP_STATICALLY_WIDEN(_CharT, "{}: {{{}, {}}}"), __result(), __info.first, __info.second);
 }
 
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+template <class _CharT, class _Traits, class _Duration, class _TimeZonePtr>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const zoned_time<_Duration, _TimeZonePtr>& __tp) {
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%F %T %Z}"), __tp);
+}
+#    endif
 #  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
 } // namespace chrono
diff --git a/libcxx/include/chrono b/libcxx/include/chrono
index 7f25c76fda542..990c415ec2e97 100644
--- a/libcxx/include/chrono
+++ b/libcxx/include/chrono
@@ -799,6 +799,11 @@ template<class Duration1, class Duration2, class TimeZonePtr>
   bool operator==(const zoned_time<Duration1, TimeZonePtr>& x,
                   const zoned_time<Duration2, TimeZonePtr>& y);
 
+template<class charT, class traits, class Duration, class TimeZonePtr>           // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const zoned_time<Duration, TimeZonePtr>& t);
+
 // [time.zone.leap], leap second support
 class leap_second {                                                              // C++20
 public:
@@ -881,6 +886,8 @@ namespace std {
     struct formatter<chrono::hh_mm_ss<duration<Rep, Period>>, charT>;             // C++20
   template<class charT> struct formatter<chrono::sys_info, charT>;                // C++20
   template<class charT> struct formatter<chrono::local_info, charT>;              // C++20
+  template<class Duration, class TimeZonePtr, class charT>                        // C++20
+    struct formatter<chrono::zoned_time<Duration, TimeZonePtr>, charT>;
 } // namespace std
 
 namespace chrono {
diff --git a/libcxx/test/std/time/time.syn/formatter.zoned_time.pass.cpp b/libcxx/test/std/time/time.syn/formatter.zoned_time.pass.cpp
new file mode 100644
index 0000000000000..1e366ac72fa9f
--- /dev/null
+++ b/libcxx/test/std/time/time.syn/formatter.zoned_time.pass.cpp
@@ -0,0 +1,974 @@
+//===----------------------------------------------------------------------===//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// TODO FMT This test should not require std::to_chars(floating-point)
+// XFAIL: availability-fp_to_chars-missing
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+
+// REQUIRES: locale.fr_FR.UTF-8
+// REQUIRES: locale.ja_JP.UTF-8
+
+// <chrono>
+//
+// template<class Duration, class TimeZonePtr, class charT>
+// struct formatter<chrono::zoned_time<Duration, TimeZonePtr>, charT>
+
+#include <chrono>
+#include <format>
+
+#include <cassert>
+#include <concepts>
+#include <locale>
+#include <iostream>
+#include <type_traits>
+
+#include "formatter_tests.h"
+#include "make_string.h"
+#include "platform_support.h" // locale name macros
+#include "test_macros.h"
+
+template <class CharT>
+static void test_no_chrono_specs() {
+  using namespace std::literals::chrono_literals;
+
+  check(SV("1970-01-01 01:00:00.000000042 +01"),
+        SV("{}"),
+        std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::nanoseconds>{42ns}));
+  check(SV("1970-01-01 01:00:00.000042 +01"),
+        SV("{}"),
+        std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::microseconds>{42us}));
+  check(SV("1970-01-01 01:00:00.042 +01"),
+        SV("{}"),
+        std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::milliseconds>{42ms}));
+  check(SV("1970-01-01 01:00:42 +01"),
+        SV("{}"),
+        std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::seconds>{42s}));
+  check(SV("1970-02-12 01:00:00 +01"),
+        SV("{}"),
+        std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::days>{std::chrono::days{42}}));
+  check(SV("1970-10-22 01:00:00 +01"),
+        SV("{}"),
+        std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::weeks>{std::chrono::weeks{42}}));
+}
+
+template <class CharT>
+static void test_valid_values_year() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt =
+      SV("{:%%C='%C'%t%%EC='%EC'%t%%y='%y'%t%%Oy='%Oy'%t%%Ey='%Ey'%t%%Y='%Y'%t%%EY='%EY'%n}");
+  constexpr std::basic_string_view<CharT> lfmt =
+      SV("{:L%%C='%C'%t%%EC='%EC'%t%%y='%y'%t%%Oy='%Oy'%t%%Ey='%Ey'%t%%Y='%Y'%t%%EY='%EY'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%C='19'\t%EC='19'\t%y='70'\t%Oy='70'\t%Ey='70'\t%Y='1970'\t%EY='1970'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%C='19'\t%EC='19'\t%y='70'\t%Oy='70'\t%Ey='70'\t%Y='1970'\t%EY='1970'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+
+  check(loc,
+        SV("%C='19'\t%EC='19'\t%y='70'\t%Oy='70'\t%Ey='70'\t%Y='1970'\t%EY='1970'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#else  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)|| defined(__FreeBSD__)
+
+  check(loc,
+        SV("%C='19'\t%EC='昭和'\t%y='70'\t%Oy='七�'\t%Ey='45'\t%Y='1970'\t%EY='昭和45年'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%C='20'\t%EC='平�'\t%y='09'\t%Oy='�'\t%Ey='21'\t%Y='2009'\t%EY='平�21年'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)|| defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_month() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%b='%b'%t%%h='%h'%t%%B='%B'%t%%m='%m'%t%%Om='%Om'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%b='%b'%t%%h='%h'%t%%B='%B'%t%%m='%m'%t%%Om='%Om'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%b='Jan'\t%h='Jan'\t%B='January'\t%m='01'\t%Om='01'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%b='May'\t%h='May'\t%B='May'\t%m='05'\t%Om='05'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__)
+  check(SV("%b='jan'\t%h='jan'\t%B='janvier'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+#else
+  check(SV("%b='janv.'\t%h='janv.'\t%B='janvier'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+#endif
+
+  check(SV("%b='mai'\t%h='mai'\t%B='mai'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#ifdef _WIN32
+  check(loc,
+        SV("%b='1'\t%h='1'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b='5'\t%h='5'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+#elif defined(_AIX)                                                         // _WIN32
+  check(loc,
+        SV("%b='1月'\t%h='1月'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b='5月'\t%h='5月'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+#elif defined(__APPLE__)                                                    // _WIN32
+  check(loc,
+        SV("%b=' 1'\t%h=' 1'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b=' 5'\t%h=' 5'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+#elif defined(__FreeBSD__)                                                  // _WIN32
+  check(loc,
+        SV("%b=' 1月'\t%h=' 1月'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b=' 5月'\t%h=' 5月'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+#else                                                                       // _WIN32
+  check(loc,
+        SV("%b=' 1月'\t%h=' 1月'\t%B='1月'\t%m='01'\t%Om='一'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b=' 5月'\t%h=' 5月'\t%B='5月'\t%m='05'\t%Om='五'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+#endif                                                                      // _WIN32
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_day() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%d='%d'%t%%Od='%Od'%t%%e='%e'%t%%Oe='%Oe'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%d='%d'%t%%Od='%Od'%t%%e='%e'%t%%Oe='%Oe'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#else // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%d='01'\t%Od='一'\t%e=' 1'\t%Oe='一'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%d='13'\t%Od='�三'\t%e='13'\t%Oe='�三'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_weekday() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt =
+      SV("{:%%a='%a'%t%%A='%A'%t%%u='%u'%t%%Ou='%Ou'%t%%w='%w'%t%%Ow='%Ow'%n}");
+  constexpr std::basic_string_view<CharT> lfmt =
+      SV("{:L%%a='%a'%t%%A='%A'%t%%u='%u'%t%%Ou='%Ou'%t%%w='%w'%t%%Ow='%Ow'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%a='Thu'\t%A='Thursday'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%a='Sun'\t%A='Sunday'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{4'294'967'295s})); // 06:28:15 UTC on Sunday, 7 February 2106
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__)
+  check(SV("%a='Jeu'\t%A='Jeudi'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%a='Dim'\t%A='Dimanche'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{4'294'967'295s})); // 06:28:15 UTC on Sunday, 7 February 2106
+#else
+  check(SV("%a='jeu.'\t%A='jeudi'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%a='dim.'\t%A='dimanche'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{4'294'967'295s})); // 06:28:15 UTC on Sunday, 7 February 2106
+#endif
+
+  // Use supplied locale (ja_JP).
+  // This locale has a different alternate, but not on all platforms
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%a='木'\t%A='木曜日'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%a='日'\t%A='日曜日'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{4'294'967'295s})); // 06:28:15 UTC on Sunday, 7 February 2106
+#else  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%a='木'\t%A='木曜日'\t%u='4'\t%Ou='四'\t%w='4'\t%Ow='四'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%a='日'\t%A='日曜日'\t%u='7'\t%Ou='七'\t%w='0'\t%Ow='〇'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{4'294'967'295s})); // 06:28:15 UTC on Sunday, 7 February 2106
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_day_of_year() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%j='%j'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%j='%j'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%j='001'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+  check(SV("%j='138'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+  check(SV("%j='001'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+  check(SV("%j='138'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+  check(loc,
+        SV("%j='001'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%j='138'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_week() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%U='%U'%t%%OU='%OU'%t%%W='%W'%t%%OW='%OW'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%U='%U'%t%%OU='%OU'%t%%W='%W'%t%%OW='%OW'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+  check(SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+#else  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%U='00'\t%OU='〇'\t%W='00'\t%OW='〇'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%U='20'\t%OU='二�'\t%W='20'\t%OW='二�'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{2'000'000'000s})); // 03:33:20 UTC on Wednesday, 18 May 2033
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_iso_8601_week() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%g='%g'%t%%G='%G'%t%%V='%V'%t%%OV='%OV'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%g='%g'%t%%G='%G'%t%%V='%V'%t%%OV='%OV'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%g='70'\t%G='1970'\t%V='01'\t%OV='01'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%g='70'\t%G='1970'\t%V='01'\t%OV='01'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%g='70'\t%G='1970'\t%V='01'\t%OV='01'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#else  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%g='70'\t%G='1970'\t%V='01'\t%OV='一'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%g='09'\t%G='2009'\t%V='07'\t%OV='七'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_date() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%D='%D'%t%%F='%F'%t%%x='%x'%t%%Ex='%Ex'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%D='%D'%t%%F='%F'%t%%x='%x'%t%%Ex='%Ex'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%D='01/01/70'\t%F='1970-01-01'\t%x='01/01/70'\t%Ex='01/01/70'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='02/13/09'\t%Ex='02/13/09'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__) || defined(__FreeBSD__)
+  check(SV("%D='01/01/70'\t%F='1970-01-01'\t%x='01.01.1970'\t%Ex='01.01.1970'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='13.02.2009'\t%Ex='13.02.2009'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#else
+  check(SV("%D='01/01/70'\t%F='1970-01-01'\t%x='01/01/1970'\t%Ex='01/01/1970'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='13/02/2009'\t%Ex='13/02/2009'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#endif
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%D='01/01/70'\t%F='1970-01-01'\t%x='1970/01/01'\t%Ex='1970/01/01'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%D='02/13/09'\t%F='2009-02-13'\t%x='2009/02/13'\t%Ex='2009/02/13'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#else  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%D='01/01/70'\t%F='1970-01-01'\t%x='1970年01月01日'\t%Ex='昭和45年01月01日'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%D='02/13/09'\t%F='2009-02-13'\t%x='2009年02月13日'\t%Ex='平�21年02月13日'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_time() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt = SV(
+      "{:"
+      "%%H='%H'%t"
+      "%%OH='%OH'%t"
+      "%%I='%I'%t"
+      "%%OI='%OI'%t"
+      "%%M='%M'%t"
+      "%%OM='%OM'%t"
+      "%%S='%S'%t"
+      "%%OS='%OS'%t"
+      "%%p='%p'%t"
+      "%%R='%R'%t"
+      "%%T='%T'%t"
+      "%%r='%r'%t"
+      "%%X='%X'%t"
+      "%%EX='%EX'%t"
+      "%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV(
+      "{:L"
+      "%%H='%H'%t"
+      "%%OH='%OH'%t"
+      "%%I='%I'%t"
+      "%%OI='%OI'%t"
+      "%%M='%M'%t"
+      "%%OM='%OM'%t"
+      "%%S='%S'%t"
+      "%%OS='%OS'%t"
+      "%%p='%p'%t"
+      "%%R='%R'%t"
+      "%%T='%T'%t"
+      "%%r='%r'%t"
+      "%%X='%X'%t"
+      "%%EX='%EX'%t"
+      "%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+           "%p='AM'\t"
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+           "%r='12:00:00 AM'\t"
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+           "\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30.123'\t"
+           "%OS='30.123'\t"
+           "%p='PM'\t"
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+           "%r='11:31:30 PM'\t"
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+           "\n"),
+        fmt,
+        std::chrono::sys_time<std::chrono::milliseconds>(
+            1'234'567'890'123ms)); // 23:31:30 UTC on Friday, 13 February 2009
+  // Use the global locale (fr_FR)
+  check(SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+#if defined(_AIX)
+           "%p='AM'\t"
+#else
+           "%p=''\t"
+#endif
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+#ifdef _WIN32
+           "%r='00:00:00'\t"
+#elif defined(_AIX)
+           "%r='12:00:00 AM'\t"
+#elif defined(__APPLE__) || defined(__FreeBSD__)
+           "%r=''\t"
+#else
+           "%r='12:00:00 '\t"
+#endif
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+           "\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30,123'\t"
+           "%OS='30,123'\t"
+#if defined(_AIX)
+           "%p='PM'\t"
+#else
+           "%p=''\t"
+#endif
+           "%R='23:31'\t"
+           "%T='23:31:30,123'\t"
+#ifdef _WIN32
+           "%r='23:31:30'\t"
+#elif defined(_AIX)
+           "%r='11:31:30 PM'\t"
+#elif defined(__APPLE__) || defined(__FreeBSD__)
+           "%r=''\t"
+#elif defined(_WIN32)
+           "%r='23:31:30 '\t"
+#else
+           "%r='11:31:30 '\t"
+#endif
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+           "\n"),
+        lfmt,
+        std::chrono::sys_time<std::chrono::milliseconds>(
+            1'234'567'890'123ms)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.a
+#if defined(__APPLE__) || defined(_AIX) || defined(_WIN32) || defined(__FreeBSD__)
+  check(loc,
+        SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+#  if defined(__APPLE__)
+           "%p='AM'\t"
+#  else
+           "%p='��'\t"
+#  endif
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+#  if defined(__APPLE__) || defined(__FreeBSD__)
+#    if defined(__APPLE__)
+           "%r='12:00:00 AM'\t"
+#    else
+           "%r='12:00:00 ��'\t"
+#    endif
+           "%X='00時00分00秒'\t"
+           "%EX='00時00分00秒'\t"
+#  elif defined(_WIN32)
+           "%r='0:00:00'\t"
+           "%X='0:00:00'\t"
+           "%EX='0:00:00'\t"
+#  else
+           "%r='��12:00:00'\t"
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+#  endif
+           "\n"),
+        lfmt,
+        std::chrono::hh_mm_ss(0s));
+
+  check(loc,
+        SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30.123'\t"
+           "%OS='30.123'\t"
+#  if defined(__APPLE__)
+           "%p='PM'\t"
+#  else
+           "%p='�後'\t"
+#  endif
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+#  if defined(__APPLE__) || defined(__FreeBSD__)
+#    if defined(__APPLE__)
+           "%r='11:31:30 PM'\t"
+#    else
+           "%r='11:31:30 �後'\t"
+#    endif
+           "%X='23時31分30秒'\t"
+           "%EX='23時31分30秒'\t"
+#  elif defined(_WIN32)
+           "%r='23:31:30'\t"
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+#  else
+           "%r='�後11:31:30'\t"
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+#  endif
+           "\n"),
+        lfmt,
+        std::chrono::hh_mm_ss(23h + 31min + 30s + 123ms));
+#else  // defined(__APPLE__) || defined(_AIX) || defined(_WIN32) || defined(__FreeBSD__)
+  check(loc,
+        SV("%H='00'\t"
+           "%OH='〇'\t"
+           "%I='12'\t"
+           "%OI='�二'\t"
+           "%M='00'\t"
+           "%OM='〇'\t"
+           "%S='00'\t"
+           "%OS='〇'\t"
+           "%p='��'\t"
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+           "%r='��12時00分00秒'\t"
+           "%X='00時00分00秒'\t"
+           "%EX='00時00分00秒'\t"
+           "\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%H='23'\t"
+           "%OH='二�三'\t"
+           "%I='11'\t"
+           "%OI='�一'\t"
+           "%M='31'\t"
+           "%OM='三�一'\t"
+           "%S='30.123'\t"
+           "%OS='三�.123'\t"
+           "%p='�後'\t"
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+           "%r='�後11時31分30秒'\t"
+           "%X='23時31分30秒'\t"
+           "%EX='23時31分30秒'\t"
+           "\n"),
+        lfmt,
+        std::chrono::sys_time<std::chrono::milliseconds>(
+            1'234'567'890'123ms)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif // defined(__APPLE__) || defined(_AIX) || defined(_WIN32) || defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_date_time() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%c='%c'%t%%Ec='%Ec'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%c='%c'%t%%Ec='%Ec'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%c='Thu Jan  1 00:00:00 1970'\t%Ec='Thu Jan  1 00:00:00 1970'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%c='Fri Feb 13 23:31:30 2009'\t%Ec='Fri Feb 13 23:31:30 2009'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(
+// https://sourceware.org/bugzilla/show_bug.cgi?id=24054
+#if defined(__powerpc__) && defined(__linux__)
+      SV("%c='jeu. 01 janv. 1970 00:00:00 UTC'\t%Ec='jeu. 01 janv. 1970 00:00:00 UTC'\n"),
+#elif defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29
+      SV("%c='jeu. 01 janv. 1970 00:00:00 GMT'\t%Ec='jeu. 01 janv. 1970 00:00:00 GMT'\n"),
+#elif defined(_AIX)
+      SV("%c=' 1 janvier 1970 à 00:00:00 UTC'\t%Ec=' 1 janvier 1970 à 00:00:00 UTC'\n"),
+#elif defined(__APPLE__)
+      SV("%c='Jeu  1 jan 00:00:00 1970'\t%Ec='Jeu  1 jan 00:00:00 1970'\n"),
+#elif defined(_WIN32)
+      SV("%c='01/01/1970 00:00:00'\t%Ec='01/01/1970 00:00:00'\n"),
+#elif defined(__FreeBSD__)
+      SV("%c='jeu.  1 janv. 00:00:00 1970'\t%Ec='jeu.  1 janv. 00:00:00 1970'\n"),
+#else
+      SV("%c='jeu. 01 janv. 1970 00:00:00'\t%Ec='jeu. 01 janv. 1970 00:00:00'\n"),
+#endif
+      lfmt,
+      std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(
+// https://sourceware.org/bugzilla/show_bug.cgi?id=24054
+#if defined(__powerpc__) && defined(__linux__)
+      SV("%c='ven. 13 févr. 2009 23:31:30 UTC'\t%Ec='ven. 13 févr. 2009 23:31:30 UTC'\n"),
+#elif defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29
+      SV("%c='ven. 13 févr. 2009 23:31:30 GMT'\t%Ec='ven. 13 févr. 2009 23:31:30 GMT'\n"),
+#elif defined(_AIX)
+      SV("%c='13 février 2009 à 23:31:30 UTC'\t%Ec='13 février 2009 à 23:31:30 UTC'\n"),
+#elif defined(__APPLE__)
+      SV("%c='Ven 13 fév 23:31:30 2009'\t%Ec='Ven 13 fév 23:31:30 2009'\n"),
+#elif defined(_WIN32)
+      SV("%c='13/02/2009 23:31:30'\t%Ec='13/02/2009 23:31:30'\n"),
+#elif defined(__FreeBSD__)
+      SV("%c='ven. 13 févr. 23:31:30 2009'\t%Ec='ven. 13 févr. 23:31:30 2009'\n"),
+#else
+      SV("%c='ven. 13 févr. 2009 23:31:30'\t%Ec='ven. 13 févr. 2009 23:31:30'\n"),
+#endif
+      lfmt,
+      std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.a
+#if defined(__APPLE__) || defined(__FreeBSD__)
+  check(loc,
+        SV("%c='木  1/ 1 00:00:00 1970'\t%Ec='木  1/ 1 00:00:00 1970'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+  check(loc,
+        SV("%c='金  2/13 23:31:30 2009'\t%Ec='金  2/13 23:31:30 2009'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#elif defined(_AIX)                                                         // defined(__APPLE__)|| defined(__FreeBSD__)
+  check(loc,
+        SV("%c='1970年01月 1日 00:00:00 UTC'\t%Ec='1970年01月 1日 00:00:00 UTC'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+  check(loc,
+        SV("%c='2009年02月13日 23:31:30 UTC'\t%Ec='2009年02月13日 23:31:30 UTC'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#elif defined(_WIN32)                                                       // defined(__APPLE__)|| defined(__FreeBSD__)
+  check(loc,
+        SV("%c='1970/01/01 0:00:00'\t%Ec='1970/01/01 0:00:00'\n"),
+        lfmt,
+        std::chrono::sys_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+  check(loc,
+        SV("%c='2009/02/13 23:31:30'\t%Ec='2009/02/13 23:31:30'\n"),
+        lfmt,
+        std::chrono::sys_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                                                       // defined(__APPLE__)|| defined(__FreeBSD__)
+  check(loc,
+        SV("%c='1970年01月01日 00時00分00秒'\t%Ec='昭和45年01月01日 00時00分00秒'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%c='2009年02月13日 23時31分30秒'\t%Ec='平�21年02月13日 23時31分30秒'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{1'234'567'890s})); // 23:31:30 UTC on Friday, 13 February 2009
+#endif                                                                      // defined(__APPLE__)|| defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_time_zone() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='UTC'\n"),
+        fmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  // Use the global locale (fr_FR)
+  check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='UTC'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  // Use supplied locale (ja_JP).
+  check(loc,
+        SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='UTC'\n"),
+        lfmt,
+        std::chrono::zoned_time(std::chrono::sys_seconds{0s})); // 00:00:00 UTC Thursday, 1 January 1970
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values() {
+  test_valid_values_year<CharT>();
+  test_valid_values_month<CharT>();
+  test_valid_values_day<CharT>();
+  test_valid_values_weekday<CharT>();
+  test_valid_values_day_of_year<CharT>();
+  test_valid_values_week<CharT>();
+  test_valid_values_iso_8601_week<CharT>();
+  test_valid_values_date<CharT>();
+  test_valid_values_time<CharT>();
+  test_valid_values_date_time<CharT>();
+  test_valid_values_time_zone<CharT>();
+}
+
+template <class CharT>
+static void test() {
+  test_no_chrono_specs<CharT>();
+  test_valid_values<CharT>();
+
+  check_invalid_types<CharT>(
+      {SV("a"),  SV("A"),  SV("b"),  SV("B"),  SV("c"),  SV("C"),  SV("d"),  SV("D"),  SV("e"),  SV("F"),  SV("g"),
+       SV("G"),  SV("h"),  SV("H"),  SV("I"),  SV("j"),  SV("m"),  SV("M"),  SV("p"),  SV("r"),  SV("R"),  SV("S"),
+       SV("T"),  SV("u"),  SV("U"),  SV("V"),  SV("w"),  SV("W"),  SV("x"),  SV("X"),  SV("y"),  SV("Y"),  SV("z"),
+       SV("Z"),  SV("Ec"), SV("EC"), SV("Ex"), SV("EX"), SV("Ey"), SV("EY"), SV("Ez"), SV("Od"), SV("Oe"), SV("OH"),
+       SV("OI"), SV("Om"), SV("OM"), SV("OS"), SV("Ou"), SV("OU"), SV("OV"), SV("Ow"), SV("OW"), SV("Oy"), SV("Oz")},
+      std::chrono::zoned_time{});
+}
+
+int main(int, char**) {
+  test<char>();
+
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.zone/time.zone.zonedtime/test_offset_time_zone.h b/libcxx/test/std/time/time.zone/time.zone.zonedtime/test_offset_time_zone.h
index c137049bde8aa..e9262c5d95db1 100644
--- a/libcxx/test/std/time/time.zone/time.zone.zonedtime/test_offset_time_zone.h
+++ b/libcxx/test/std/time/time.zone/time.zone.zonedtime/test_offset_time_zone.h
@@ -13,6 +13,7 @@
 #include <cassert>
 #include <charconv>
 #include <chrono>
+#include <format>
 #include <string_view>
 #include <type_traits>
 
@@ -42,6 +43,8 @@ class offset_time_zone {
 
   offset_time_zone* operator->() { return this; }
 
+  const offset_time_zone* operator->() const { return this; }
+
   template <class Duration>
   std::chrono::sys_time<std::common_type_t<Duration, std::chrono::seconds>>
   to_sys(const std::chrono::local_time<Duration>& local) const {
@@ -49,6 +52,22 @@ class offset_time_zone {
         local.time_since_epoch() + offset_};
   }
 
+  template <class Duration>
+  std::chrono::local_time<std::common_type_t<Duration, std::chrono::seconds>>
+  to_local(const std::chrono::sys_time<Duration>& sys) const {
+    return std::chrono::local_time<std::common_type_t<Duration, std::chrono::seconds>>{
+        sys.time_since_epoch() - offset_};
+  }
+
+  template <class Duration>
+  std::chrono::sys_info get_info(const std::chrono::sys_time<Duration>&) const {
+    return {std::chrono::sys_seconds::min(),
+            std::chrono::sys_seconds::max(),
+            offset_,
+            std::chrono::minutes{0},
+            std::format("{:+03d}s", offset_.count())};
+  }
+
 private:
   std::chrono::seconds offset_;
 };
diff --git a/libcxx/test/std/time/time.zone/time.zone.zonedtime/time.zone.zonedtime.nonmembers/ostream.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.zonedtime/time.zone.zonedtime.nonmembers/ostream.pass.cpp
new file mode 100644
index 0000000000000..06131d66c0f5c
--- /dev/null
+++ b/libcxx/test/std/time/time.zone/time.zone.zonedtime/time.zone.zonedtime.nonmembers/ostream.pass.cpp
@@ -0,0 +1,351 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// TODO FMT This test should not require std::to_chars(floating-point)
+// XFAIL: availability-fp_to_chars-missing
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+
+// REQUIRES: locale.fr_FR.UTF-8
+// REQUIRES: locale.ja_JP.UTF-8
+
+// <chrono>
+
+// template<class charT, class traits, class Duration, class TimeZonePtr>
+// basic_ostream<charT, traits>&
+// operator<<(basic_ostream<charT, traits>& os,
+//            const zoned_time<Duration, TimeZonePtr>& t);
+
+#include <chrono>
+#include <cassert>
+#include <sstream>
+
+#include "assert_macros.h"
+#include "concat_macros.h"
+#include "make_string.h"
+#include "platform_support.h" // locale name macros
+#include "test_macros.h"
+#include "../test_offset_time_zone.h"
+
+#define SV(S) MAKE_STRING_VIEW(CharT, S)
+
+#define TEST_EQUAL(OUT, EXPECTED)                                                                                      \
+  TEST_REQUIRE(OUT == EXPECTED,                                                                                        \
+               TEST_WRITE_CONCATENATED(                                                                                \
+                   "\nExpression      ", #OUT, "\nExpected output ", EXPECTED, "\nActual output   ", OUT, '\n'));
+
+template <class CharT, class Duration, class TimeZonePtr>
+static std::basic_string<CharT> stream_c_locale(std::chrono::zoned_time<Duration, TimeZonePtr> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  sstr << time_point;
+  return sstr.str();
+}
+
+template <class CharT, class Duration, class TimeZonePtr>
+static std::basic_string<CharT> stream_fr_FR_locale(std::chrono::zoned_time<Duration, TimeZonePtr> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  const std::locale locale(LOCALE_fr_FR_UTF_8);
+  sstr.imbue(locale);
+  sstr << time_point;
+  return sstr.str();
+}
+
+template <class CharT, class Duration, class TimeZonePtr>
+static std::basic_string<CharT> stream_ja_JP_locale(std::chrono::zoned_time<Duration, TimeZonePtr> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  const std::locale locale(LOCALE_ja_JP_UTF_8);
+  sstr.imbue(locale);
+  sstr << time_point;
+  return sstr.str();
+}
+
+template <class CharT>
+static void test_c() {
+  using namespace std::literals::chrono_literals;
+
+  { //  Different durations
+    TEST_EQUAL(stream_c_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::nanoseconds>{42ns})),
+               SV("1970-01-01 01:00:00.000000042 +01"));
+
+    TEST_EQUAL(stream_c_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::microseconds>{42us})),
+               SV("1970-01-01 01:00:00.000042 +01"));
+
+    TEST_EQUAL(stream_c_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::milliseconds>{42ms})),
+               SV("1970-01-01 01:00:00.042 +01"));
+
+    TEST_EQUAL(
+        stream_c_locale<CharT>(std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::seconds>{42s})),
+        SV("1970-01-01 01:00:42 +01"));
+
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   "Etc/GMT-1", std::chrono::sys_time<std::chrono::days>{std::chrono::days{42}})),
+               SV("1970-02-12 01:00:00 +01"));
+
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   "Etc/GMT-1", std::chrono::sys_time<std::chrono::weeks>{std::chrono::weeks{42}})),
+               SV("1970-10-22 01:00:00 +01"));
+  }
+
+  { // Daylight saving time switches
+    // Pick an historic date where it's well known what the time zone rules were.
+    // This makes it unlikely updates to the database change these rules.
+
+    // Z Europe/Berlin 0:53:28 - LMT 1893 Ap
+    // ...
+    // 1 DE CE%sT 1980
+    // 1 E CE%sT
+    //
+    // ...
+    // R E 1979 1995 - S lastSu 1u 0 -
+    // R E 1981 ma - Mar lastSu 1u 1 S
+
+    // Pick an historic date where it's well known what the time zone rules were.
+    // This makes it unlikely updates to the database change these rules.
+
+    // Start of daylight saving time
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::March / 30 / 1986} + 0h + 59min + 59s)),
+               SV("1986-03-30 01:59:59 CET"));
+
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::March / 30 / 1986} + 1h)),
+               SV("1986-03-30 03:00:00 CEST"));
+
+    // End of daylight saving time
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 0h + 59min + 59s)),
+               SV("1986-09-28 02:59:59 CEST"));
+
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 1h)),
+               SV("1986-09-28 02:00:00 CET"));
+
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 1h + 59min + 59s)),
+               SV("1986-09-28 02:59:59 CET"));
+
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 2h)),
+               SV("1986-09-28 03:00:00 CET"));
+  }
+
+  { // offset pointer
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   offset_time_zone<offset_time_zone_flags::none>{}, std::chrono::sys_seconds{})),
+               SV("1970-01-01 00:00:00 +00s"));
+
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   offset_time_zone<offset_time_zone_flags::none>{"42"}, std::chrono::sys_seconds{})),
+               SV("1969-12-31 23:59:18 +42s"));
+
+    TEST_EQUAL(stream_c_locale<CharT>(std::chrono::zoned_time(
+                   offset_time_zone<offset_time_zone_flags::none>{"-42"}, std::chrono::sys_seconds{})),
+               SV("1970-01-01 00:00:42 -42s"));
+  }
+}
+
+template <class CharT>
+static void test_fr_FR() {
+  using namespace std::literals::chrono_literals;
+
+  { //  Different durations
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::nanoseconds>{42ns})),
+               SV("1970-01-01 01:00:00,000000042 +01"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::microseconds>{42us})),
+               SV("1970-01-01 01:00:00,000042 +01"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::milliseconds>{42ms})),
+               SV("1970-01-01 01:00:00,042 +01"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::seconds>{42s})),
+               SV("1970-01-01 01:00:42 +01"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   "Etc/GMT-1", std::chrono::sys_time<std::chrono::days>{std::chrono::days{42}})),
+               SV("1970-02-12 01:00:00 +01"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   "Etc/GMT-1", std::chrono::sys_time<std::chrono::weeks>{std::chrono::weeks{42}})),
+               SV("1970-10-22 01:00:00 +01"));
+  }
+
+  { // Daylight saving time switches
+    // Pick an historic date where it's well known what the time zone rules were.
+    // This makes it unlikely updates to the database change these rules.
+
+    // Z Europe/Berlin 0:53:28 - LMT 1893 Ap
+    // ...
+    // 1 DE CE%sT 1980
+    // 1 E CE%sT
+    //
+    // ...
+    // R E 1979 1995 - S lastSu 1u 0 -
+    // R E 1981 ma - Mar lastSu 1u 1 S
+
+    // Pick an historic date where it's well known what the time zone rules were.
+    // This makes it unlikely updates to the database change these rules.
+
+    // Start of daylight saving time
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::March / 30 / 1986} + 0h + 59min + 59s)),
+               SV("1986-03-30 01:59:59 CET"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::March / 30 / 1986} + 1h)),
+               SV("1986-03-30 03:00:00 CEST"));
+
+    // End of daylight saving time
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 0h + 59min + 59s)),
+               SV("1986-09-28 02:59:59 CEST"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 1h)),
+               SV("1986-09-28 02:00:00 CET"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 1h + 59min + 59s)),
+               SV("1986-09-28 02:59:59 CET"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 2h)),
+               SV("1986-09-28 03:00:00 CET"));
+  }
+
+  { // offset pointer
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   offset_time_zone<offset_time_zone_flags::none>{}, std::chrono::sys_seconds{})),
+               SV("1970-01-01 00:00:00 +00s"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   offset_time_zone<offset_time_zone_flags::none>{"42"}, std::chrono::sys_seconds{})),
+               SV("1969-12-31 23:59:18 +42s"));
+
+    TEST_EQUAL(stream_fr_FR_locale<CharT>(std::chrono::zoned_time(
+                   offset_time_zone<offset_time_zone_flags::none>{"-42"}, std::chrono::sys_seconds{})),
+               SV("1970-01-01 00:00:42 -42s"));
+  }
+}
+
+template <class CharT>
+static void test_ja_JP() {
+  using namespace std::literals::chrono_literals;
+
+  { //  Different durations
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::nanoseconds>{42ns})),
+               SV("1970-01-01 01:00:00.000000042 +01"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::microseconds>{42us})),
+               SV("1970-01-01 01:00:00.000042 +01"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::milliseconds>{42ms})),
+               SV("1970-01-01 01:00:00.042 +01"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(
+                   std::chrono::zoned_time("Etc/GMT-1", std::chrono::sys_time<std::chrono::seconds>{42s})),
+               SV("1970-01-01 01:00:42 +01"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   "Etc/GMT-1", std::chrono::sys_time<std::chrono::days>{std::chrono::days{42}})),
+               SV("1970-02-12 01:00:00 +01"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   "Etc/GMT-1", std::chrono::sys_time<std::chrono::weeks>{std::chrono::weeks{42}})),
+               SV("1970-10-22 01:00:00 +01"));
+  }
+
+  { // Daylight saving time switches
+    // Pick an historic date where it's well known what the time zone rules were.
+    // This makes it unlikely updates to the database change these rules.
+
+    // Z Europe/Berlin 0:53:28 - LMT 1893 Ap
+    // ...
+    // 1 DE CE%sT 1980
+    // 1 E CE%sT
+    //
+    // ...
+    // R E 1979 1995 - S lastSu 1u 0 -
+    // R E 1981 ma - Mar lastSu 1u 1 S
+
+    // Pick an historic date where it's well known what the time zone rules were.
+    // This makes it unlikely updates to the database change these rules.
+
+    // Start of daylight saving time
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::March / 30 / 1986} + 0h + 59min + 59s)),
+               SV("1986-03-30 01:59:59 CET"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::March / 30 / 1986} + 1h)),
+               SV("1986-03-30 03:00:00 CEST"));
+
+    // End of daylight saving time
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 0h + 59min + 59s)),
+               SV("1986-09-28 02:59:59 CEST"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 1h)),
+               SV("1986-09-28 02:00:00 CET"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 1h + 59min + 59s)),
+               SV("1986-09-28 02:59:59 CET"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   "Europe/Berlin", std::chrono::sys_days{std::chrono::September / 28 / 1986} + 2h)),
+               SV("1986-09-28 03:00:00 CET"));
+  }
+
+  { // offset pointer
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   offset_time_zone<offset_time_zone_flags::none>{}, std::chrono::sys_seconds{})),
+               SV("1970-01-01 00:00:00 +00s"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   offset_time_zone<offset_time_zone_flags::none>{"42"}, std::chrono::sys_seconds{})),
+               SV("1969-12-31 23:59:18 +42s"));
+
+    TEST_EQUAL(stream_ja_JP_locale<CharT>(std::chrono::zoned_time(
+                   offset_time_zone<offset_time_zone_flags::none>{"-42"}, std::chrono::sys_seconds{})),
+               SV("1970-01-01 00:00:42 -42s"));
+  }
+}
+
+template <class CharT>
+static void test() {
+  test_c<CharT>();
+  test_fr_FR<CharT>();
+  test_ja_JP<CharT>();
+}
+
+int main(int, char**) {
+  test<char>();
+
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>();
+#endif
+
+  return 0;
+}

From 29be889c2c9c9a92e7ed89bd71d141961517d7e5 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Sat, 20 Jul 2024 10:25:40 -0700
Subject: [PATCH 294/486] reland "[libc] implement cached process/thread
 identity (#98989)" (#99765)

---
 libc/config/config.json                       | 10 +++++
 libc/config/linux/aarch64/entrypoints.txt     |  1 +
 libc/config/linux/riscv/entrypoints.txt       |  1 +
 libc/config/linux/x86_64/entrypoints.txt      |  1 +
 libc/docs/configure.rst                       |  3 ++
 libc/docs/dev/undefined_behavior.rst          | 23 +++++++++++
 libc/spec/posix.td                            | 15 +++----
 libc/src/__support/OSUtil/CMakeLists.txt      | 17 ++++++++
 .../src/__support/OSUtil/linux/CMakeLists.txt | 13 ++++++
 libc/src/__support/OSUtil/linux/pid.cpp       | 20 +++++++++
 libc/src/__support/OSUtil/pid.h               | 41 +++++++++++++++++++
 libc/src/__support/threads/CMakeLists.txt     | 27 ++++++++++++
 .../__support/threads/linux/CMakeLists.txt    |  1 +
 libc/src/__support/threads/linux/rwlock.h     |  9 ++--
 libc/src/__support/threads/linux/thread.cpp   |  2 +
 libc/src/__support/threads/thread.h           | 27 +++++++++++-
 libc/src/__support/threads/tid.h              | 34 +++++++++++++++
 libc/src/unistd/CMakeLists.txt                | 10 +++++
 libc/src/unistd/getpid.h                      |  4 +-
 libc/src/unistd/gettid.cpp                    | 17 ++++++++
 libc/src/unistd/gettid.h                      | 21 ++++++++++
 libc/src/unistd/linux/CMakeLists.txt          |  4 +-
 libc/src/unistd/linux/fork.cpp                | 32 ++++++++++-----
 libc/src/unistd/linux/getpid.cpp              | 11 +----
 libc/startup/linux/CMakeLists.txt             |  1 +
 libc/startup/linux/do_start.cpp               |  5 +++
 .../integration/src/unistd/CMakeLists.txt     |  4 ++
 .../test/integration/src/unistd/fork_test.cpp | 24 ++++++++++-
 libc/test/src/unistd/CMakeLists.txt           | 10 +++++
 libc/test/src/unistd/gettid_test.cpp          | 15 +++++++
 30 files changed, 363 insertions(+), 40 deletions(-)
 create mode 100644 libc/src/__support/OSUtil/linux/pid.cpp
 create mode 100644 libc/src/__support/OSUtil/pid.h
 create mode 100644 libc/src/__support/threads/tid.h
 create mode 100644 libc/src/unistd/gettid.cpp
 create mode 100644 libc/src/unistd/gettid.h
 create mode 100644 libc/test/src/unistd/gettid_test.cpp

diff --git a/libc/config/config.json b/libc/config/config.json
index 94bfed894c173..2005f4297bfc1 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -77,6 +77,16 @@
       "doc": "Default size for the constinit freelist buffer used for the freelist malloc implementation (default 1o 1GB)."
     }
   },
+  "unistd": {
+    "LIBC_CONF_ENABLE_TID_CACHE": {
+      "value": true,
+      "doc": "Enable caching mechanism for gettid to avoid syscall (only effective in fullbuild mode, default to true). Please refer to Undefined Behavior documentation for implications."
+    },
+    "LIBC_CONF_ENABLE_PID_CACHE": {
+      "value": true,
+      "doc": "Enable caching mechanism for getpid to avoid syscall (default to true). Please refer to Undefined Behavior documentation for implications."
+    }
+  },
   "math": {
     "LIBC_CONF_MATH_OPTIMIZATIONS": {
       "value": 0,
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index e2f6bd74bb694..8afd3fb67197e 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -297,6 +297,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.geteuid
     libc.src.unistd.getpid
     libc.src.unistd.getppid
+    libc.src.unistd.gettid
     libc.src.unistd.getuid
     libc.src.unistd.isatty
     libc.src.unistd.link
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 33dd8d06173b2..f61936b758bd9 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -296,6 +296,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.geteuid
     libc.src.unistd.getpid
     libc.src.unistd.getppid
+    libc.src.unistd.gettid
     libc.src.unistd.getuid
     libc.src.unistd.isatty
     libc.src.unistd.link
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 7309e95644c74..f2029da83ee71 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -315,6 +315,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.geteuid
     libc.src.unistd.getpid
     libc.src.unistd.getppid
+    libc.src.unistd.gettid
     libc.src.unistd.getuid
     libc.src.unistd.isatty
     libc.src.unistd.link
diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst
index dfb35f6a6611a..5c55e4ab0f181 100644
--- a/libc/docs/configure.rst
+++ b/libc/docs/configure.rst
@@ -52,3 +52,6 @@ to learn about the defaults for your platform and target.
 * **"string" options**
     - ``LIBC_CONF_MEMSET_X86_USE_SOFTWARE_PREFETCHING``: Inserts prefetch for write instructions (PREFETCHW) for memset on x86 to recover performance when hardware prefetcher is disabled.
     - ``LIBC_CONF_STRING_UNSAFE_WIDE_READ``: Read more than a byte at a time to perform byte-string operations like strlen.
+* **"unistd" options**
+    - ``LIBC_CONF_ENABLE_PID_CACHE``: Enable caching mechanism for getpid to avoid syscall (default to true). Please refer to Undefined Behavior documentation for implications.
+    - ``LIBC_CONF_ENABLE_TID_CACHE``: Enable caching mechanism for gettid to avoid syscall (only effective in fullbuild mode, default to true). Please refer to Undefined Behavior documentation for implications.
diff --git a/libc/docs/dev/undefined_behavior.rst b/libc/docs/dev/undefined_behavior.rst
index 3faae3134ce2a..b712780222aa3 100644
--- a/libc/docs/dev/undefined_behavior.rst
+++ b/libc/docs/dev/undefined_behavior.rst
@@ -93,3 +93,26 @@ direction in this case.
 Non-const Constant Return Values
 --------------------------------
 Some libc functions, like ``dlerror()``, return ``char *`` instead of ``const char *`` and then tell the caller they promise not to to modify this value. Any modification of this value is undefined behavior.
+
+Cached ``getpid/gettid``
+------------------------
+Since version ``2.25``, glibc removes its cache mechanism for ``getpid/gettid`` 
+(See the history section in https://man7.org/linux/man-pages/man2/getpid.2.html).
+LLVM's libc still implements the cache as it is useful for fast deadlock detection.
+The cache mechanism is also implemented in MUSL and bionic. The tid/pid cache can 
+be disabled by setting ``LIBC_CONF_ENABLE_TID_CACHE`` and ``LIBC_CONF_ENABLE_PID_CACHE``
+to ``false`` respectively.
+
+Unwrapped ``SYS_clone/SYS_fork/SYS_vfork``
+------------------------------------------
+It is highly discouraged to use unwrapped ``SYS_clone/SYS_fork/SYS_vfork``. 
+First, calling such syscalls without provided libc wrappers ignores 
+all the ``pthread_atfork`` entries as libc can no longer detect the ``fork``. 
+Second, libc relies on the ``fork/clone`` wrappers to correctly maintain cache for
+process id and thread id, and other important process-specific states such as the list 
+of robust mutexes. Third, even if the user is to call ``exec*`` functions immediately, 
+there can still be other unexpected issues. For instance, there can be signal handlers 
+inherited from parent process triggered inside the instruction window between ``fork`` 
+and ``exec*``. As libc failed to maintain its internal states correctly, even though the
+functions used inside the signal handlers are marked as ``async-signal-safe`` (such as
+``getpid``), they will still return wrong values or lead to other even worse situations.
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index 1878b1ee2ae41..48f743dff4e6f 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -546,6 +546,11 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<PidT>,
           [ArgSpec<VoidType>]
         >,
+        FunctionSpec<
+          "gettid",
+          RetValSpec<PidT>,
+          [ArgSpec<VoidType>]
+        >,
         FunctionSpec<
           "getuid",
           RetValSpec<UidT>,
@@ -601,16 +606,6 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<IntType>,
           [ArgSpec<ConstCharPtr>]
         >,
-        FunctionSpec<
-          "getpid",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "getppid",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidType>]
-        >,
         FunctionSpec<
           "link",
           RetValSpec<IntType>,
diff --git a/libc/src/__support/OSUtil/CMakeLists.txt b/libc/src/__support/OSUtil/CMakeLists.txt
index 94d1042ccbb4a..517f888178718 100644
--- a/libc/src/__support/OSUtil/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/CMakeLists.txt
@@ -15,3 +15,20 @@ add_object_library(
   DEPENDS
     ${target_os_util}
 )
+
+if (LIBC_CONF_ENABLE_PID_CACHE)
+  set(libc_copt_enable_pid_cache 1)
+else()
+  set(libc_copt_enable_pid_cache 0)
+endif()
+
+if(TARGET libc.src.__support.OSUtil.${LIBC_TARGET_OS}.pid)
+  add_object_library(
+    pid
+    ALIAS
+    DEPENDS
+      .${LIBC_TARGET_OS}.pid
+    COMPILE_OPTIONS
+      -DLIBC_COPT_ENABLE_PID_CACHE=${libc_copt_enable_pid_cache}
+  )
+endif()
diff --git a/libc/src/__support/OSUtil/linux/CMakeLists.txt b/libc/src/__support/OSUtil/linux/CMakeLists.txt
index 089cad454d534..95a83d77d0257 100644
--- a/libc/src/__support/OSUtil/linux/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/CMakeLists.txt
@@ -23,3 +23,16 @@ add_object_library(
     libc.hdr.types.struct_f_owner_ex
     libc.hdr.types.off_t
 )
+
+add_object_library(
+  pid
+  SRCS
+    pid.cpp
+  HDRS
+    ../pid.h
+  DEPENDS
+    libc.src.__support.OSUtil.osutil
+    libc.src.__support.common
+    libc.hdr.types.pid_t
+    libc.include.sys_syscall
+)
diff --git a/libc/src/__support/OSUtil/linux/pid.cpp b/libc/src/__support/OSUtil/linux/pid.cpp
new file mode 100644
index 0000000000000..a8499af596229
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/pid.cpp
@@ -0,0 +1,20 @@
+//===------------ pid_t utilities implementation ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/OSUtil/pid.h"
+#include "src/__support/OSUtil/syscall.h"
+#include <sys/syscall.h>
+
+namespace LIBC_NAMESPACE_DECL {
+
+pid_t ProcessIdentity::cache = -1;
+pid_t ProcessIdentity::get_uncached() {
+  return syscall_impl<pid_t>(SYS_getpid);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/OSUtil/pid.h b/libc/src/__support/OSUtil/pid.h
new file mode 100644
index 0000000000000..d723abe728569
--- /dev/null
+++ b/libc/src/__support/OSUtil/pid.h
@@ -0,0 +1,41 @@
+//===------------ pid_t utilities -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_PID_H
+#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_PID_H
+#include "hdr/types/pid_t.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/optimization.h"
+
+#ifndef LIBC_COPT_ENABLE_PID_CACHE
+#define LIBC_COPT_ENABLE_PID_CACHE 1
+#endif
+
+namespace LIBC_NAMESPACE_DECL {
+
+class ProcessIdentity {
+  static LIBC_INLINE_VAR thread_local bool fork_inflight = true;
+  static pid_t cache;
+  static pid_t get_uncached();
+
+public:
+  LIBC_INLINE static void start_fork() { fork_inflight = true; }
+  LIBC_INLINE static void end_fork() { fork_inflight = false; }
+  LIBC_INLINE static void refresh_cache() { cache = get_uncached(); }
+  LIBC_INLINE static pid_t get() {
+#if LIBC_COPT_ENABLE_PID_CACHE
+    if (LIBC_LIKELY(!fork_inflight))
+      return cache;
+#endif
+    return get_uncached();
+  }
+};
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_PID_H
diff --git a/libc/src/__support/threads/CMakeLists.txt b/libc/src/__support/threads/CMakeLists.txt
index d2e46b8e2574e..f1a2f162acfc2 100644
--- a/libc/src/__support/threads/CMakeLists.txt
+++ b/libc/src/__support/threads/CMakeLists.txt
@@ -44,6 +44,12 @@ if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.mutex)
   )
 endif()
 
+if (LIBC_CONF_ENABLE_TID_CACHE)
+  set(libc_copt_enable_tid_cache 1)
+else()
+  set(libc_copt_enable_tid_cache 0)
+endif()
+
 add_header_library(
   thread_common
   HDRS
@@ -54,6 +60,9 @@ add_header_library(
     libc.src.__support.CPP.optional
     libc.src.__support.CPP.string_view
     libc.src.__support.CPP.stringstream
+    libc.hdr.types.pid_t
+  COMPILE_OPTIONS
+    -DLIBC_COPT_ENABLE_TID_CACHE=${libc_copt_enable_tid_cache}
 )
 
 if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.thread)
@@ -89,3 +98,21 @@ if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.CndVar)
     .${LIBC_TARGET_OS}.CndVar
   )
 endif()
+
+set(tid_dep)
+if (LLVM_LIBC_FULL_BUILD)
+  list(APPEND tid_dep libc.src.__support.thread)
+else()
+  list(APPEND tid_dep libc.src.__support.OSUtil.osutil)
+  list(APPEND tid_dep libc.include.sys_syscall)
+endif()
+
+add_header_library(
+  tid
+  HDRS
+    tid.h
+  DEPENDS
+    libc.src.__support.common
+    libc.hdr.types.pid_t
+    ${tid_dep}
+)
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index 8b7971584e77e..d86441dd67cd7 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -55,6 +55,7 @@ add_header_library(
     libc.src.__support.common
     libc.src.__support.OSUtil.osutil
     libc.src.__support.CPP.limits
+    libc.src.__support.threads.tid
   COMPILE_OPTIONS
     -DLIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT=${LIBC_CONF_RWLOCK_DEFAULT_SPIN_COUNT}
     ${monotonicity_flags}
diff --git a/libc/src/__support/threads/linux/rwlock.h b/libc/src/__support/threads/linux/rwlock.h
index d2fb0ce1a3c08..cae8aa6410686 100644
--- a/libc/src/__support/threads/linux/rwlock.h
+++ b/libc/src/__support/threads/linux/rwlock.h
@@ -23,6 +23,7 @@
 #include "src/__support/threads/linux/futex_word.h"
 #include "src/__support/threads/linux/raw_mutex.h"
 #include "src/__support/threads/sleep.h"
+#include "src/__support/threads/tid.h"
 
 #ifndef LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT
 #define LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT 100
@@ -336,8 +337,6 @@ class RwLock {
   LIBC_INLINE Role get_preference() const {
     return static_cast<Role>(preference);
   }
-  // TODO: use cached thread id once implemented.
-  LIBC_INLINE static pid_t gettid() { return syscall_impl<pid_t>(SYS_gettid); }
 
   template <Role role> LIBC_INLINE LockResult try_lock(RwState &old) {
     if constexpr (role == Role::Reader) {
@@ -359,7 +358,7 @@ class RwLock {
         if (LIBC_LIKELY(old.compare_exchange_weak_with(
                 state, old.set_writer_bit(), cpp::MemoryOrder::ACQUIRE,
                 cpp::MemoryOrder::RELAXED))) {
-          writer_tid.store(gettid(), cpp::MemoryOrder::RELAXED);
+          writer_tid.store(gettid_inline(), cpp::MemoryOrder::RELAXED);
           return LockResult::Success;
         }
         // Notice that old is updated by the compare_exchange_weak_with
@@ -394,7 +393,7 @@ class RwLock {
             unsigned spin_count = LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT) {
     // Phase 1: deadlock detection.
     // A deadlock happens if this is a RAW/WAW lock in the same thread.
-    if (writer_tid.load(cpp::MemoryOrder::RELAXED) == gettid())
+    if (writer_tid.load(cpp::MemoryOrder::RELAXED) == gettid_inline())
       return LockResult::Deadlock;
 
 #if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
@@ -520,7 +519,7 @@ class RwLock {
     if (old.has_active_writer()) {
       // The lock is held by a writer.
       // Check if we are the owner of the lock.
-      if (writer_tid.load(cpp::MemoryOrder::RELAXED) != gettid())
+      if (writer_tid.load(cpp::MemoryOrder::RELAXED) != gettid_inline())
         return LockResult::PermissionDenied;
       // clear writer tid.
       writer_tid.store(0, cpp::MemoryOrder::RELAXED);
diff --git a/libc/src/__support/threads/linux/thread.cpp b/libc/src/__support/threads/linux/thread.cpp
index 36b4a88eba9b4..c8ad086f3d1cb 100644
--- a/libc/src/__support/threads/linux/thread.cpp
+++ b/libc/src/__support/threads/linux/thread.cpp
@@ -518,4 +518,6 @@ void thread_exit(ThreadReturnValue retval, ThreadStyle style) {
   __builtin_unreachable();
 }
 
+pid_t Thread::get_uncached_tid() { return syscall_impl<pid_t>(SYS_gettid); }
+
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/threads/thread.h b/libc/src/__support/threads/thread.h
index ce23a880e048a..1805b6fd6182a 100644
--- a/libc/src/__support/threads/thread.h
+++ b/libc/src/__support/threads/thread.h
@@ -9,6 +9,11 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_THREAD_H
 #define LLVM_LIBC_SRC___SUPPORT_THREADS_THREAD_H
 
+#ifndef LIBC_COPT_ENABLE_TID_CACHE
+#define LIBC_COPT_ENABLE_TID_CACHE 1
+#endif
+
+#include "hdr/types/pid_t.h"
 #include "src/__support/CPP/atomic.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/CPP/string_view.h"
@@ -103,7 +108,7 @@ struct alignas(STACK_ALIGNMENT) ThreadAttributes {
   uintptr_t tls;                // Address to the thread TLS memory
   uintptr_t tls_size;           // The size of area pointed to by |tls|.
   unsigned char owned_stack; // Indicates if the thread owns this stack memory
-  int tid;
+  pid_t tid;
   ThreadStyle style;
   ThreadReturnValue retval;
   ThreadAtExitCallbackMgr *atexit_callback_mgr;
@@ -228,6 +233,26 @@ struct Thread {
 
   // Return the name of the thread in |name|. Return the error number of error.
   int get_name(cpp::StringStream &name) const;
+
+  static pid_t get_uncached_tid();
+
+  LIBC_INLINE void refresh_tid(pid_t cached = -1) {
+    if (cached >= 0)
+      this->attrib->tid = cached;
+    else
+      this->attrib->tid = get_uncached_tid();
+  }
+  LIBC_INLINE void invalidate_tid() { this->attrib->tid = -1; }
+
+  LIBC_INLINE pid_t get_tid() {
+#if LIBC_COPT_ENABLE_TID_CACHE
+    if (LIBC_UNLIKELY(this->attrib->tid < 0))
+      return get_uncached_tid();
+    return this->attrib->tid;
+#else
+    return get_uncached_tid();
+#endif
+  }
 };
 
 extern LIBC_THREAD_LOCAL Thread self;
diff --git a/libc/src/__support/threads/tid.h b/libc/src/__support/threads/tid.h
new file mode 100644
index 0000000000000..a575cff508a0f
--- /dev/null
+++ b/libc/src/__support/threads/tid.h
@@ -0,0 +1,34 @@
+//===--- Tid wrapper --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_TID_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_TID_H
+
+// This header is for internal usage which automatically dispatches full build
+// and overlay build behaviors.
+
+#include "hdr/types/pid_t.h"
+#include "src/__support/common.h"
+#ifdef LIBC_FULL_BUILD
+#include "src/__support/threads/thread.h"
+#else
+#include "src/__support/OSUtil/syscall.h"
+#include <sys/syscall.h>
+#endif // LIBC_FULL_BUILD
+
+namespace LIBC_NAMESPACE_DECL {
+LIBC_INLINE pid_t gettid_inline() {
+#ifdef LIBC_FULL_BUILD
+  return self.get_tid();
+#else
+  return syscall_impl<pid_t>(SYS_gettid);
+#endif
+}
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_TID_H
diff --git a/libc/src/unistd/CMakeLists.txt b/libc/src/unistd/CMakeLists.txt
index ddafcd7c92f21..ec767128588f6 100644
--- a/libc/src/unistd/CMakeLists.txt
+++ b/libc/src/unistd/CMakeLists.txt
@@ -333,3 +333,13 @@ add_entrypoint_external(
 add_entrypoint_external(
   opterr
 )
+
+add_entrypoint_object(
+  gettid
+  SRCS
+    gettid.cpp
+  HDRS
+    gettid.h
+  DEPENDS
+    libc.src.__support.threads.tid
+)
diff --git a/libc/src/unistd/getpid.h b/libc/src/unistd/getpid.h
index c3c55b0c06b10..5812df0dfecd6 100644
--- a/libc/src/unistd/getpid.h
+++ b/libc/src/unistd/getpid.h
@@ -9,12 +9,12 @@
 #ifndef LLVM_LIBC_SRC_UNISTD_GETPID_H
 #define LLVM_LIBC_SRC_UNISTD_GETPID_H
 
+#include "hdr/types/pid_t.h"
 #include "src/__support/macros/config.h"
-#include <unistd.h>
 
 namespace LIBC_NAMESPACE_DECL {
 
-pid_t getpid();
+pid_t getpid(void);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/unistd/gettid.cpp b/libc/src/unistd/gettid.cpp
new file mode 100644
index 0000000000000..6d8ed65fb753d
--- /dev/null
+++ b/libc/src/unistd/gettid.cpp
@@ -0,0 +1,17 @@
+//===-- Implementation file for gettid --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/unistd/gettid.h"
+#include "src/__support/common.h"
+#include "src/__support/threads/tid.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(pid_t, gettid, (void)) { return gettid_inline(); }
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/unistd/gettid.h b/libc/src/unistd/gettid.h
new file mode 100644
index 0000000000000..42283191be49b
--- /dev/null
+++ b/libc/src/unistd/gettid.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for gettid ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_UNISTD_GETTID_H
+#define LLVM_LIBC_SRC_UNISTD_GETTID_H
+
+#include "hdr/types/pid_t.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+pid_t gettid(void);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_UNISTD_GETTID_H
diff --git a/libc/src/unistd/linux/CMakeLists.txt b/libc/src/unistd/linux/CMakeLists.txt
index 7e733d7f002c3..651ea60d07a30 100644
--- a/libc/src/unistd/linux/CMakeLists.txt
+++ b/libc/src/unistd/linux/CMakeLists.txt
@@ -101,6 +101,7 @@ add_entrypoint_object(
     libc.include.sys_syscall
     libc.src.__support.threads.fork_callbacks
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.OSUtil.pid
     libc.src.__support.threads.thread
     libc.src.errno.errno
 )
@@ -204,8 +205,7 @@ add_entrypoint_object(
     ../getpid.h
   DEPENDS
     libc.include.unistd
-    libc.include.sys_syscall
-    libc.src.__support.OSUtil.osutil
+    libc.src.__support.OSUtil.pid
 )
 
 add_entrypoint_object(
diff --git a/libc/src/unistd/linux/fork.cpp b/libc/src/unistd/linux/fork.cpp
index 7d47665b16d3f..8fe1881733f34 100644
--- a/libc/src/unistd/linux/fork.cpp
+++ b/libc/src/unistd/linux/fork.cpp
@@ -8,13 +8,14 @@
 
 #include "src/unistd/fork.h"
 
+#include "src/__support/OSUtil/pid.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/fork_callbacks.h"
 #include "src/__support/threads/thread.h" // For thread self object
-
 #include "src/errno/libc_errno.h"
+
 #include <signal.h>      // For SIGCHLD
 #include <sys/syscall.h> // For syscall numbers.
 
@@ -25,6 +26,14 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(pid_t, fork, (void)) {
   invoke_prepare_callbacks();
+
+  // Invalidate tid/pid cache before fork to avoid post fork signal handler from
+  // getting wrong values. gettid() is not async-signal-safe, but let's provide
+  // our best efforts here.
+  pid_t parent_tid = self.get_tid();
+  self.invalidate_tid();
+  ProcessIdentity::start_fork();
+
 #ifdef SYS_fork
   pid_t ret = LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_fork);
 #elif defined(SYS_clone)
@@ -32,15 +41,6 @@ LLVM_LIBC_FUNCTION(pid_t, fork, (void)) {
 #else
 #error "fork and clone syscalls not available."
 #endif
-  if (ret == 0) {
-    // Return value is 0 in the child process.
-    // The child is created with a single thread whose self object will be a
-    // copy of parent process' thread which called fork. So, we have to fix up
-    // the child process' self object with the new process' tid.
-    self.attrib->tid = LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_gettid);
-    invoke_child_callbacks();
-    return 0;
-  }
 
   if (ret < 0) {
     // Error case, a child process was not created.
@@ -48,6 +48,18 @@ LLVM_LIBC_FUNCTION(pid_t, fork, (void)) {
     return -1;
   }
 
+  // Child process
+  if (ret == 0) {
+    self.refresh_tid();
+    ProcessIdentity::refresh_cache();
+    ProcessIdentity::end_fork();
+    invoke_child_callbacks();
+    return 0;
+  }
+
+  // Parent process
+  self.refresh_tid(parent_tid);
+  ProcessIdentity::end_fork();
   invoke_parent_callbacks();
   return ret;
 }
diff --git a/libc/src/unistd/linux/getpid.cpp b/libc/src/unistd/linux/getpid.cpp
index b24c86a15990f..65d6c8a3bea95 100644
--- a/libc/src/unistd/linux/getpid.cpp
+++ b/libc/src/unistd/linux/getpid.cpp
@@ -7,17 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/unistd/getpid.h"
-
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/OSUtil/pid.h"
 #include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-
-#include <sys/syscall.h> // For syscall numbers.
-
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(pid_t, getpid, ()) {
-  return LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_getpid);
-}
+LLVM_LIBC_FUNCTION(pid_t, getpid, (void)) { return ProcessIdentity::get(); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/startup/linux/CMakeLists.txt b/libc/startup/linux/CMakeLists.txt
index 336c5d0f6bfa2..585edf20f65be 100644
--- a/libc/startup/linux/CMakeLists.txt
+++ b/libc/startup/linux/CMakeLists.txt
@@ -101,6 +101,7 @@ add_object_library(
     libc.include.llvm-libc-macros.link_macros
     libc.src.__support.threads.thread
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.OSUtil.pid
     libc.src.stdlib.exit
     libc.src.stdlib.atexit
     libc.src.unistd.environ
diff --git a/libc/startup/linux/do_start.cpp b/libc/startup/linux/do_start.cpp
index 824c0e1cf8f26..4047c06ff25c1 100644
--- a/libc/startup/linux/do_start.cpp
+++ b/libc/startup/linux/do_start.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 #include "startup/linux/do_start.h"
 #include "include/llvm-libc-macros/link-macros.h"
+#include "src/__support/OSUtil/pid.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/thread.h"
@@ -127,6 +128,10 @@ static ThreadAttributes main_thread_attrib;
   if (tls.size != 0 && !set_thread_ptr(tls.tp))
     syscall_impl<long>(SYS_exit, 1);
 
+  // Validate process identity cache (TLS needed).
+  ProcessIdentity::refresh_cache();
+  ProcessIdentity::end_fork();
+
   self.attrib = &main_thread_attrib;
   main_thread_attrib.atexit_callback_mgr =
       internal::get_thread_atexit_callback_mgr();
diff --git a/libc/test/integration/src/unistd/CMakeLists.txt b/libc/test/integration/src/unistd/CMakeLists.txt
index 3f18231209512..f50405d0925e2 100644
--- a/libc/test/integration/src/unistd/CMakeLists.txt
+++ b/libc/test/integration/src/unistd/CMakeLists.txt
@@ -31,6 +31,10 @@ add_integration_test(
     libc.src.sys.wait.wait4
     libc.src.sys.wait.waitpid
     libc.src.unistd.fork
+    libc.src.unistd.getpid
+    libc.src.unistd.gettid
+    libc.src.stdlib.exit
+    libc.include.sys_syscall
 )
 
 if((${LIBC_TARGET_OS} STREQUAL "linux") AND (${LIBC_TARGET_ARCHITECTURE_IS_X86}))
diff --git a/libc/test/integration/src/unistd/fork_test.cpp b/libc/test/integration/src/unistd/fork_test.cpp
index 9c9213ed46316..4b82d5f195627 100644
--- a/libc/test/integration/src/unistd/fork_test.cpp
+++ b/libc/test/integration/src/unistd/fork_test.cpp
@@ -6,17 +6,21 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/OSUtil/syscall.h"
 #include "src/pthread/pthread_atfork.h"
 #include "src/signal/raise.h"
+#include "src/stdlib/exit.h"
 #include "src/sys/wait/wait.h"
 #include "src/sys/wait/wait4.h"
 #include "src/sys/wait/waitpid.h"
 #include "src/unistd/fork.h"
-
+#include "src/unistd/getpid.h"
+#include "src/unistd/gettid.h"
 #include "test/IntegrationTest/test.h"
 
 #include <errno.h>
 #include <signal.h>
+#include <sys/syscall.h>
 #include <sys/wait.h>
 #include <unistd.h>
 
@@ -140,7 +144,25 @@ void fork_with_atfork_callbacks() {
   ASSERT_NE(child, DONE);
 }
 
+void fork_pid_tid_test() {
+  pid_t pid = fork();
+  ASSERT_TRUE(pid >= 0);
+  ASSERT_EQ(LIBC_NAMESPACE::gettid(),
+            LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_gettid));
+  ASSERT_EQ(LIBC_NAMESPACE::getpid(),
+            LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_getpid));
+
+  if (pid == 0) {
+    LIBC_NAMESPACE::exit(0);
+  } else {
+    int status;
+    LIBC_NAMESPACE::waitpid(pid, &status, 0);
+    ASSERT_EQ(status, 0);
+  }
+}
+
 TEST_MAIN(int argc, char **argv, char **envp) {
+  fork_pid_tid_test();
   fork_and_wait_normal_exit();
   fork_and_wait4_normal_exit();
   fork_and_waitpid_normal_exit();
diff --git a/libc/test/src/unistd/CMakeLists.txt b/libc/test/src/unistd/CMakeLists.txt
index 332455b791aee..f8292653081f1 100644
--- a/libc/test/src/unistd/CMakeLists.txt
+++ b/libc/test/src/unistd/CMakeLists.txt
@@ -378,6 +378,16 @@ add_libc_unittest(
     libc.src.unistd.getpid
 )
 
+add_libc_unittest(
+  gettid_test
+  SUITE
+    libc_unistd_unittests
+  SRCS
+    gettid_test.cpp
+  DEPENDS
+    libc.src.unistd.gettid
+)
+
 add_libc_unittest(
   getppid_test
   SUITE
diff --git a/libc/test/src/unistd/gettid_test.cpp b/libc/test/src/unistd/gettid_test.cpp
new file mode 100644
index 0000000000000..c2330f4002279
--- /dev/null
+++ b/libc/test/src/unistd/gettid_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for gettid ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/unistd/gettid.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcGetTidTest, SmokeTest) {
+  // gettid always succeeds. So, we just call it as a smoke test.
+  ASSERT_GT(LIBC_NAMESPACE::gettid(), 0);
+}

From aa86f4f18549583f9227276cd116dbaf5625aa78 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 10:43:52 -0700
Subject: [PATCH 295/486] [MC] Remove unnecessary DWARFMustBeAtTheEnd check

36a15cb975334403216e6145d4abece3026af17a introduced the
DWARFMustBeAtTheEnd check to ensure DWARF sections were placed after all
text sections to help avoid out-of-range branches for Darwin ARM. The
commit removed a Darwin ARM hack from
20e5f5ed7930efdf2bd34bf099f24ac88798c5ea (2009), likely due to a
no-longer-relevant assembler limitation.

However, this check is no longer relevant due to the following:

* Our CodeGen approach reliably places DWARF sections at the end.
* Darwin AArch32 is less relevant today.

Removing this check also addresses a minor clang cc1as crash that could
occur when text sections were placed after DWARF sections
(e9ad54b3ee905ea3a77c35ca7d6e843b2c552e0b (2015)).
---
 llvm/include/llvm/MC/TargetRegistry.h         |  9 ++-
 llvm/lib/CodeGen/LLVMTargetMachine.cpp        |  9 +--
 llvm/lib/MC/MCMachOStreamer.cpp               | 56 ++-----------------
 .../MCTargetDesc/AArch64MCTargetDesc.cpp      | 11 ++--
 .../ARM/MCTargetDesc/ARMMCTargetDesc.cpp      |  5 +-
 5 files changed, 19 insertions(+), 71 deletions(-)

diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index 75387f84149db..68eb02076a8f6 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -201,8 +201,7 @@ class Target {
   using MachOStreamerCtorTy =
       MCStreamer *(*)(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB,
                       std::unique_ptr<MCObjectWriter> &&OW,
-                      std::unique_ptr<MCCodeEmitter> &&Emitter,
-                      bool DWARFMustBeAtTheEnd);
+                      std::unique_ptr<MCCodeEmitter> &&Emitter);
   using COFFStreamerCtorTy =
       MCStreamer *(*)(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB,
                       std::unique_ptr<MCObjectWriter> &&OW,
@@ -559,7 +558,7 @@ class Target {
                                      std::unique_ptr<MCObjectWriter> &&OW,
                                      std::unique_ptr<MCCodeEmitter> &&Emitter,
                                      const MCSubtargetInfo &STI, bool, bool,
-                                     bool DWARFMustBeAtTheEnd) const {
+                                     bool) const {
     MCStreamer *S = nullptr;
     switch (T.getObjectFormat()) {
     case Triple::UnknownObjectFormat:
@@ -573,10 +572,10 @@ class Target {
     case Triple::MachO:
       if (MachOStreamerCtorFn)
         S = MachOStreamerCtorFn(Ctx, std::move(TAB), std::move(OW),
-                                std::move(Emitter), DWARFMustBeAtTheEnd);
+                                std::move(Emitter));
       else
         S = createMachOStreamer(Ctx, std::move(TAB), std::move(OW),
-                                std::move(Emitter), DWARFMustBeAtTheEnd);
+                                std::move(Emitter), false);
       break;
     case Triple::ELF:
       if (ELFStreamerCtorFn)
diff --git a/llvm/lib/CodeGen/LLVMTargetMachine.cpp b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
index 1d13173632833..3092177e37e98 100644
--- a/llvm/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
@@ -208,9 +208,8 @@ Expected<std::unique_ptr<MCStreamer>> LLVMTargetMachine::createMCStreamer(
         T, Context, std::unique_ptr<MCAsmBackend>(MAB),
         DwoOut ? MAB->createDwoObjectWriter(Out, *DwoOut)
                : MAB->createObjectWriter(Out),
-        std::unique_ptr<MCCodeEmitter>(MCE), STI, Options.MCOptions.MCRelaxAll,
-        Options.MCOptions.MCIncrementalLinkerCompatible,
-        /*DWARFMustBeAtTheEnd*/ true));
+        std::unique_ptr<MCCodeEmitter>(MCE), STI, /*ignore=*/false, false,
+        false));
     break;
   }
   case CodeGenFileType::Null:
@@ -284,9 +283,7 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, MCContext *&Ctx,
   const Triple &T = getTargetTriple();
   std::unique_ptr<MCStreamer> AsmStreamer(getTarget().createMCObjectStreamer(
       T, *Ctx, std::move(MAB), MAB->createObjectWriter(Out), std::move(MCE),
-      STI, Options.MCOptions.MCRelaxAll,
-      Options.MCOptions.MCIncrementalLinkerCompatible,
-      /*DWARFMustBeAtTheEnd*/ true));
+      STI, /*ignore=*/false, false, false));
 
   // Create the AsmPrinter, which takes ownership of AsmStreamer if successful.
   FunctionPass *Printer =
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index c8bc819129850..5231d10626f85 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -54,9 +54,6 @@ class MCMachOStreamer : public MCObjectStreamer {
   /// need for local relocations. False by default.
   bool LabelSections;
 
-  bool DWARFMustBeAtTheEnd;
-  bool CreatedADWARFSection;
-
   /// HasSectionLabel - map of which sections have already had a non-local
   /// label emitted to them. Used so we don't emit extraneous linker local
   /// labels in the middle of the section.
@@ -70,16 +67,13 @@ class MCMachOStreamer : public MCObjectStreamer {
 public:
   MCMachOStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> MAB,
                   std::unique_ptr<MCObjectWriter> OW,
-                  std::unique_ptr<MCCodeEmitter> Emitter,
-                  bool DWARFMustBeAtTheEnd, bool label)
+                  std::unique_ptr<MCCodeEmitter> Emitter, bool label)
       : MCObjectStreamer(Context, std::move(MAB), std::move(OW),
                          std::move(Emitter)),
-        LabelSections(label), DWARFMustBeAtTheEnd(DWARFMustBeAtTheEnd),
-        CreatedADWARFSection(false) {}
+        LabelSections(label) {}
 
   /// state management
   void reset() override {
-    CreatedADWARFSection = false;
     HasSectionLabel.clear();
     MCObjectStreamer::reset();
   }
@@ -141,48 +135,9 @@ class MCMachOStreamer : public MCObjectStreamer {
 
 } // end anonymous namespace.
 
-static bool canGoAfterDWARF(const MCSectionMachO &MSec) {
-  // These sections are created by the assembler itself after the end of
-  // the .s file.
-  StringRef SegName = MSec.getSegmentName();
-  StringRef SecName = MSec.getName();
-
-  if (SegName == "__LD" && SecName == "__compact_unwind")
-    return true;
-
-  if (SegName == "__IMPORT") {
-    if (SecName == "__jump_table")
-      return true;
-
-    if (SecName == "__pointers")
-      return true;
-  }
-
-  if (SegName == "__TEXT" && SecName == "__eh_frame")
-    return true;
-
-  if (SegName == "__DATA" &&
-      (SecName == "__llvm_addrsig" || SecName == "__nl_symbol_ptr" ||
-       SecName == "__thread_ptr"))
-    return true;
-  if (SegName == "__LLVM" && (SecName == "__cg_profile"))
-    return true;
-
-  if (SegName == "__DATA" && SecName == "__auth_ptr")
-    return true;
-
-  return false;
-}
-
 void MCMachOStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
   // Change the section normally.
-  bool Created = changeSectionImpl(Section, Subsection);
-  const MCSectionMachO &MSec = *cast<MCSectionMachO>(Section);
-  StringRef SegName = MSec.getSegmentName();
-  if (SegName == "__DWARF")
-    CreatedADWARFSection = true;
-  else if (Created && DWARFMustBeAtTheEnd && !canGoAfterDWARF(MSec))
-    assert(!CreatedADWARFSection && "Creating regular section after DWARF");
+  changeSectionImpl(Section, Subsection);
 
   // Output a linker-local symbol so we don't need section-relative local
   // relocations. The linker hates us when we do that.
@@ -576,9 +531,8 @@ MCStreamer *llvm::createMachOStreamer(MCContext &Context,
                                       std::unique_ptr<MCCodeEmitter> &&CE,
                                       bool DWARFMustBeAtTheEnd,
                                       bool LabelSections) {
-  MCMachOStreamer *S =
-      new MCMachOStreamer(Context, std::move(MAB), std::move(OW), std::move(CE),
-                          DWARFMustBeAtTheEnd, LabelSections);
+  MCMachOStreamer *S = new MCMachOStreamer(
+      Context, std::move(MAB), std::move(OW), std::move(CE), LabelSections);
   const Triple &Target = Context.getTargetTriple();
   S->emitVersionForTarget(
       Target, Context.getObjectFileInfo()->getSDKVersion(),
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index 1e367be9fa4ae..97c5f96388abe 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -386,13 +386,12 @@ static MCStreamer *createELFStreamer(const Triple &T, MCContext &Ctx,
                                   std::move(Emitter));
 }
 
-static MCStreamer *createMachOStreamer(MCContext &Ctx,
-                                       std::unique_ptr<MCAsmBackend> &&TAB,
-                                       std::unique_ptr<MCObjectWriter> &&OW,
-                                       std::unique_ptr<MCCodeEmitter> &&Emitter,
-                                       bool DWARFMustBeAtTheEnd) {
+static MCStreamer *
+createMachOStreamer(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB,
+                    std::unique_ptr<MCObjectWriter> &&OW,
+                    std::unique_ptr<MCCodeEmitter> &&Emitter) {
   return createMachOStreamer(Ctx, std::move(TAB), std::move(OW),
-                             std::move(Emitter), DWARFMustBeAtTheEnd,
+                             std::move(Emitter), /*ignore=*/false,
                              /*LabelSections*/ true);
 }
 
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
index 20603b6cf1b0b..cf4fc37f84553 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
@@ -369,10 +369,9 @@ static MCStreamer *createELFStreamer(const Triple &T, MCContext &Ctx,
 static MCStreamer *
 createARMMachOStreamer(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&MAB,
                        std::unique_ptr<MCObjectWriter> &&OW,
-                       std::unique_ptr<MCCodeEmitter> &&Emitter,
-                       bool DWARFMustBeAtTheEnd) {
+                       std::unique_ptr<MCCodeEmitter> &&Emitter) {
   return createMachOStreamer(Ctx, std::move(MAB), std::move(OW),
-                             std::move(Emitter), DWARFMustBeAtTheEnd);
+                             std::move(Emitter), false);
 }
 
 static MCInstPrinter *createARMMCInstPrinter(const Triple &T,

From d1578848e99a9190ec9d1a5e56a5a88cb0d2649c Mon Sep 17 00:00:00 2001
From: Kevin Gleason <gleasonk@google.com>
Date: Sat, 20 Jul 2024 12:57:50 -0500
Subject: [PATCH 296/486] Add logging for emit functions in BytecodeWriter.cpp
 (#99558)

Recently there was a change to materializing unrealized conversion
casts, which inserted conversion that previously did not exist during
legalization (https://github.com/llvm/llvm-project/pull/97903), after
these cases are inserted and then washed away after transformation
completes, it caused the use-list ordering of an op to change in some
cases: `my.add %arg0(use1), %arg0(use2) --> my.add %arg0(use2),
%arg0(use1)`, which subtly changes the bytecode emitted since this is
considered a custom use-list.

When investigating why the bytecode had changed I added the following
logging which helped track down the difference, in my case it showed
extra bytes with "use-list section". With
`-debug-only=mlir-bytecode-writer` emits logs like the following,
detailing the source of written bytes:

```
emitBytes(4b)	bytecode header
emitVarInt(6)	bytecode version
emitByte(13)	bytecode version
emitBytes(17b)	bytecode producer
emitByte(0)	null terminator
emitVarInt(2)	dialects count
...
emitByte(5)	dialect version
emitVarInt(4)	op names count
emitByte(9)	op names count
emitVarInt(0)	dialect number
...
emitVarInt(2)	dialect writer
emitByte(5)	dialect writer
emitVarInt(9259963783827161088)	dialect APInt
...
emitVarInt(3)	attr/type offset
emitByte(7)	attr/type offset
emitByte(3)	section code
emitVarInt(18)	section size
...
```

Note: this uses string constants and `StringLiteral`, I'm not sure if
these are washed away during compilation / OK to have these around for
debuggin, or if there's a better way to do this? Alternative was adding
many braces and `LLVM_DEBUG` calls at each callsite, but this felt more
error prone / likely to miss some callsites.
---
 mlir/lib/Bytecode/Writer/BytecodeWriter.cpp | 266 +++++++++++---------
 1 file changed, 153 insertions(+), 113 deletions(-)

diff --git a/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp b/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
index 449d7549eb724..0e96aa97abeba 100644
--- a/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
+++ b/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/CachedHashString.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
@@ -145,7 +146,9 @@ class EncodingEmitter {
   //===--------------------------------------------------------------------===//
 
   /// Backpatch a byte in the result buffer at the given offset.
-  void patchByte(uint64_t offset, uint8_t value) {
+  void patchByte(uint64_t offset, uint8_t value, StringLiteral desc) {
+    LLVM_DEBUG(llvm::dbgs() << "patchByte(" << offset << ',' << uint64_t(value)
+                            << ")\t" << desc << '\n');
     assert(offset < size() && offset >= prevResultSize &&
            "cannot patch previously emitted data");
     currentResult[offset - prevResultSize] = value;
@@ -153,7 +156,9 @@ class EncodingEmitter {
 
   /// Emit the provided blob of data, which is owned by the caller and is
   /// guaranteed to not die before the end of the bytecode process.
-  void emitOwnedBlob(ArrayRef<uint8_t> data) {
+  void emitOwnedBlob(ArrayRef<uint8_t> data, StringLiteral desc) {
+    LLVM_DEBUG(llvm::dbgs()
+               << "emitOwnedBlob(" << data.size() << "b)\t" << desc << '\n');
     // Push the current buffer before adding the provided data.
     appendResult(std::move(currentResult));
     appendOwnedResult(data);
@@ -163,17 +168,19 @@ class EncodingEmitter {
   /// owned by the caller and is guaranteed to not die before the end of the
   /// bytecode process. The alignment value is also encoded, making it available
   /// on load.
-  void emitOwnedBlobAndAlignment(ArrayRef<uint8_t> data, uint32_t alignment) {
-    emitVarInt(alignment);
-    emitVarInt(data.size());
+  void emitOwnedBlobAndAlignment(ArrayRef<uint8_t> data, uint32_t alignment,
+                                 StringLiteral desc) {
+    emitVarInt(alignment, desc);
+    emitVarInt(data.size(), desc);
 
     alignTo(alignment);
-    emitOwnedBlob(data);
+    emitOwnedBlob(data, desc);
   }
-  void emitOwnedBlobAndAlignment(ArrayRef<char> data, uint32_t alignment) {
+  void emitOwnedBlobAndAlignment(ArrayRef<char> data, uint32_t alignment,
+                                 StringLiteral desc) {
     ArrayRef<uint8_t> castedData(reinterpret_cast<const uint8_t *>(data.data()),
                                  data.size());
-    emitOwnedBlobAndAlignment(castedData, alignment);
+    emitOwnedBlobAndAlignment(castedData, alignment, desc);
   }
 
   /// Align the emitter to the given alignment.
@@ -187,7 +194,7 @@ class EncodingEmitter {
     size_t curOffset = size();
     size_t paddingSize = llvm::alignTo(curOffset, alignment) - curOffset;
     while (paddingSize--)
-      emitByte(bytecode::kAlignmentByte);
+      emitByte(bytecode::kAlignmentByte, "alignment byte");
 
     // Keep track of the maximum required alignment.
     requiredAlignment = std::max(requiredAlignment, alignment);
@@ -198,12 +205,16 @@ class EncodingEmitter {
 
   /// Emit a single byte.
   template <typename T>
-  void emitByte(T byte) {
+  void emitByte(T byte, StringLiteral desc) {
+    LLVM_DEBUG(llvm::dbgs()
+               << "emitByte(" << uint64_t(byte) << ")\t" << desc << '\n');
     currentResult.push_back(static_cast<uint8_t>(byte));
   }
 
   /// Emit a range of bytes.
-  void emitBytes(ArrayRef<uint8_t> bytes) {
+  void emitBytes(ArrayRef<uint8_t> bytes, StringLiteral desc) {
+    LLVM_DEBUG(llvm::dbgs()
+               << "emitBytes(" << bytes.size() << "b)\t" << desc << '\n');
     llvm::append_range(currentResult, bytes);
   }
 
@@ -214,40 +225,43 @@ class EncodingEmitter {
   /// All remaining bits in the first byte, along with all of the bits in
   /// additional bytes, provide the value of the integer encoded in
   /// little-endian order.
-  void emitVarInt(uint64_t value) {
+  void emitVarInt(uint64_t value, StringLiteral desc) {
+    LLVM_DEBUG(llvm::dbgs() << "emitVarInt(" << value << ")\t" << desc << '\n');
+
     // In the most common case, the value can be represented in a single byte.
     // Given how hot this case is, explicitly handle that here.
     if ((value >> 7) == 0)
-      return emitByte((value << 1) | 0x1);
-    emitMultiByteVarInt(value);
+      return emitByte((value << 1) | 0x1, desc);
+    emitMultiByteVarInt(value, desc);
   }
 
   /// Emit a signed variable length integer. Signed varints are encoded using
   /// a varint with zigzag encoding, meaning that we use the low bit of the
   /// value to indicate the sign of the value. This allows for more efficient
   /// encoding of negative values by limiting the number of active bits
-  void emitSignedVarInt(uint64_t value) {
-    emitVarInt((value << 1) ^ (uint64_t)((int64_t)value >> 63));
+  void emitSignedVarInt(uint64_t value, StringLiteral desc) {
+    emitVarInt((value << 1) ^ (uint64_t)((int64_t)value >> 63), desc);
   }
 
   /// Emit a variable length integer whose low bit is used to encode the
   /// provided flag, i.e. encoded as: (value << 1) | (flag ? 1 : 0).
-  void emitVarIntWithFlag(uint64_t value, bool flag) {
-    emitVarInt((value << 1) | (flag ? 1 : 0));
+  void emitVarIntWithFlag(uint64_t value, bool flag, StringLiteral desc) {
+    emitVarInt((value << 1) | (flag ? 1 : 0), desc);
   }
 
   //===--------------------------------------------------------------------===//
   // String Emission
 
   /// Emit the given string as a nul terminated string.
-  void emitNulTerminatedString(StringRef str) {
-    emitString(str);
-    emitByte(0);
+  void emitNulTerminatedString(StringRef str, StringLiteral desc) {
+    emitString(str, desc);
+    emitByte(0, "null terminator");
   }
 
   /// Emit the given string without a nul terminator.
-  void emitString(StringRef str) {
-    emitBytes({reinterpret_cast<const uint8_t *>(str.data()), str.size()});
+  void emitString(StringRef str, StringLiteral desc) {
+    emitBytes({reinterpret_cast<const uint8_t *>(str.data()), str.size()},
+              desc);
   }
 
   //===--------------------------------------------------------------------===//
@@ -260,14 +274,14 @@ class EncodingEmitter {
     // indicate whether the section alignment is present, so save an offset to
     // it.
     uint64_t codeOffset = currentResult.size();
-    emitByte(code);
-    emitVarInt(emitter.size());
+    emitByte(code, "section code");
+    emitVarInt(emitter.size(), "section size");
 
     // Integrate the alignment of the section into this emitter if necessary.
     unsigned emitterAlign = emitter.requiredAlignment;
     if (emitterAlign > 1) {
       if (size() & (emitterAlign - 1)) {
-        emitVarInt(emitterAlign);
+        emitVarInt(emitterAlign, "section alignment");
         alignTo(emitterAlign);
 
         // Indicate that we needed to align the section, the high bit of the
@@ -295,7 +309,8 @@ class EncodingEmitter {
   /// fallback when the number of bytes needed to encode the value is greater
   /// than 1. We mark it noinline here so that the single byte hot path isn't
   /// pessimized.
-  LLVM_ATTRIBUTE_NOINLINE void emitMultiByteVarInt(uint64_t value);
+  LLVM_ATTRIBUTE_NOINLINE void emitMultiByteVarInt(uint64_t value,
+                                                   StringLiteral desc);
 
   /// Append a new result buffer to the current contents.
   void appendResult(std::vector<uint8_t> &&result) {
@@ -345,15 +360,15 @@ class StringSectionBuilder {
 
   /// Write the current set of strings to the given emitter.
   void write(EncodingEmitter &emitter) {
-    emitter.emitVarInt(strings.size());
+    emitter.emitVarInt(strings.size(), "string section size");
 
     // Emit the sizes in reverse order, so that we don't need to backpatch an
     // offset to the string data or have a separate section.
     for (const auto &it : llvm::reverse(strings))
-      emitter.emitVarInt(it.first.size() + 1);
+      emitter.emitVarInt(it.first.size() + 1, "string size");
     // Emit the string data itself.
     for (const auto &it : strings)
-      emitter.emitNulTerminatedString(it.first.val());
+      emitter.emitNulTerminatedString(it.first.val(), "string");
   }
 
 private:
@@ -380,32 +395,35 @@ class DialectWriter : public DialectBytecodeWriter {
   //===--------------------------------------------------------------------===//
 
   void writeAttribute(Attribute attr) override {
-    emitter.emitVarInt(numberingState.getNumber(attr));
+    emitter.emitVarInt(numberingState.getNumber(attr), "dialect attr");
   }
   void writeOptionalAttribute(Attribute attr) override {
     if (!attr) {
-      emitter.emitVarInt(0);
+      emitter.emitVarInt(0, "dialect optional attr none");
       return;
     }
-    emitter.emitVarIntWithFlag(numberingState.getNumber(attr), true);
+    emitter.emitVarIntWithFlag(numberingState.getNumber(attr), true,
+                               "dialect optional attr");
   }
 
   void writeType(Type type) override {
-    emitter.emitVarInt(numberingState.getNumber(type));
+    emitter.emitVarInt(numberingState.getNumber(type), "dialect type");
   }
 
   void writeResourceHandle(const AsmDialectResourceHandle &resource) override {
-    emitter.emitVarInt(numberingState.getNumber(resource));
+    emitter.emitVarInt(numberingState.getNumber(resource), "dialect resource");
   }
 
   //===--------------------------------------------------------------------===//
   // Primitives
   //===--------------------------------------------------------------------===//
 
-  void writeVarInt(uint64_t value) override { emitter.emitVarInt(value); }
+  void writeVarInt(uint64_t value) override {
+    emitter.emitVarInt(value, "dialect writer");
+  }
 
   void writeSignedVarInt(int64_t value) override {
-    emitter.emitSignedVarInt(value);
+    emitter.emitSignedVarInt(value, "dialect writer");
   }
 
   void writeAPIntWithKnownWidth(const APInt &value) override {
@@ -414,21 +432,21 @@ class DialectWriter : public DialectBytecodeWriter {
     // If the value is a single byte, just emit it directly without going
     // through a varint.
     if (bitWidth <= 8)
-      return emitter.emitByte(value.getLimitedValue());
+      return emitter.emitByte(value.getLimitedValue(), "dialect APInt");
 
     // If the value fits within a single varint, emit it directly.
     if (bitWidth <= 64)
-      return emitter.emitSignedVarInt(value.getLimitedValue());
+      return emitter.emitSignedVarInt(value.getLimitedValue(), "dialect APInt");
 
     // Otherwise, we need to encode a variable number of active words. We use
     // active words instead of the number of total words under the observation
     // that smaller values will be more common.
     unsigned numActiveWords = value.getActiveWords();
-    emitter.emitVarInt(numActiveWords);
+    emitter.emitVarInt(numActiveWords, "dialect APInt word count");
 
     const uint64_t *rawValueData = value.getRawData();
     for (unsigned i = 0; i < numActiveWords; ++i)
-      emitter.emitSignedVarInt(rawValueData[i]);
+      emitter.emitSignedVarInt(rawValueData[i], "dialect APInt word");
   }
 
   void writeAPFloatWithKnownSemantics(const APFloat &value) override {
@@ -436,16 +454,20 @@ class DialectWriter : public DialectBytecodeWriter {
   }
 
   void writeOwnedString(StringRef str) override {
-    emitter.emitVarInt(stringSection.insert(str));
+    emitter.emitVarInt(stringSection.insert(str), "dialect string");
   }
 
   void writeOwnedBlob(ArrayRef<char> blob) override {
-    emitter.emitVarInt(blob.size());
-    emitter.emitOwnedBlob(ArrayRef<uint8_t>(
-        reinterpret_cast<const uint8_t *>(blob.data()), blob.size()));
+    emitter.emitVarInt(blob.size(), "dialect blob");
+    emitter.emitOwnedBlob(
+        ArrayRef<uint8_t>(reinterpret_cast<const uint8_t *>(blob.data()),
+                          blob.size()),
+        "dialect blob");
   }
 
-  void writeOwnedBool(bool value) override { emitter.emitByte(value); }
+  void writeOwnedBool(bool value) override {
+    emitter.emitByte(value, "dialect bool");
+  }
 
   int64_t getBytecodeVersion() const override { return bytecodeVersion; }
 
@@ -486,7 +508,7 @@ class PropertiesSectionBuilder {
       if (!prop)
         return std::nullopt;
       EncodingEmitter sizeEmitter;
-      sizeEmitter.emitVarInt(numberingState.getNumber(prop));
+      sizeEmitter.emitVarInt(numberingState.getNumber(prop), "properties size");
       scratch.clear();
       llvm::raw_svector_ostream os(scratch);
       sizeEmitter.writeTo(os);
@@ -507,16 +529,17 @@ class PropertiesSectionBuilder {
 
   /// Write the current set of properties to the given emitter.
   void write(EncodingEmitter &emitter) {
-    emitter.emitVarInt(propertiesStorage.size());
+    emitter.emitVarInt(propertiesStorage.size(), "properties size");
     if (propertiesStorage.empty())
       return;
     for (const auto &storage : propertiesStorage) {
       if (storage.empty()) {
-        emitter.emitBytes(ArrayRef<uint8_t>());
+        emitter.emitBytes(ArrayRef<uint8_t>(), "empty properties");
         continue;
       }
       emitter.emitBytes(ArrayRef(reinterpret_cast<const uint8_t *>(&storage[0]),
-                                 storage.size()));
+                                 storage.size()),
+                        "property");
     }
   }
 
@@ -532,7 +555,7 @@ class PropertiesSectionBuilder {
     SmallVector<char> sizeScratch;
     {
       EncodingEmitter sizeEmitter;
-      sizeEmitter.emitVarInt(rawProperties.size());
+      sizeEmitter.emitVarInt(rawProperties.size(), "properties");
       llvm::raw_svector_ostream os(sizeScratch);
       sizeEmitter.writeTo(os);
     }
@@ -576,7 +599,8 @@ class RawEmitterOstream : public raw_ostream {
 
 private:
   void write_impl(const char *ptr, size_t size) override {
-    emitter.emitBytes({reinterpret_cast<const uint8_t *>(ptr), size});
+    emitter.emitBytes({reinterpret_cast<const uint8_t *>(ptr), size},
+                      "raw emitter");
   }
   uint64_t current_pos() const override { return emitter.size(); }
 
@@ -591,7 +615,7 @@ void EncodingEmitter::writeTo(raw_ostream &os) const {
   os.write((const char *)currentResult.data(), currentResult.size());
 }
 
-void EncodingEmitter::emitMultiByteVarInt(uint64_t value) {
+void EncodingEmitter::emitMultiByteVarInt(uint64_t value, StringLiteral desc) {
   // Compute the number of bytes needed to encode the value. Each byte can hold
   // up to 7-bits of data. We only check up to the number of bits we can encode
   // in the first byte (8).
@@ -601,16 +625,16 @@ void EncodingEmitter::emitMultiByteVarInt(uint64_t value) {
       uint64_t encodedValue = (value << 1) | 0x1;
       encodedValue <<= (numBytes - 1);
       llvm::support::ulittle64_t encodedValueLE(encodedValue);
-      emitBytes({reinterpret_cast<uint8_t *>(&encodedValueLE), numBytes});
+      emitBytes({reinterpret_cast<uint8_t *>(&encodedValueLE), numBytes}, desc);
       return;
     }
   }
 
   // If the value is too large to encode in a single byte, emit a special all
   // zero marker byte and splat the value directly.
-  emitByte(0);
+  emitByte(0, desc);
   llvm::support::ulittle64_t valueLE(value);
-  emitBytes({reinterpret_cast<uint8_t *>(&valueLE), sizeof(valueLE)});
+  emitBytes({reinterpret_cast<uint8_t *>(&valueLE), sizeof(valueLE)}, desc);
 }
 
 //===----------------------------------------------------------------------===//
@@ -696,7 +720,7 @@ LogicalResult BytecodeWriter::write(Operation *rootOp, raw_ostream &os) {
 
   // Emit the bytecode file header. This is how we identify the output as a
   // bytecode file.
-  emitter.emitString("ML\xefR");
+  emitter.emitString("ML\xefR", "bytecode header");
 
   // Emit the bytecode version.
   if (config.bytecodeVersion < bytecode::kMinSupportedVersion ||
@@ -706,10 +730,10 @@ LogicalResult BytecodeWriter::write(Operation *rootOp, raw_ostream &os) {
            << ", must be in range ["
            << static_cast<int64_t>(bytecode::kMinSupportedVersion) << ", "
            << static_cast<int64_t>(bytecode::kVersion) << ']';
-  emitter.emitVarInt(config.bytecodeVersion);
+  emitter.emitVarInt(config.bytecodeVersion, "bytecode version");
 
   // Emit the producer.
-  emitter.emitNulTerminatedString(config.producer);
+  emitter.emitNulTerminatedString(config.producer, "bytecode producer");
 
   // Emit the dialect section.
   writeDialectSection(emitter);
@@ -760,8 +784,8 @@ static void writeDialectGrouping(EncodingEmitter &emitter, EntriesT &&entries,
     });
 
     // Emit the dialect and number of elements.
-    emitter.emitVarInt(currentDialect->number);
-    emitter.emitVarInt(std::distance(groupStart, it));
+    emitter.emitVarInt(currentDialect->number, "dialect number");
+    emitter.emitVarInt(std::distance(groupStart, it), "dialect offset");
 
     // Emit the entries within the group.
     for (auto &entry : llvm::make_range(groupStart, it))
@@ -774,13 +798,13 @@ void BytecodeWriter::writeDialectSection(EncodingEmitter &emitter) {
 
   // Emit the referenced dialects.
   auto dialects = numberingState.getDialects();
-  dialectEmitter.emitVarInt(llvm::size(dialects));
+  dialectEmitter.emitVarInt(llvm::size(dialects), "dialects count");
   for (DialectNumbering &dialect : dialects) {
     // Write the string section and get the ID.
     size_t nameID = stringSection.insert(dialect.name);
 
     if (config.bytecodeVersion < bytecode::kDialectVersioning) {
-      dialectEmitter.emitVarInt(nameID);
+      dialectEmitter.emitVarInt(nameID, "dialect name ID");
       continue;
     }
 
@@ -798,22 +822,25 @@ void BytecodeWriter::writeDialectSection(EncodingEmitter &emitter) {
     // this in the dialect ID, so if there is no version, we don't write the
     // section.
     size_t versionAvailable = versionEmitter.size() > 0;
-    dialectEmitter.emitVarIntWithFlag(nameID, versionAvailable);
+    dialectEmitter.emitVarIntWithFlag(nameID, versionAvailable,
+                                      "dialect version");
     if (versionAvailable)
       dialectEmitter.emitSection(bytecode::Section::kDialectVersions,
                                  std::move(versionEmitter));
   }
 
   if (config.bytecodeVersion >= bytecode::kElideUnknownBlockArgLocation)
-    dialectEmitter.emitVarInt(size(numberingState.getOpNames()));
+    dialectEmitter.emitVarInt(size(numberingState.getOpNames()),
+                              "op names count");
 
   // Emit the referenced operation names grouped by dialect.
   auto emitOpName = [&](OpNameNumbering &name) {
     size_t stringId = stringSection.insert(name.name.stripDialect());
     if (config.bytecodeVersion < bytecode::kNativePropertiesEncoding)
-      dialectEmitter.emitVarInt(stringId);
+      dialectEmitter.emitVarInt(stringId, "dialect op name");
     else
-      dialectEmitter.emitVarIntWithFlag(stringId, name.name.isRegistered());
+      dialectEmitter.emitVarIntWithFlag(stringId, name.name.isRegistered(),
+                                        "dialect op name");
   };
   writeDialectGrouping(dialectEmitter, numberingState.getOpNames(), emitOpName);
 
@@ -826,8 +853,10 @@ void BytecodeWriter::writeDialectSection(EncodingEmitter &emitter) {
 void BytecodeWriter::writeAttrTypeSection(EncodingEmitter &emitter) {
   EncodingEmitter attrTypeEmitter;
   EncodingEmitter offsetEmitter;
-  offsetEmitter.emitVarInt(llvm::size(numberingState.getAttributes()));
-  offsetEmitter.emitVarInt(llvm::size(numberingState.getTypes()));
+  offsetEmitter.emitVarInt(llvm::size(numberingState.getAttributes()),
+                           "attributes count");
+  offsetEmitter.emitVarInt(llvm::size(numberingState.getTypes()),
+                           "types count");
 
   // A functor used to emit an attribute or type entry.
   uint64_t prevOffset = 0;
@@ -836,7 +865,7 @@ void BytecodeWriter::writeAttrTypeSection(EncodingEmitter &emitter) {
 
     auto emitAttrOrTypeRawImpl = [&]() -> void {
       RawEmitterOstream(attrTypeEmitter) << entryValue;
-      attrTypeEmitter.emitByte(0);
+      attrTypeEmitter.emitByte(0, "attr/type separator");
     };
     auto emitAttrOrTypeImpl = [&]() -> bool {
       // TODO: We don't currently support custom encoded mutable types and
@@ -882,7 +911,8 @@ void BytecodeWriter::writeAttrTypeSection(EncodingEmitter &emitter) {
 
     // Record the offset of this entry.
     uint64_t curOffset = attrTypeEmitter.size();
-    offsetEmitter.emitVarIntWithFlag(curOffset - prevOffset, hasCustomEncoding);
+    offsetEmitter.emitVarIntWithFlag(curOffset - prevOffset, hasCustomEncoding,
+                                     "attr/type offset");
     prevOffset = curOffset;
   };
 
@@ -910,30 +940,33 @@ LogicalResult BytecodeWriter::writeBlock(EncodingEmitter &emitter,
   // use the low bit of the operation count to indicate if the block has
   // arguments.
   unsigned numOps = numberingState.getOperationCount(block);
-  emitter.emitVarIntWithFlag(numOps, hasArgs);
+  emitter.emitVarIntWithFlag(numOps, hasArgs, "block num ops");
 
   // Emit the arguments of the block.
   if (hasArgs) {
-    emitter.emitVarInt(args.size());
+    emitter.emitVarInt(args.size(), "block args count");
     for (BlockArgument arg : args) {
       Location argLoc = arg.getLoc();
       if (config.bytecodeVersion >= bytecode::kElideUnknownBlockArgLocation) {
         emitter.emitVarIntWithFlag(numberingState.getNumber(arg.getType()),
-                                   !isa<UnknownLoc>(argLoc));
+                                   !isa<UnknownLoc>(argLoc), "block arg type");
         if (!isa<UnknownLoc>(argLoc))
-          emitter.emitVarInt(numberingState.getNumber(argLoc));
+          emitter.emitVarInt(numberingState.getNumber(argLoc),
+                             "block arg location");
       } else {
-        emitter.emitVarInt(numberingState.getNumber(arg.getType()));
-        emitter.emitVarInt(numberingState.getNumber(argLoc));
+        emitter.emitVarInt(numberingState.getNumber(arg.getType()),
+                           "block arg type");
+        emitter.emitVarInt(numberingState.getNumber(argLoc),
+                           "block arg location");
       }
     }
     if (config.bytecodeVersion >= bytecode::kUseListOrdering) {
       uint64_t maskOffset = emitter.size();
       uint8_t encodingMask = 0;
-      emitter.emitByte(0);
+      emitter.emitByte(0, "use-list separator");
       writeUseListOrders(emitter, encodingMask, args);
       if (encodingMask)
-        emitter.patchByte(maskOffset, encodingMask);
+        emitter.patchByte(maskOffset, encodingMask, "block patch encoding");
     }
   }
 
@@ -945,17 +978,17 @@ LogicalResult BytecodeWriter::writeBlock(EncodingEmitter &emitter,
 }
 
 LogicalResult BytecodeWriter::writeOp(EncodingEmitter &emitter, Operation *op) {
-  emitter.emitVarInt(numberingState.getNumber(op->getName()));
+  emitter.emitVarInt(numberingState.getNumber(op->getName()), "op name ID");
 
   // Emit a mask for the operation components. We need to fill this in later
   // (when we actually know what needs to be emitted), so emit a placeholder for
   // now.
   uint64_t maskOffset = emitter.size();
   uint8_t opEncodingMask = 0;
-  emitter.emitByte(0);
+  emitter.emitByte(0, "op separator");
 
   // Emit the location for this operation.
-  emitter.emitVarInt(numberingState.getNumber(op->getLoc()));
+  emitter.emitVarInt(numberingState.getNumber(op->getLoc()), "op location");
 
   // Emit the attributes of this operation.
   DictionaryAttr attrs = op->getDiscardableAttrDictionary();
@@ -969,7 +1002,7 @@ LogicalResult BytecodeWriter::writeOp(EncodingEmitter &emitter, Operation *op) {
   }
   if (!attrs.empty()) {
     opEncodingMask |= bytecode::OpEncodingMask::kHasAttrs;
-    emitter.emitVarInt(numberingState.getNumber(attrs));
+    emitter.emitVarInt(numberingState.getNumber(attrs), "op attrs count");
   }
 
   // Emit the properties of this operation, for now we still support deployment
@@ -978,32 +1011,32 @@ LogicalResult BytecodeWriter::writeOp(EncodingEmitter &emitter, Operation *op) {
     std::optional<ssize_t> propertiesId = propertiesSection.emit(op);
     if (propertiesId.has_value()) {
       opEncodingMask |= bytecode::OpEncodingMask::kHasProperties;
-      emitter.emitVarInt(*propertiesId);
+      emitter.emitVarInt(*propertiesId, "op properties ID");
     }
   }
 
   // Emit the result types of the operation.
   if (unsigned numResults = op->getNumResults()) {
     opEncodingMask |= bytecode::OpEncodingMask::kHasResults;
-    emitter.emitVarInt(numResults);
+    emitter.emitVarInt(numResults, "op results count");
     for (Type type : op->getResultTypes())
-      emitter.emitVarInt(numberingState.getNumber(type));
+      emitter.emitVarInt(numberingState.getNumber(type), "op result type");
   }
 
   // Emit the operands of the operation.
   if (unsigned numOperands = op->getNumOperands()) {
     opEncodingMask |= bytecode::OpEncodingMask::kHasOperands;
-    emitter.emitVarInt(numOperands);
+    emitter.emitVarInt(numOperands, "op operands count");
     for (Value operand : op->getOperands())
-      emitter.emitVarInt(numberingState.getNumber(operand));
+      emitter.emitVarInt(numberingState.getNumber(operand), "op operand types");
   }
 
   // Emit the successors of the operation.
   if (unsigned numSuccessors = op->getNumSuccessors()) {
     opEncodingMask |= bytecode::OpEncodingMask::kHasSuccessors;
-    emitter.emitVarInt(numSuccessors);
+    emitter.emitVarInt(numSuccessors, "op successors count");
     for (Block *successor : op->getSuccessors())
-      emitter.emitVarInt(numberingState.getNumber(successor));
+      emitter.emitVarInt(numberingState.getNumber(successor), "op successor");
   }
 
   // Emit the use-list orders to bytecode, so we can reconstruct the same order
@@ -1017,7 +1050,7 @@ LogicalResult BytecodeWriter::writeOp(EncodingEmitter &emitter, Operation *op) {
     opEncodingMask |= bytecode::OpEncodingMask::kHasInlineRegions;
 
   // Update the mask for the operation.
-  emitter.patchByte(maskOffset, opEncodingMask);
+  emitter.patchByte(maskOffset, opEncodingMask, "op encoding mask");
 
   // With the mask emitted, we can now emit the regions of the operation. We do
   // this after mask emission to avoid offset complications that may arise by
@@ -1025,7 +1058,8 @@ LogicalResult BytecodeWriter::writeOp(EncodingEmitter &emitter, Operation *op) {
   // op encoding mask is more annoying).
   if (numRegions) {
     bool isIsolatedFromAbove = numberingState.isIsolatedFromAbove(op);
-    emitter.emitVarIntWithFlag(numRegions, isIsolatedFromAbove);
+    emitter.emitVarIntWithFlag(numRegions, isIsolatedFromAbove,
+                               "op regions count");
 
     // If the region is not isolated from above, or we are emitting bytecode
     // targeting version <kLazyLoading, we don't use a section.
@@ -1096,8 +1130,9 @@ void BytecodeWriter::writeUseListOrders(EncodingEmitter &emitter,
   opEncodingMask |= bytecode::OpEncodingMask::kHasUseListOrders;
   // Emit the number of results that have a custom use-list order if the number
   // of results is greater than one.
-  if (range.size() != 1)
-    emitter.emitVarInt(map.size());
+  if (range.size() != 1) {
+    emitter.emitVarInt(map.size(), "custom use-list size");
+  }
 
   for (const auto &item : map) {
     auto resultIdx = item.getFirst();
@@ -1113,20 +1148,22 @@ void BytecodeWriter::writeUseListOrders(EncodingEmitter &emitter,
 
     // For single result, we don't need to store the result index.
     if (range.size() != 1)
-      emitter.emitVarInt(resultIdx);
+      emitter.emitVarInt(resultIdx, "use-list result index");
 
     if (indexPairEncoding) {
-      emitter.emitVarIntWithFlag(shuffledElements * 2, indexPairEncoding);
+      emitter.emitVarIntWithFlag(shuffledElements * 2, indexPairEncoding,
+                                 "use-list index pair size");
       for (auto pair : llvm::enumerate(useListOrder)) {
         if (pair.index() != pair.value()) {
-          emitter.emitVarInt(pair.value());
-          emitter.emitVarInt(pair.index());
+          emitter.emitVarInt(pair.value(), "use-list index pair first");
+          emitter.emitVarInt(pair.index(), "use-list index pair second");
         }
       }
     } else {
-      emitter.emitVarIntWithFlag(useListOrder.size(), indexPairEncoding);
+      emitter.emitVarIntWithFlag(useListOrder.size(), indexPairEncoding,
+                                 "use-list size");
       for (const auto &index : useListOrder)
-        emitter.emitVarInt(index);
+        emitter.emitVarInt(index, "use-list order");
     }
   }
 }
@@ -1136,15 +1173,15 @@ LogicalResult BytecodeWriter::writeRegion(EncodingEmitter &emitter,
   // If the region is empty, we only need to emit the number of blocks (which is
   // zero).
   if (region->empty()) {
-    emitter.emitVarInt(/*numBlocks*/ 0);
+    emitter.emitVarInt(/*numBlocks*/ 0, "region block count empty");
     return success();
   }
 
   // Emit the number of blocks and values within the region.
   unsigned numBlocks, numValues;
   std::tie(numBlocks, numValues) = numberingState.getBlockValueCount(region);
-  emitter.emitVarInt(numBlocks);
-  emitter.emitVarInt(numValues);
+  emitter.emitVarInt(numBlocks, "region block count");
+  emitter.emitVarInt(numValues, "region value count");
 
   // Emit the blocks within the region.
   for (Block &block : *region)
@@ -1160,7 +1197,7 @@ LogicalResult BytecodeWriter::writeIRSection(EncodingEmitter &emitter,
   // Write the IR section the same way as a block with no arguments. Note that
   // the low-bit of the operation count for a block is used to indicate if the
   // block has arguments, which in this case is always false.
-  irEmitter.emitVarIntWithFlag(/*numOps*/ 1, /*hasArgs*/ false);
+  irEmitter.emitVarIntWithFlag(/*numOps*/ 1, /*hasArgs*/ false, "ir section");
 
   // Emit the operations.
   if (failed(writeOp(irEmitter, op)))
@@ -1189,17 +1226,17 @@ class ResourceBuilder : public AsmResourceBuilder {
   void buildBlob(StringRef key, ArrayRef<char> data,
                  uint32_t dataAlignment) final {
     if (!shouldElideData)
-      emitter.emitOwnedBlobAndAlignment(data, dataAlignment);
+      emitter.emitOwnedBlobAndAlignment(data, dataAlignment, "resource blob");
     postProcessFn(key, AsmResourceEntryKind::Blob);
   }
   void buildBool(StringRef key, bool data) final {
     if (!shouldElideData)
-      emitter.emitByte(data);
+      emitter.emitByte(data, "resource bool");
     postProcessFn(key, AsmResourceEntryKind::Bool);
   }
   void buildString(StringRef key, StringRef data) final {
     if (!shouldElideData)
-      emitter.emitVarInt(stringSection.insert(data));
+      emitter.emitVarInt(stringSection.insert(data), "resource string");
     postProcessFn(key, AsmResourceEntryKind::String);
   }
 
@@ -1229,12 +1266,14 @@ void BytecodeWriter::writeResourceSection(Operation *op,
 
   // Functor used to emit a resource group defined by 'key'.
   auto emitResourceGroup = [&](uint64_t key) {
-    resourceOffsetEmitter.emitVarInt(key);
-    resourceOffsetEmitter.emitVarInt(curResourceEntries.size());
+    resourceOffsetEmitter.emitVarInt(key, "resource group key");
+    resourceOffsetEmitter.emitVarInt(curResourceEntries.size(),
+                                     "resource group size");
     for (auto [key, kind, size] : curResourceEntries) {
-      resourceOffsetEmitter.emitVarInt(stringSection.insert(key));
-      resourceOffsetEmitter.emitVarInt(size);
-      resourceOffsetEmitter.emitByte(kind);
+      resourceOffsetEmitter.emitVarInt(stringSection.insert(key),
+                                       "resource key");
+      resourceOffsetEmitter.emitVarInt(size, "resource size");
+      resourceOffsetEmitter.emitByte(kind, "resource kind");
     }
   };
 
@@ -1244,7 +1283,8 @@ void BytecodeWriter::writeResourceSection(Operation *op,
                                config.shouldElideResourceData);
 
   // Emit the external resource entries.
-  resourceOffsetEmitter.emitVarInt(config.externalResourcePrinters.size());
+  resourceOffsetEmitter.emitVarInt(config.externalResourcePrinters.size(),
+                                   "external resource printer count");
   for (const auto &printer : config.externalResourcePrinters) {
     curResourceEntries.clear();
     printer->buildResources(op, entryBuilder);

From f9dd885cb6e6b70deff935689bb0dfb7d5b6a1a4 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Sat, 20 Jul 2024 19:59:06 +0200
Subject: [PATCH 297/486] [libc++] Make std::pair trivially copyable if its
 members are (#89652)

This makes `std::pair` trivially copyable if its members are and we have
a way to do so. We need either C++20 with requires clauses or support
for `__attribute__((enable_if))`. Only Clang has support for this
attribute, so it's effectively clang or C++20.

Co-authored-by: Christopher Di Bella <cjdb@google.com>
---
 libcxx/include/__configuration/abi.h          |  4 ++
 libcxx/include/__type_traits/datasizeof.h     |  1 +
 libcxx/include/__utility/pair.h               | 46 +++++++++++++++++--
 .../pairs.pair/abi.trivial_copy_move.pass.cpp |  5 ++
 .../abi.trivially_copyable.compile.pass.cpp   | 22 ++++++++-
 5 files changed, 72 insertions(+), 6 deletions(-)

diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h
index cbde7887becf1..710548d90a649 100644
--- a/libcxx/include/__configuration/abi.h
+++ b/libcxx/include/__configuration/abi.h
@@ -98,6 +98,10 @@
 // and WCHAR_MAX. This ABI setting determines whether we should instead track whether the fill
 // value has been initialized using a separate boolean, which changes the ABI.
 #  define _LIBCPP_ABI_IOS_ALLOW_ARBITRARY_FILL_VALUE
+// Make a std::pair of trivially copyable types trivially copyable.
+// While this technically doesn't change the layout of pair itself, other types may decide to programatically change
+// their representation based on whether something is trivially copyable.
+#  define _LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR
 #elif _LIBCPP_ABI_VERSION == 1
 #  if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
 // Enable compiling copies of now inline methods into the dylib to support
diff --git a/libcxx/include/__type_traits/datasizeof.h b/libcxx/include/__type_traits/datasizeof.h
index 35c12921e8ffa..a27baf67cc2d8 100644
--- a/libcxx/include/__type_traits/datasizeof.h
+++ b/libcxx/include/__type_traits/datasizeof.h
@@ -54,6 +54,7 @@ struct _FirstPaddingByte<_Tp, true> {
 // the use as an extension.
 _LIBCPP_DIAGNOSTIC_PUSH
 _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Winvalid-offsetof")
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Winvalid-offsetof")
 template <class _Tp>
 inline const size_t __datasizeof_v = offsetof(_FirstPaddingByte<_Tp>, __first_padding_byte_);
 _LIBCPP_DIAGNOSTIC_POP
diff --git a/libcxx/include/__utility/pair.h b/libcxx/include/__utility/pair.h
index 0afbebcdc9f2a..c0002b7abb3ca 100644
--- a/libcxx/include/__utility/pair.h
+++ b/libcxx/include/__utility/pair.h
@@ -32,6 +32,7 @@
 #include <__type_traits/is_implicitly_default_constructible.h>
 #include <__type_traits/is_nothrow_assignable.h>
 #include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_reference.h>
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_swappable.h>
 #include <__type_traits/is_trivially_relocatable.h>
@@ -80,6 +81,38 @@ struct _LIBCPP_TEMPLATE_VIS pair
   _LIBCPP_HIDE_FROM_ABI pair(pair const&) = default;
   _LIBCPP_HIDE_FROM_ABI pair(pair&&)      = default;
 
+  // When we are requested for pair to be trivially copyable by the ABI macro, we use defaulted members
+  // if it is both legal to do it (i.e. no references) and we have a way to actually implement it, which requires
+  // the __enable_if__ attribute before C++20.
+#ifdef _LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR
+  // FIXME: This should really just be a static constexpr variable. It's in a struct to avoid gdb printing the value
+  // when printing a pair
+  struct __has_defaulted_members {
+    static const bool value = !is_reference<first_type>::value && !is_reference<second_type>::value;
+  };
+#  if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr pair& operator=(const pair&)
+    requires __has_defaulted_members::value
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr pair& operator=(pair&&)
+    requires __has_defaulted_members::value
+  = default;
+#  elif __has_attribute(__enable_if__)
+  _LIBCPP_HIDE_FROM_ABI pair& operator=(const pair&)
+      __attribute__((__enable_if__(__has_defaulted_members::value, ""))) = default;
+
+  _LIBCPP_HIDE_FROM_ABI pair& operator=(pair&&)
+      __attribute__((__enable_if__(__has_defaulted_members::value, ""))) = default;
+#  else
+#    error "_LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR isn't supported with this compiler"
+#  endif
+#else
+  struct __has_defaulted_members {
+    static const bool value = false;
+  };
+#endif // defined(_LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR) && __has_attribute(__enable_if__)
+
 #ifdef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI pair() : first(), second() {}
 
@@ -225,7 +258,8 @@ struct _LIBCPP_TEMPLATE_VIS pair
              typename __make_tuple_indices<sizeof...(_Args2) >::type()) {}
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair&
-  operator=(__conditional_t<is_copy_assignable<first_type>::value && is_copy_assignable<second_type>::value,
+  operator=(__conditional_t<!__has_defaulted_members::value && is_copy_assignable<first_type>::value &&
+                                is_copy_assignable<second_type>::value,
                             pair,
                             __nat> const& __p) noexcept(is_nothrow_copy_assignable<first_type>::value &&
                                                         is_nothrow_copy_assignable<second_type>::value) {
@@ -234,10 +268,12 @@ struct _LIBCPP_TEMPLATE_VIS pair
     return *this;
   }
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair& operator=(
-      __conditional_t<is_move_assignable<first_type>::value && is_move_assignable<second_type>::value, pair, __nat>&&
-          __p) noexcept(is_nothrow_move_assignable<first_type>::value &&
-                        is_nothrow_move_assignable<second_type>::value) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair&
+  operator=(__conditional_t<!__has_defaulted_members::value && is_move_assignable<first_type>::value &&
+                                is_move_assignable<second_type>::value,
+                            pair,
+                            __nat>&& __p) noexcept(is_nothrow_move_assignable<first_type>::value &&
+                                                   is_nothrow_move_assignable<second_type>::value) {
     first  = std::forward<first_type>(__p.first);
     second = std::forward<second_type>(__p.second);
     return *this;
diff --git a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/abi.trivial_copy_move.pass.cpp b/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/abi.trivial_copy_move.pass.cpp
index 3ec60c08b8eab..5481ba443046d 100644
--- a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/abi.trivial_copy_move.pass.cpp
+++ b/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/abi.trivial_copy_move.pass.cpp
@@ -162,8 +162,13 @@ void test_trivial()
         static_assert(!std::is_trivially_copy_constructible<P>::value, "");
         static_assert(!std::is_trivially_move_constructible<P>::value, "");
 #endif // TEST_STD_VER >= 11
+#ifndef _LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR
         static_assert(!std::is_trivially_copy_assignable<P>::value, "");
         static_assert(!std::is_trivially_move_assignable<P>::value, "");
+#else
+        static_assert(std::is_trivially_copy_assignable<P>::value, "");
+        static_assert(std::is_trivially_move_assignable<P>::value, "");
+#endif
         static_assert(std::is_trivially_destructible<P>::value, "");
     }
 }
diff --git a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/abi.trivially_copyable.compile.pass.cpp b/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/abi.trivially_copyable.compile.pass.cpp
index 1132b3e5def18..c5f9c8d0f2559 100644
--- a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/abi.trivially_copyable.compile.pass.cpp
+++ b/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/abi.trivially_copyable.compile.pass.cpp
@@ -47,11 +47,20 @@ static_assert(!std::is_trivially_copyable<std::pair<int&, int> >::value, "");
 static_assert(!std::is_trivially_copyable<std::pair<int, int&> >::value, "");
 static_assert(!std::is_trivially_copyable<std::pair<int&, int&> >::value, "");
 
+#ifdef _LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR
+static_assert(std::is_trivially_copyable<std::pair<int, int> >::value, "");
+static_assert(std::is_trivially_copyable<std::pair<int, char> >::value, "");
+static_assert(std::is_trivially_copyable<std::pair<char, int> >::value, "");
+static_assert(std::is_trivially_copyable<std::pair<std::pair<char, char>, int> >::value, "");
+static_assert(std::is_trivially_copyable<std::pair<trivially_copyable, int> >::value, "");
+#else
 static_assert(!std::is_trivially_copyable<std::pair<int, int> >::value, "");
 static_assert(!std::is_trivially_copyable<std::pair<int, char> >::value, "");
 static_assert(!std::is_trivially_copyable<std::pair<char, int> >::value, "");
 static_assert(!std::is_trivially_copyable<std::pair<std::pair<char, char>, int> >::value, "");
 static_assert(!std::is_trivially_copyable<std::pair<trivially_copyable, int> >::value, "");
+#endif // _LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR
+
 #if TEST_STD_VER == 03 // Known ABI difference
 static_assert(!std::is_trivially_copyable<std::pair<trivially_copyable_no_copy_assignment, int> >::value, "");
 static_assert(!std::is_trivially_copyable<std::pair<trivially_copyable_no_move_assignment, int> >::value, "");
@@ -59,10 +68,21 @@ static_assert(!std::is_trivially_copyable<std::pair<trivially_copyable_no_move_a
 static_assert(std::is_trivially_copyable<std::pair<trivially_copyable_no_copy_assignment, int> >::value, "");
 static_assert(std::is_trivially_copyable<std::pair<trivially_copyable_no_move_assignment, int> >::value, "");
 #endif
+
+#ifdef _LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR
+static_assert(std::is_trivially_copyable<std::pair<trivially_copyable_no_construction, int> >::value, "");
+#else
 static_assert(!std::is_trivially_copyable<std::pair<trivially_copyable_no_construction, int> >::value, "");
+#endif
 
 static_assert(std::is_trivially_copy_constructible<std::pair<int, int> >::value, "");
 static_assert(std::is_trivially_move_constructible<std::pair<int, int> >::value, "");
+static_assert(std::is_trivially_destructible<std::pair<int, int> >::value, "");
+
+#ifdef _LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR
+static_assert(std::is_trivially_copy_assignable<std::pair<int, int> >::value, "");
+static_assert(std::is_trivially_move_assignable<std::pair<int, int> >::value, "");
+#else
 static_assert(!std::is_trivially_copy_assignable<std::pair<int, int> >::value, "");
 static_assert(!std::is_trivially_move_assignable<std::pair<int, int> >::value, "");
-static_assert(std::is_trivially_destructible<std::pair<int, int> >::value, "");
+#endif // _LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR

From 91bf0a073951aa1ca4c357967ed5aff891941785 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Sat, 20 Jul 2024 20:12:31 +0200
Subject: [PATCH 298/486] [libc] Added static casts to fix implicit conversion
 warnings in 32-bit systems

This patch fixes:

randomness.h and getauxval.cpp were passing ssize_t as size_t
kernel_statx.h was assigning an uint64_t to uintptr_t
fopencookie.cpp was trying to create a FileIOResult using ssize_t but the constructor expected a size_t
thread.h was trying to call free_stack (which takes a size_t) with an unsigned long long. free_stack does the calculations using uintptr_t, so I changed the passing values to size_t
---
 libc/src/__support/HashTable/randomness.h |  2 +-
 libc/src/__support/threads/thread.h       | 10 +++++-----
 libc/src/stdio/fopencookie.cpp            |  8 ++++----
 libc/src/sys/auxv/linux/getauxval.cpp     |  2 +-
 libc/src/sys/stat/linux/kernel_statx.h    |  5 +++--
 5 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/libc/src/__support/HashTable/randomness.h b/libc/src/__support/HashTable/randomness.h
index 06d3e84a710e7..244dd41be3eec 100644
--- a/libc/src/__support/HashTable/randomness.h
+++ b/libc/src/__support/HashTable/randomness.h
@@ -36,7 +36,7 @@ LIBC_INLINE uint64_t next_random_seed() {
     entropy[1] = reinterpret_cast<uint64_t>(&state);
 #if defined(LIBC_HASHTABLE_USE_GETRANDOM)
     int errno_backup = libc_errno;
-    ssize_t count = sizeof(entropy);
+    size_t count = sizeof(entropy);
     uint8_t *buffer = reinterpret_cast<uint8_t *>(entropy);
     while (count > 0) {
       ssize_t len = getrandom(buffer, count, 0);
diff --git a/libc/src/__support/threads/thread.h b/libc/src/__support/threads/thread.h
index 1805b6fd6182a..931745299ffac 100644
--- a/libc/src/__support/threads/thread.h
+++ b/libc/src/__support/threads/thread.h
@@ -102,11 +102,11 @@ struct alignas(STACK_ALIGNMENT) ThreadAttributes {
   //          exits. It will clean up the thread resources once the thread
   //          exits.
   cpp::Atomic<uint32_t> detach_state;
-  void *stack;                  // Pointer to the thread stack
-  unsigned long long stacksize; // Size of the stack
-  unsigned long long guardsize; // Guard size on stack
-  uintptr_t tls;                // Address to the thread TLS memory
-  uintptr_t tls_size;           // The size of area pointed to by |tls|.
+  void *stack;               // Pointer to the thread stack
+  size_t stacksize;          // Size of the stack
+  size_t guardsize;          // Guard size on stack
+  uintptr_t tls;             // Address to the thread TLS memory
+  uintptr_t tls_size;        // The size of area pointed to by |tls|.
   unsigned char owned_stack; // Indicates if the thread owns this stack memory
   pid_t tid;
   ThreadStyle style;
diff --git a/libc/src/stdio/fopencookie.cpp b/libc/src/stdio/fopencookie.cpp
index 07be9a5635a92..9f5694e8e0581 100644
--- a/libc/src/stdio/fopencookie.cpp
+++ b/libc/src/stdio/fopencookie.cpp
@@ -43,16 +43,16 @@ FileIOResult CookieFile::cookie_write(File *f, const void *data, size_t size) {
   auto cookie_file = reinterpret_cast<CookieFile *>(f);
   if (cookie_file->ops.write == nullptr)
     return 0;
-  return cookie_file->ops.write(cookie_file->cookie,
-                                reinterpret_cast<const char *>(data), size);
+  return static_cast<size_t>(cookie_file->ops.write(
+      cookie_file->cookie, reinterpret_cast<const char *>(data), size));
 }
 
 FileIOResult CookieFile::cookie_read(File *f, void *data, size_t size) {
   auto cookie_file = reinterpret_cast<CookieFile *>(f);
   if (cookie_file->ops.read == nullptr)
     return 0;
-  return cookie_file->ops.read(cookie_file->cookie,
-                               reinterpret_cast<char *>(data), size);
+  return static_cast<size_t>(cookie_file->ops.read(
+      cookie_file->cookie, reinterpret_cast<char *>(data), size));
 }
 
 ErrorOr<off_t> CookieFile::cookie_seek(File *f, off_t offset, int whence) {
diff --git a/libc/src/sys/auxv/linux/getauxval.cpp b/libc/src/sys/auxv/linux/getauxval.cpp
index 2ca894d091532..bfa6b23b5ef91 100644
--- a/libc/src/sys/auxv/linux/getauxval.cpp
+++ b/libc/src/sys/auxv/linux/getauxval.cpp
@@ -155,7 +155,7 @@ static void initialize_auxv_once(void) {
 
 static AuxEntry read_entry(int fd) {
   AuxEntry buf;
-  ssize_t size = sizeof(AuxEntry);
+  size_t size = sizeof(AuxEntry);
   char *ptr = reinterpret_cast<char *>(&buf);
   while (size > 0) {
     ssize_t ret = read(fd, ptr, size);
diff --git a/libc/src/sys/stat/linux/kernel_statx.h b/libc/src/sys/stat/linux/kernel_statx.h
index f26f0b826ac1e..d0e223aec3e1e 100644
--- a/libc/src/sys/stat/linux/kernel_statx.h
+++ b/libc/src/sys/stat/linux/kernel_statx.h
@@ -80,7 +80,7 @@ LIBC_INLINE int statx(int dirfd, const char *__restrict path, int flags,
     return -ret;
 
   statbuf->st_dev = MKDEV(xbuf.stx_dev_major, xbuf.stx_dev_minor);
-  statbuf->st_ino = xbuf.stx_ino;
+  statbuf->st_ino = static_cast<decltype(statbuf->st_ino)>(xbuf.stx_ino);
   statbuf->st_mode = xbuf.stx_mode;
   statbuf->st_nlink = xbuf.stx_nlink;
   statbuf->st_uid = xbuf.stx_uid;
@@ -94,7 +94,8 @@ LIBC_INLINE int statx(int dirfd, const char *__restrict path, int flags,
   statbuf->st_ctim.tv_sec = xbuf.stx_ctime.tv_sec;
   statbuf->st_ctim.tv_nsec = xbuf.stx_ctime.tv_nsec;
   statbuf->st_blksize = xbuf.stx_blksize;
-  statbuf->st_blocks = xbuf.stx_blocks;
+  statbuf->st_blocks =
+      static_cast<decltype(statbuf->st_blocks)>(xbuf.stx_blocks);
 
   return 0;
 }

From 18f7ee5f13037acfc96d8324822b7a9d6c7a4f8b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 11:23:20 -0700
Subject: [PATCH 299/486] [MC] Remove unused Register*Streamer

Object streamers that do not need lib/Target customization do not need
Register*Streamer.
---
 llvm/include/llvm/MC/TargetRegistry.h         | 84 +++----------------
 llvm/lib/MC/MCXCOFFStreamer.cpp               |  9 --
 .../SPIRV/MCTargetDesc/SPIRVMCTargetDesc.cpp  | 10 ---
 3 files changed, 10 insertions(+), 93 deletions(-)

diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index 68eb02076a8f6..bb4566aa65e03 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -107,10 +107,6 @@ MCStreamer *createWasmStreamer(MCContext &Ctx,
                                std::unique_ptr<MCAsmBackend> &&TAB,
                                std::unique_ptr<MCObjectWriter> &&OW,
                                std::unique_ptr<MCCodeEmitter> &&CE);
-MCStreamer *createXCOFFStreamer(MCContext &Ctx,
-                                std::unique_ptr<MCAsmBackend> &&TAB,
-                                std::unique_ptr<MCObjectWriter> &&OW,
-                                std::unique_ptr<MCCodeEmitter> &&CE);
 MCStreamer *createSPIRVStreamer(MCContext &Ctx,
                                 std::unique_ptr<MCAsmBackend> &&TAB,
                                 std::unique_ptr<MCObjectWriter> &&OW,
@@ -194,10 +190,6 @@ class Target {
                       std::unique_ptr<MCAsmBackend> &&TAB,
                       std::unique_ptr<MCObjectWriter> &&OW,
                       std::unique_ptr<MCCodeEmitter> &&Emitter);
-  using GOFFStreamerCtorTy =
-      MCStreamer *(*)(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB,
-                      std::unique_ptr<MCObjectWriter> &&OW,
-                      std::unique_ptr<MCCodeEmitter> &&Emitter);
   using MachOStreamerCtorTy =
       MCStreamer *(*)(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB,
                       std::unique_ptr<MCObjectWriter> &&OW,
@@ -206,27 +198,11 @@ class Target {
       MCStreamer *(*)(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB,
                       std::unique_ptr<MCObjectWriter> &&OW,
                       std::unique_ptr<MCCodeEmitter> &&Emitter);
-  using WasmStreamerCtorTy =
-      MCStreamer *(*)(const Triple &T, MCContext &Ctx,
-                      std::unique_ptr<MCAsmBackend> &&TAB,
-                      std::unique_ptr<MCObjectWriter> &&OW,
-                      std::unique_ptr<MCCodeEmitter> &&Emitter);
   using XCOFFStreamerCtorTy =
       MCStreamer *(*)(const Triple &T, MCContext &Ctx,
                       std::unique_ptr<MCAsmBackend> &&TAB,
                       std::unique_ptr<MCObjectWriter> &&OW,
                       std::unique_ptr<MCCodeEmitter> &&Emitter);
-  using SPIRVStreamerCtorTy =
-      MCStreamer *(*)(const Triple &T, MCContext &Ctx,
-                      std::unique_ptr<MCAsmBackend> &&TAB,
-                      std::unique_ptr<MCObjectWriter> &&OW,
-                      std::unique_ptr<MCCodeEmitter> &&Emitter);
-
-  using DXContainerStreamerCtorTy =
-      MCStreamer *(*)(const Triple &T, MCContext &Ctx,
-                      std::unique_ptr<MCAsmBackend> &&TAB,
-                      std::unique_ptr<MCObjectWriter> &&OW,
-                      std::unique_ptr<MCCodeEmitter> &&Emitter);
 
   using NullTargetStreamerCtorTy = MCTargetStreamer *(*)(MCStreamer &S);
   using AsmTargetStreamerCtorTy = MCTargetStreamer *(*)(
@@ -328,13 +304,9 @@ class Target {
 
   // Construction functions for the various object formats, if registered.
   COFFStreamerCtorTy COFFStreamerCtorFn = nullptr;
-  GOFFStreamerCtorTy GOFFStreamerCtorFn = nullptr;
   MachOStreamerCtorTy MachOStreamerCtorFn = nullptr;
   ELFStreamerCtorTy ELFStreamerCtorFn = nullptr;
-  WasmStreamerCtorTy WasmStreamerCtorFn = nullptr;
   XCOFFStreamerCtorTy XCOFFStreamerCtorFn = nullptr;
-  SPIRVStreamerCtorTy SPIRVStreamerCtorFn = nullptr;
-  DXContainerStreamerCtorTy DXContainerStreamerCtorFn = nullptr;
 
   /// Construction function for this target's null TargetStreamer, if
   /// registered (default = nullptr).
@@ -586,44 +558,24 @@ class Target {
                               std::move(Emitter));
       break;
     case Triple::Wasm:
-      if (WasmStreamerCtorFn)
-        S = WasmStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW),
-                               std::move(Emitter));
-      else
-        S = createWasmStreamer(Ctx, std::move(TAB), std::move(OW),
-                               std::move(Emitter));
+      S = createWasmStreamer(Ctx, std::move(TAB), std::move(OW),
+                             std::move(Emitter));
       break;
     case Triple::GOFF:
-      if (GOFFStreamerCtorFn)
-        S = GOFFStreamerCtorFn(Ctx, std::move(TAB), std::move(OW),
-                               std::move(Emitter));
-      else
-        S = createGOFFStreamer(Ctx, std::move(TAB), std::move(OW),
-                               std::move(Emitter));
+      S = createGOFFStreamer(Ctx, std::move(TAB), std::move(OW),
+                             std::move(Emitter));
       break;
     case Triple::XCOFF:
-      if (XCOFFStreamerCtorFn)
-        S = XCOFFStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW),
-                                std::move(Emitter));
-      else
-        S = createXCOFFStreamer(Ctx, std::move(TAB), std::move(OW),
-                                std::move(Emitter));
+      S = XCOFFStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW),
+                              std::move(Emitter));
       break;
     case Triple::SPIRV:
-      if (SPIRVStreamerCtorFn)
-        S = SPIRVStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW),
-                                std::move(Emitter));
-      else
-        S = createSPIRVStreamer(Ctx, std::move(TAB), std::move(OW),
-                                std::move(Emitter));
+      S = createSPIRVStreamer(Ctx, std::move(TAB), std::move(OW),
+                              std::move(Emitter));
       break;
     case Triple::DXContainer:
-      if (DXContainerStreamerCtorFn)
-        S = DXContainerStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW),
-                                      std::move(Emitter));
-      else
-        S = createDXContainerStreamer(Ctx, std::move(TAB), std::move(OW),
-                                      std::move(Emitter));
+      S = createDXContainerStreamer(Ctx, std::move(TAB), std::move(OW),
+                                    std::move(Emitter));
       break;
     }
     if (ObjectTargetStreamerCtorFn)
@@ -1007,10 +959,6 @@ struct TargetRegistry {
     T.COFFStreamerCtorFn = Fn;
   }
 
-  static void RegisterGOFFStreamer(Target &T, Target::GOFFStreamerCtorTy Fn) {
-    T.GOFFStreamerCtorFn = Fn;
-  }
-
   static void RegisterMachOStreamer(Target &T, Target::MachOStreamerCtorTy Fn) {
     T.MachOStreamerCtorFn = Fn;
   }
@@ -1019,18 +967,6 @@ struct TargetRegistry {
     T.ELFStreamerCtorFn = Fn;
   }
 
-  static void RegisterSPIRVStreamer(Target &T, Target::SPIRVStreamerCtorTy Fn) {
-    T.SPIRVStreamerCtorFn = Fn;
-  }
-
-  static void RegisterDXContainerStreamer(Target &T, Target::DXContainerStreamerCtorTy Fn) {
-    T.DXContainerStreamerCtorFn = Fn;
-  }
-
-  static void RegisterWasmStreamer(Target &T, Target::WasmStreamerCtorTy Fn) {
-    T.WasmStreamerCtorFn = Fn;
-  }
-
   static void RegisterXCOFFStreamer(Target &T, Target::XCOFFStreamerCtorTy Fn) {
     T.XCOFFStreamerCtorFn = Fn;
   }
diff --git a/llvm/lib/MC/MCXCOFFStreamer.cpp b/llvm/lib/MC/MCXCOFFStreamer.cpp
index 175d7d6b6c31a..9cd46e504b554 100644
--- a/llvm/lib/MC/MCXCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCXCOFFStreamer.cpp
@@ -159,15 +159,6 @@ void MCXCOFFStreamer::emitInstToData(const MCInst &Inst,
   DF->getContents().append(Code.begin(), Code.end());
 }
 
-MCStreamer *llvm::createXCOFFStreamer(MCContext &Context,
-                                      std::unique_ptr<MCAsmBackend> &&MAB,
-                                      std::unique_ptr<MCObjectWriter> &&OW,
-                                      std::unique_ptr<MCCodeEmitter> &&CE) {
-  MCXCOFFStreamer *S = new MCXCOFFStreamer(Context, std::move(MAB),
-                                           std::move(OW), std::move(CE));
-  return S;
-}
-
 void MCXCOFFStreamer::emitXCOFFLocalCommonSymbol(MCSymbol *LabelSym,
                                                  uint64_t Size,
                                                  MCSymbol *CsectSym,
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCTargetDesc.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCTargetDesc.cpp
index 74ebaa9d0c004..2f302ed4c9e83 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCTargetDesc.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCTargetDesc.cpp
@@ -49,15 +49,6 @@ createSPIRVMCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
   return createSPIRVMCSubtargetInfoImpl(TT, CPU, /*TuneCPU*/ CPU, FS);
 }
 
-static MCStreamer *
-createSPIRVMCStreamer(const Triple &T, MCContext &Ctx,
-                      std::unique_ptr<MCAsmBackend> &&MAB,
-                      std::unique_ptr<MCObjectWriter> &&OW,
-                      std::unique_ptr<MCCodeEmitter> &&Emitter) {
-  return createSPIRVStreamer(Ctx, std::move(MAB), std::move(OW),
-                             std::move(Emitter));
-}
-
 static MCTargetStreamer *createTargetAsmStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &,
                                                  MCInstPrinter *, bool) {
@@ -94,7 +85,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSPIRVTargetMC() {
     TargetRegistry::RegisterMCInstrInfo(*T, createSPIRVMCInstrInfo);
     TargetRegistry::RegisterMCRegInfo(*T, createSPIRVMCRegisterInfo);
     TargetRegistry::RegisterMCSubtargetInfo(*T, createSPIRVMCSubtargetInfo);
-    TargetRegistry::RegisterSPIRVStreamer(*T, createSPIRVMCStreamer);
     TargetRegistry::RegisterMCInstPrinter(*T, createSPIRVMCInstPrinter);
     TargetRegistry::RegisterMCInstrAnalysis(*T, createSPIRVInstrAnalysis);
     TargetRegistry::RegisterMCCodeEmitter(*T, createSPIRVMCCodeEmitter);

From 2308c7f5c5b5dbb4b9a5a693b122f794fdaf00cd Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Sat, 20 Jul 2024 20:47:09 +0200
Subject: [PATCH 300/486] [libc] Enable most of the libc entrypoitns for riscv
 (#99771)

This patch enables most of the libc entrypoints for riscv, except for fstatvfs, statvfs, dmull and fmull which are currently failing compilation. float16 is also not added, as rv32 doesn't seem to support it yet.

This patch also fixes the call to seek, which should take an off_t, and was missed in PR #68269.
---
 libc/config/linux/riscv/entrypoints.txt       | 107 ++++++++++++++++--
 libc/config/linux/x86_64/entrypoints.txt      |   4 +-
 libc/src/__support/File/file.cpp              |   2 +-
 libc/src/__support/File/file.h                |   2 +-
 .../src/__support/File/platform_file_test.cpp |   3 +-
 5 files changed, 102 insertions(+), 16 deletions(-)

diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index f61936b758bd9..026100736bb67 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -17,6 +17,12 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.ctype.tolower
     libc.src.ctype.toupper
 
+    # dlfcn.h entrypoints
+    libc.src.dlfcn.dlclose
+    libc.src.dlfcn.dlerror
+    libc.src.dlfcn.dlopen
+    libc.src.dlfcn.dlsym
+
     # errno.h entrypoints
     libc.src.errno.errno
 
@@ -52,6 +58,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.mempcpy
     libc.src.string.memrchr
     libc.src.string.memset
+    libc.src.string.memset_explicit
     libc.src.string.rindex
     libc.src.string.stpcpy
     libc.src.string.stpncpy
@@ -180,6 +187,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.stdlib.qsort_r
     libc.src.stdlib.rand
     libc.src.stdlib.srand
+    libc.src.stdlib.strfromd
+    libc.src.stdlib.strfromf
+    libc.src.stdlib.strfroml
     libc.src.stdlib.strtod
     libc.src.stdlib.strtof
     libc.src.stdlib.strtol
@@ -197,6 +207,7 @@ set(TARGET_LIBC_ENTRYPOINTS
 
     # stdio.h entrypoints
     libc.src.stdio.fdopen
+    libc.src.stdio.fileno
     libc.src.stdio.fprintf
     libc.src.stdio.fscanf
     libc.src.stdio.printf
@@ -211,6 +222,14 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.stdio.vsnprintf
     libc.src.stdio.vsprintf
 
+    # sys/epoll.h entrypoints
+    libc.src.sys.epoll.epoll_create
+    libc.src.sys.epoll.epoll_create1
+    libc.src.sys.epoll.epoll_ctl
+    libc.src.sys.epoll.epoll_pwait
+    libc.src.sys.epoll.epoll_wait
+    libc.src.sys.epoll.epoll_pwait2
+
     # sys/mman.h entrypoints
     libc.src.sys.mman.madvise
     libc.src.sys.mman.mincore
@@ -261,12 +280,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     # sys/auxv.h entrypoints
     libc.src.sys.auxv.getauxval
 
-    # sys/epoll.h entrypoints
-    # Disabled due to epoll_wait syscalls not being available on this platform.
-    # libc.src.sys.epoll.epoll_wait
-    # libc.src.sys.epoll.epoll_pwait
-    # libc.src.sys.epoll.epoll_pwait2
-
     # termios.h entrypoints
     libc.src.termios.cfgetispeed
     libc.src.termios.cfgetospeed
@@ -303,6 +316,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.linkat
     libc.src.unistd.lseek
     libc.src.unistd.pathconf
+    libc.src.unistd.pipe
     libc.src.unistd.pread
     libc.src.unistd.pwrite
     libc.src.unistd.read
@@ -348,6 +362,9 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.atan2f
     libc.src.math.atanf
     libc.src.math.atanhf
+    libc.src.math.canonicalize
+    libc.src.math.canonicalizef
+    libc.src.math.canonicalizel
     libc.src.math.cbrt
     libc.src.math.cbrtf
     libc.src.math.ceil
@@ -366,6 +383,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.exp10f
     libc.src.math.exp2
     libc.src.math.exp2f
+    libc.src.math.exp2m1f
     libc.src.math.expf
     libc.src.math.expm1
     libc.src.math.expm1f
@@ -493,6 +511,9 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.rintf
     libc.src.math.rintl
     libc.src.math.round
+    libc.src.math.roundeven
+    libc.src.math.roundevenf
+    libc.src.math.roundevenl
     libc.src.math.roundf
     libc.src.math.roundl
     libc.src.math.scalbn
@@ -524,8 +545,10 @@ set(TARGET_LIBM_ENTRYPOINTS
 if(LIBC_TYPES_HAS_FLOAT128)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # math.h C23 _Float128 entrypoints
+    libc.src.math.canonicalizef128
     libc.src.math.ceilf128
     libc.src.math.copysignf128
+    libc.src.math.dmulf128
     libc.src.math.fabsf128
     libc.src.math.fdimf128
     libc.src.math.floorf128
@@ -540,6 +563,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.fminimum_numf128
     libc.src.math.fminimumf128
     libc.src.math.fmodf128
+    libc.src.math.fmulf128
     libc.src.math.frexpf128
     libc.src.math.fromfpf128
     libc.src.math.fromfpxf128
@@ -557,7 +581,9 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.nextafterf128
     libc.src.math.nextdownf128
     libc.src.math.nextupf128
+    libc.src.math.remquof128
     libc.src.math.rintf128
+    libc.src.math.roundevenf128
     libc.src.math.roundf128
     libc.src.math.scalbnf128
     libc.src.math.sqrtf128
@@ -567,14 +593,47 @@ if(LIBC_TYPES_HAS_FLOAT128)
   )
 endif()
 
+if(LIBC_COMPILER_HAS_FIXED_POINT)
+  list(APPEND TARGET_LIBM_ENTRYPOINTS
+    # stdfix.h _Fract and _Accum entrypoints
+    libc.src.stdfix.abshk
+    libc.src.stdfix.abshr
+    libc.src.stdfix.absk
+    libc.src.stdfix.abslk
+    libc.src.stdfix.abslr
+    libc.src.stdfix.absr
+    libc.src.stdfix.exphk
+    libc.src.stdfix.expk
+    libc.src.stdfix.roundhk
+    libc.src.stdfix.roundhr
+    libc.src.stdfix.roundk
+    libc.src.stdfix.roundlk
+    libc.src.stdfix.roundlr
+    libc.src.stdfix.roundr
+    libc.src.stdfix.rounduhk
+    libc.src.stdfix.rounduhr
+    libc.src.stdfix.rounduk
+    libc.src.stdfix.roundulk
+    libc.src.stdfix.roundulr
+    libc.src.stdfix.roundur
+    libc.src.stdfix.sqrtuhk
+    libc.src.stdfix.sqrtuhr
+    libc.src.stdfix.sqrtuk
+    libc.src.stdfix.sqrtur
+    libc.src.stdfix.sqrtulr
+    libc.src.stdfix.uhksqrtus
+    libc.src.stdfix.uksqrtui
+  )
+endif()
+
 if(LLVM_LIBC_FULL_BUILD)
   list(APPEND TARGET_LIBC_ENTRYPOINTS
-    # compiler entrypoints (no corresponding header)
-    libc.src.compiler.__stack_chk_fail
-
     # assert.h entrypoints
     libc.src.assert.__assert_fail
 
+    # compiler entrypoints (no corresponding header)
+    libc.src.compiler.__stack_chk_fail
+
     # dirent.h entrypoints
     libc.src.dirent.closedir
     libc.src.dirent.dirfd
@@ -599,6 +658,12 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.pthread.pthread_attr_setguardsize
     libc.src.pthread.pthread_attr_setstack
     libc.src.pthread.pthread_attr_setstacksize
+    libc.src.pthread.pthread_condattr_destroy
+    libc.src.pthread.pthread_condattr_getclock
+    libc.src.pthread.pthread_condattr_getpshared
+    libc.src.pthread.pthread_condattr_init
+    libc.src.pthread.pthread_condattr_setclock
+    libc.src.pthread.pthread_condattr_setpshared
     libc.src.pthread.pthread_create
     libc.src.pthread.pthread_detach
     libc.src.pthread.pthread_equal
@@ -621,6 +686,21 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.pthread.pthread_mutexattr_setrobust
     libc.src.pthread.pthread_mutexattr_settype
     libc.src.pthread.pthread_once
+    libc.src.pthread.pthread_rwlock_destroy
+    libc.src.pthread.pthread_rwlock_init
+    libc.src.pthread.pthread_rwlock_rdlock
+    libc.src.pthread.pthread_rwlock_timedrdlock
+    libc.src.pthread.pthread_rwlock_timedwrlock
+    libc.src.pthread.pthread_rwlock_tryrdlock
+    libc.src.pthread.pthread_rwlock_trywrlock
+    libc.src.pthread.pthread_rwlock_unlock
+    libc.src.pthread.pthread_rwlock_wrlock
+    libc.src.pthread.pthread_rwlockattr_destroy
+    libc.src.pthread.pthread_rwlockattr_getkind_np
+    libc.src.pthread.pthread_rwlockattr_getpshared
+    libc.src.pthread.pthread_rwlockattr_init
+    libc.src.pthread.pthread_rwlockattr_setkind_np
+    libc.src.pthread.pthread_rwlockattr_setpshared
     libc.src.pthread.pthread_self
     libc.src.pthread.pthread_setname_np
     libc.src.pthread.pthread_setspecific
@@ -644,7 +724,6 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.stdio.fgetc
     libc.src.stdio.fgetc_unlocked
     libc.src.stdio.fgets
-    libc.src.stdio.fileno
     libc.src.stdio.flockfile
     libc.src.stdio.fopen
     libc.src.stdio.fopencookie
@@ -653,7 +732,9 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.stdio.fread
     libc.src.stdio.fread_unlocked
     libc.src.stdio.fseek
+    libc.src.stdio.fseeko
     libc.src.stdio.ftell
+    libc.src.stdio.ftello
     libc.src.stdio.funlockfile
     libc.src.stdio.fwrite
     libc.src.stdio.fwrite_unlocked
@@ -674,9 +755,11 @@ if(LLVM_LIBC_FULL_BUILD)
     # stdlib.h entrypoints
     libc.src.stdlib._Exit
     libc.src.stdlib.abort
+    libc.src.stdlib.at_quick_exit
     libc.src.stdlib.atexit
     libc.src.stdlib.exit
     libc.src.stdlib.getenv
+    libc.src.stdlib.quick_exit
 
     # signal.h entrypoints
     libc.src.signal.kill
@@ -758,6 +841,10 @@ if(LLVM_LIBC_FULL_BUILD)
 
     # sys/select.h entrypoints
     libc.src.sys.select.select
+
+    # sys/socket.h entrypoints
+    libc.src.sys.socket.bind
+    libc.src.sys.socket.socket
   )
 endif()
 
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index f2029da83ee71..d4e246efb3f8a 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -228,9 +228,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.sys.epoll.epoll_ctl
     libc.src.sys.epoll.epoll_pwait
     libc.src.sys.epoll.epoll_wait
-    # TODO: Need to check if pwait2 is available before providing.
-    # https://github.com/llvm/llvm-project/issues/80060
-    # libc.src.sys.epoll.epoll_pwait2
+    libc.src.sys.epoll.epoll_pwait2
 
     # sys/mman.h entrypoints
     libc.src.sys.mman.madvise
diff --git a/libc/src/__support/File/file.cpp b/libc/src/__support/File/file.cpp
index 1b545c5096936..51811a27c1acd 100644
--- a/libc/src/__support/File/file.cpp
+++ b/libc/src/__support/File/file.cpp
@@ -282,7 +282,7 @@ int File::ungetc_unlocked(int c) {
   return c;
 }
 
-ErrorOr<int> File::seek(long offset, int whence) {
+ErrorOr<int> File::seek(off_t offset, int whence) {
   FileLock lock(this);
   if (prev_op == FileOp::WRITE && pos > 0) {
 
diff --git a/libc/src/__support/File/file.h b/libc/src/__support/File/file.h
index 0cedf866519d6..42e1d11b4ab1a 100644
--- a/libc/src/__support/File/file.h
+++ b/libc/src/__support/File/file.h
@@ -183,7 +183,7 @@ class File {
     return read_unlocked(data, len);
   }
 
-  ErrorOr<int> seek(long offset, int whence);
+  ErrorOr<int> seek(off_t offset, int whence);
 
   ErrorOr<off_t> tell();
 
diff --git a/libc/test/src/__support/File/platform_file_test.cpp b/libc/test/src/__support/File/platform_file_test.cpp
index 469d7500032b9..8aa07219a6527 100644
--- a/libc/test/src/__support/File/platform_file_test.cpp
+++ b/libc/test/src/__support/File/platform_file_test.cpp
@@ -103,7 +103,8 @@ TEST(LlvmLibcPlatformFileTest, CreateAppendSeekAndReadBack) {
   constexpr size_t APPEND_TEXT_SIZE = sizeof(APPEND_TEXT) - 1;
   ASSERT_EQ(file->write(APPEND_TEXT, APPEND_TEXT_SIZE).value, APPEND_TEXT_SIZE);
 
-  ASSERT_EQ(file->seek(-APPEND_TEXT_SIZE, SEEK_END).value(), 0);
+  ASSERT_EQ(file->seek(-static_cast<off_t>(APPEND_TEXT_SIZE), SEEK_END).value(),
+            0);
   char data[APPEND_TEXT_SIZE + 1];
   ASSERT_EQ(file->read(data, APPEND_TEXT_SIZE).value, APPEND_TEXT_SIZE);
   data[APPEND_TEXT_SIZE] = '\0';

From c59ee7ec627533e00f654d052ade51c9daf7e722 Mon Sep 17 00:00:00 2001
From: AdityaK <hiraditya@msn.com>
Date: Sat, 20 Jul 2024 11:53:11 -0700
Subject: [PATCH 301/486] Fix 40056: Prevent outlining of blocks with token
 type instructions (#99759)

Hot cold splitting should not outline:
1. Basic blocks with token type instructions
1. Functions with scoped EH personality (As suggested by Vedant in
https://github.com/llvm/llvm-project/issues/40056#issuecomment-981009129)

Fixes: #40056
---
 llvm/lib/Transforms/IPO/HotColdSplitting.cpp | 22 +++++-
 llvm/test/Transforms/HotColdSplit/pr40056.ll | 72 ++++++++++++++++++++
 2 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/HotColdSplit/pr40056.ll

diff --git a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
index 5aefcbf13182c..2ec5da4886839 100644
--- a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
+++ b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
@@ -39,6 +39,7 @@
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Dominators.h"
+#include "llvm/IR/EHPersonalities.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
@@ -168,10 +169,24 @@ static bool mayExtractBlock(const BasicBlock &BB) {
   //
   // Resumes that are not reachable from a cleanup landing pad are considered to
   // be unreachable. It’s not safe to split them out either.
+
   if (BB.hasAddressTaken() || BB.isEHPad())
     return false;
   auto Term = BB.getTerminator();
-  return !isa<InvokeInst>(Term) && !isa<ResumeInst>(Term);
+  if (isa<InvokeInst>(Term) || isa<ResumeInst>(Term))
+    return false;
+
+  // Do not outline basic blocks that have token type instructions. e.g.,
+  // exception:
+  // %0 = cleanuppad within none []
+  // call void @"?terminate@@YAXXZ"() [ "funclet"(token %0) ]
+  // br label %continue-exception
+  if (llvm::any_of(
+          BB, [](const Instruction &I) { return I.getType()->isTokenTy(); })) {
+    return false;
+  }
+
+  return true;
 }
 
 /// Mark \p F cold. Based on this assumption, also optimize it for minimum size.
@@ -258,6 +273,11 @@ bool HotColdSplitting::shouldOutlineFrom(const Function &F) const {
       F.hasFnAttribute(Attribute::SanitizeMemory))
     return false;
 
+  // Do not outline scoped EH personality functions.
+  if (F.hasPersonalityFn())
+    if (isScopedEHPersonality(classifyEHPersonality(F.getPersonalityFn())))
+      return false;
+
   return true;
 }
 
diff --git a/llvm/test/Transforms/HotColdSplit/pr40056.ll b/llvm/test/Transforms/HotColdSplit/pr40056.ll
new file mode 100644
index 0000000000000..950b62c673fbf
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/pr40056.ll
@@ -0,0 +1,72 @@
+; RUN: opt -passes=hotcoldsplit -hotcoldsplit-threshold=-1 -S < %s | FileCheck %s
+; Hot cold splitting should not outline:
+; 1. Basic blocks with token type instructions
+; 2. Functions with scoped EH personality
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.0"
+
+; CHECK-LABEL: define {{.*}}@with_funclet
+; CHECK-NOT: with_funclet.cold
+define void @with_funclet() personality ptr @__CxxFrameHandler3 {
+entry:
+  invoke void @fYAXXZ()
+          to label %normal unwind label %exception
+
+normal:                                           ; preds = %entry
+  ret void
+
+exception:                                        ; preds = %entry
+  %0 = cleanuppad within none []
+  call void @terminateYAXXZ() [ "funclet"(token %0) ]
+  br label %continueexception
+
+continueexception:                                ; preds = %exception
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@with_personality
+; CHECK-NOT: with_personality.cold
+define void @with_personality(i32 %cond) personality ptr @__CxxFrameHandler3 {
+entry:
+  %cond.addr = alloca i32
+  store i32 %cond, ptr %cond.addr
+  %0 = load i32, ptr %cond.addr
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.then, label %if.end2
+
+if.then:                                          ; preds = %entry
+  %1 = load i32, ptr %cond.addr
+  %cmp = icmp sgt i32 %1, 10
+  br i1 %cmp, label %if.then1, label %if.else
+
+if.then1:                                         ; preds = %if.then
+  call void @sideeffect(i32 0)
+  br label %if.end
+
+if.else:                                          ; preds = %if.then
+  call void @sideeffect(i32 1)
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then1
+  call void (...) @sink()
+  ret void
+
+if.end2:                                          ; preds = %entry
+  call void @sideeffect(i32 2)
+  ret void
+}
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare void @fYAXXZ()
+
+declare void @bar() #0
+
+declare void @terminateYAXXZ()
+
+declare void @sideeffect(i32)
+
+declare void @sink(...) #0
+
+attributes #0 = { cold }

From 1492e5f1d5a6d302f32ab95910c75b960b611128 Mon Sep 17 00:00:00 2001
From: dyung <douglas.yung@sony.com>
Date: Sat, 20 Jul 2024 12:08:35 -0700
Subject: [PATCH 302/486] Replace distutils.version with looseversion since the
 former was deprecated in python 3.10 and removed in 3.12. (#99549)

Python deprecated the distutils package in 3.10, and removed it in 3.12
causing problems when trying to run the lit tests with 3.12.

https://docs.python.org/3.10/library/distutils.html

Replace usage with the looseversion package which should be a drop-in
replacement for the original usage.

If your testing fails after this commit, you need to install the looseversion package.
---
 cross-project-tests/lit.cfg.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cross-project-tests/lit.cfg.py b/cross-project-tests/lit.cfg.py
index 774c4eaf4d976..47062866fbc26 100644
--- a/cross-project-tests/lit.cfg.py
+++ b/cross-project-tests/lit.cfg.py
@@ -4,8 +4,7 @@
 import subprocess
 import sys
 
-# TODO: LooseVersion is undocumented; use something else.
-from distutils.version import LooseVersion
+from looseversion import LooseVersion
 
 import lit.formats
 import lit.util

From 98c0e55d9d31ea0a7b8a1e5395257f10a6a27cc6 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Sat, 20 Jul 2024 12:20:50 -0700
Subject: [PATCH 303/486] Revert "SelectionDAG: Avoid using
 MachineFunction::getMMI" (#99777)

Reverts llvm/llvm-project#99696

https://lab.llvm.org/buildbot/#/builders/164/builds/1262
---
 llvm/include/llvm/CodeGen/SelectionDAG.h          |  9 +++------
 llvm/include/llvm/CodeGen/SelectionDAGISel.h      |  1 -
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp    |  3 +--
 .../CodeGen/SelectionDAG/SelectionDAGBuilder.cpp  |  6 +++---
 .../lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 15 ++++++---------
 .../unittests/CodeGen/AArch64SelectionDAGTest.cpp |  3 +--
 .../CodeGen/SelectionDAGAddressAnalysisTest.cpp   |  3 +--
 .../CodeGen/SelectionDAGPatternMatchTest.cpp      |  3 +--
 8 files changed, 16 insertions(+), 27 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 24eab7b408675..16ec65f2e7daa 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -245,7 +245,6 @@ class SelectionDAG {
 
   ProfileSummaryInfo *PSI = nullptr;
   BlockFrequencyInfo *BFI = nullptr;
-  MachineModuleInfo *MMI = nullptr;
 
   /// List of non-single value types.
   FoldingSet<SDVTListNode> VTListMap;
@@ -460,15 +459,14 @@ class SelectionDAG {
   void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
             Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
             UniformityInfo *UA, ProfileSummaryInfo *PSIin,
-            BlockFrequencyInfo *BFIin, MachineModuleInfo &MMI,
-            FunctionVarLocs const *FnVarLocs);
+            BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs);
 
   void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
             MachineFunctionAnalysisManager &AM,
             const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA,
             ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin,
-            MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs) {
-    init(NewMF, NewORE, nullptr, LibraryInfo, UA, PSIin, BFIin, MMI, FnVarLocs);
+            FunctionVarLocs const *FnVarLocs) {
+    init(NewMF, NewORE, nullptr, LibraryInfo, UA, PSIin, BFIin, FnVarLocs);
     MFAM = &AM;
   }
 
@@ -502,7 +500,6 @@ class SelectionDAG {
   OptimizationRemarkEmitter &getORE() const { return *ORE; }
   ProfileSummaryInfo *getPSI() const { return PSI; }
   BlockFrequencyInfo *getBFI() const { return BFI; }
-  MachineModuleInfo *getMMI() const { return MMI; }
 
   FlagInserter *getFlagInserter() { return Inserter; }
   void setFlagInserter(FlagInserter *FI) { Inserter = FI; }
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index fc0590b1a1b69..aa0efa5d9bf5d 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -48,7 +48,6 @@ class SelectionDAGISel {
   std::unique_ptr<FunctionLoweringInfo> FuncInfo;
   SwiftErrorValueTracking *SwiftError;
   MachineFunction *MF;
-  MachineModuleInfo *MMI;
   MachineRegisterInfo *RegInfo;
   SelectionDAG *CurDAG;
   std::unique_ptr<SelectionDAGBuilder> SDB;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index a1376aaaf12a9..02d44cd36ae53 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1333,7 +1333,7 @@ void SelectionDAG::init(MachineFunction &NewMF,
                         OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
                         const TargetLibraryInfo *LibraryInfo,
                         UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
-                        BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
+                        BlockFrequencyInfo *BFIin,
                         FunctionVarLocs const *VarLocs) {
   MF = &NewMF;
   SDAGISelPass = PassPtr;
@@ -1345,7 +1345,6 @@ void SelectionDAG::init(MachineFunction &NewMF,
   UA = NewUA;
   PSI = PSIin;
   BFI = BFIin;
-  MMI = &MMIin;
   FnVarLocs = VarLocs;
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 02eb1305b24ae..98a795edb7a03 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6707,11 +6707,11 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                              getValue(I.getArgOperand(0))));
     return;
   case Intrinsic::eh_sjlj_callsite: {
+    MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
-    assert(DAG.getMMI()->getCurrentCallSite() == 0 &&
-           "Overlapping call sites!");
+    assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
 
-    DAG.getMMI()->setCurrentCallSite(CI->getZExtValue());
+    MMI.setCurrentCallSite(CI->getZExtValue());
     return;
   }
   case Intrinsic::eh_sjlj_functioncontext: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 2670ff488fbed..df3d207d85d35 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -509,7 +509,7 @@ void SelectionDAGISel::initializeAnalysisResults(
     FnVarLocs = &FAM.getResult<DebugAssignmentTrackingAnalysis>(Fn);
 
   auto *UA = FAM.getCachedResult<UniformityInfoAnalysis>(Fn);
-  CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, *MMI, FnVarLocs);
+  CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, FnVarLocs);
 
   // Now get the optional analyzes if we want to.
   // This is based on the possibly changed OptLevel (after optnone is taken
@@ -562,11 +562,7 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
   UniformityInfo *UA = nullptr;
   if (auto *UAPass = MFP.getAnalysisIfAvailable<UniformityInfoWrapperPass>())
     UA = &UAPass->getUniformityInfo();
-
-  MachineModuleInfo &MMI =
-      MFP.getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
-
-  CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, MMI, FnVarLocs);
+  CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, FnVarLocs);
 
   // Now get the optional analyzes if we want to.
   // This is based on the possibly changed OptLevel (after optnone is taken
@@ -800,7 +796,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
   }
 
   // Determine if floating point is used for msvc
-  computeUsesMSVCFloatingPoint(TM.getTargetTriple(), Fn, *CurDAG->getMMI());
+  computeUsesMSVCFloatingPoint(TM.getTargetTriple(), Fn, MF->getMMI());
 
   // Release function-specific state. SDB and CurDAG are already cleared
   // at this point.
@@ -1447,6 +1443,7 @@ bool SelectionDAGISel::PrepareEHLandingPad() {
 
 // Mark and Report IPToState for each Block under IsEHa
 void SelectionDAGISel::reportIPToStateForBlocks(MachineFunction *MF) {
+  MachineModuleInfo &MMI = MF->getMMI();
   llvm::WinEHFuncInfo *EHInfo = MF->getWinEHFuncInfo();
   if (!EHInfo)
     return;
@@ -1461,8 +1458,8 @@ void SelectionDAGISel::reportIPToStateForBlocks(MachineFunction *MF) {
         continue;
 
       // Insert EH Labels
-      MCSymbol *BeginLabel = MF->getContext().createTempSymbol();
-      MCSymbol *EndLabel = MF->getContext().createTempSymbol();
+      MCSymbol *BeginLabel = MMI.getContext().createTempSymbol();
+      MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
       EHInfo->addIPToStateRange(State, BeginLabel, EndLabel);
       BuildMI(MBB, MBBb, SDB->getCurDebugLoc(),
               TII->get(TargetOpcode::EH_LABEL))
diff --git a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
index 877793a28b769..cc574b5887efc 100644
--- a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
+++ b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
@@ -68,8 +68,7 @@ class AArch64SelectionDAGTest : public testing::Test {
     if (!DAG)
       report_fatal_error("DAG?");
     OptimizationRemarkEmitter ORE(F);
-    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
-              nullptr);
+    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr);
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {
diff --git a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
index ef30bd491d352..88c4dbc2eec78 100644
--- a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
@@ -78,8 +78,7 @@ class SelectionDAGAddressAnalysisTest : public testing::Test {
     if (!DAG)
       report_fatal_error("DAG?");
     OptimizationRemarkEmitter ORE(F);
-    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
-              nullptr);
+    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr);
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {
diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index e318f467bbf27..a3d5e5f94b610 100644
--- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -77,8 +77,7 @@ class SelectionDAGPatternMatchTest : public testing::Test {
     if (!DAG)
       report_fatal_error("DAG?");
     OptimizationRemarkEmitter ORE(F);
-    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
-              nullptr);
+    DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr);
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {

From ce1a87437cc143889665c41046107e84cdf6246e Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sat, 20 Jul 2024 12:22:50 -0700
Subject: [PATCH 304/486] [clang-format] Fix a bug in annotating `*` in
 `#define`s (#99433)

Fixes #99271.
---
 clang/lib/Format/TokenAnnotator.cpp           | 19 +++++++++++++-----
 clang/unittests/Format/TokenAnnotatorTest.cpp | 20 +++++++++++++++++++
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index b6d6e52ccb8f8..db66911f00f63 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -372,10 +372,6 @@ class AnnotatingParser {
                OpeningParen.Previous->is(tok::kw__Generic)) {
       Contexts.back().ContextType = Context::C11GenericSelection;
       Contexts.back().IsExpression = true;
-    } else if (Line.InPPDirective &&
-               (!OpeningParen.Previous ||
-                OpeningParen.Previous->isNot(tok::identifier))) {
-      Contexts.back().IsExpression = true;
     } else if (Contexts[Contexts.size() - 2].CaretFound) {
       // This is the parameter list of an ObjC block.
       Contexts.back().IsExpression = false;
@@ -388,7 +384,20 @@ class AnnotatingParser {
                OpeningParen.Previous->MatchingParen->isOneOf(
                    TT_ObjCBlockLParen, TT_FunctionTypeLParen)) {
       Contexts.back().IsExpression = false;
-    } else if (!Line.MustBeDeclaration && !Line.InPPDirective) {
+    } else if (Line.InPPDirective) {
+      auto IsExpr = [&OpeningParen] {
+        const auto *Tok = OpeningParen.Previous;
+        if (!Tok || Tok->isNot(tok::identifier))
+          return true;
+        Tok = Tok->Previous;
+        while (Tok && Tok->endsSequence(tok::coloncolon, tok::identifier)) {
+          assert(Tok->Previous);
+          Tok = Tok->Previous->Previous;
+        }
+        return !Tok || !Tok->Tok.getIdentifierInfo();
+      };
+      Contexts.back().IsExpression = IsExpr();
+    } else if (!Line.MustBeDeclaration) {
       bool IsForOrCatch =
           OpeningParen.Previous &&
           OpeningParen.Previous->isOneOf(tok::kw_for, tok::kw_catch);
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index c5e8aa72cd2cb..f70424c3ee060 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -75,6 +75,26 @@ TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
   EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
 
+  Tokens = annotate("#define FOO bar(a * b)");
+  ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[6], tok::star, TT_BinaryOperator);
+
+  Tokens = annotate("#define FOO foo.bar(a & b)");
+  ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_TOKEN(Tokens[8], tok::amp, TT_BinaryOperator);
+
+  Tokens = annotate("#define FOO foo::bar(a && b)");
+  ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_BinaryOperator);
+
+  Tokens = annotate("#define FOO foo bar(a *b)");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[7], tok::star, TT_PointerOrReference);
+
+  Tokens = annotate("#define FOO void foo::bar(a &b)");
+  ASSERT_EQ(Tokens.size(), 13u) << Tokens;
+  EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
+
   Tokens = annotate("void f() {\n"
                     "  while (p < a && *p == 'a')\n"
                     "    p++;\n"

From cc2fb58639a6b87f155e6052f5ef2bbe05d5c378 Mon Sep 17 00:00:00 2001
From: Dmitriy Chestnykh <dm.chestnykh@gmail.com>
Date: Sat, 20 Jul 2024 22:33:39 +0300
Subject: [PATCH 305/486] [MC,ELF] Use loc from the directive for `.abort`
 (#99648)

---
 llvm/lib/MC/MCParser/AsmParser.cpp       | 18 +++++++-----------
 llvm/test/MC/AsmParser/directive_abort.s | 13 ++++++++-----
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index d05712bca73cd..992b69f1c5f32 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -658,7 +658,7 @@ class AsmParser : public MCAsmParser {
 
   bool parseDirectiveComm(bool IsLocal); // ".comm" and ".lcomm"
 
-  bool parseDirectiveAbort(); // ".abort"
+  bool parseDirectiveAbort(SMLoc DirectiveLoc); // ".abort"
   bool parseDirectiveInclude(); // ".include"
   bool parseDirectiveIncbin(); // ".incbin"
 
@@ -2120,7 +2120,7 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
     case DK_LCOMM:
       return parseDirectiveComm(/*IsLocal=*/true);
     case DK_ABORT:
-      return parseDirectiveAbort();
+      return parseDirectiveAbort(IDLoc);
     case DK_INCLUDE:
       return parseDirectiveInclude();
     case DK_INCBIN:
@@ -5095,21 +5095,17 @@ bool AsmParser::parseDirectiveComm(bool IsLocal) {
 
 /// parseDirectiveAbort
 ///  ::= .abort [... message ...]
-bool AsmParser::parseDirectiveAbort() {
-  // FIXME: Use loc from directive.
-  SMLoc Loc = getLexer().getLoc();
-
+bool AsmParser::parseDirectiveAbort(SMLoc DirectiveLoc) {
   StringRef Str = parseStringToEndOfStatement();
   if (parseEOL())
     return true;
 
   if (Str.empty())
-    return Error(Loc, ".abort detected. Assembly stopping.");
-  else
-    return Error(Loc, ".abort '" + Str + "' detected. Assembly stopping.");
-  // FIXME: Actually abort assembly here.
+    return Error(DirectiveLoc, ".abort detected. Assembly stopping");
 
-  return false;
+  // FIXME: Actually abort assembly here.
+  return Error(DirectiveLoc,
+               ".abort '" + Str + "' detected. Assembly stopping");
 }
 
 /// parseDirectiveInclude
diff --git a/llvm/test/MC/AsmParser/directive_abort.s b/llvm/test/MC/AsmParser/directive_abort.s
index 86e6267a7a1eb..f4dda229a017b 100644
--- a/llvm/test/MC/AsmParser/directive_abort.s
+++ b/llvm/test/MC/AsmParser/directive_abort.s
@@ -1,6 +1,9 @@
-# RUN: not llvm-mc -triple i386-unknown-unknown %s 2> %t
-# RUN: FileCheck -input-file %t %s
+// RUN: not llvm-mc -filetype=obj -triple x86_64 %s 2>&1 -o /dev/null | FileCheck %s
 
-# CHECK: error: .abort 'please stop assembing'
-TEST0:
-	.abort       please stop assembing
+.abort
+// CHECK:      [[#@LINE-1]]:1: error: .abort detected. Assembly stopping
+// CHECK-NEXT: abort
+
+.abort "abort message"
+// CHECK:      [[#@LINE-1]]:1: error: .abort '"abort message"' detected. Assembly stopping
+// CHECK-NEXT: abort

From 569814e862a7ce1de3144e76d0a97253ac161d05 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Sat, 20 Jul 2024 22:27:39 +0200
Subject: [PATCH 306/486] [libc] Implement pwait2 using pwait (#99781)

This patch implements pwait2 using pwait. The implementation is an
approximation of pwait2, since pwait only only supports timeouts in
milliseconds, not nanoseconds, as required by pwait2.
---
 libc/src/sys/epoll/linux/epoll_pwait2.cpp | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/libc/src/sys/epoll/linux/epoll_pwait2.cpp b/libc/src/sys/epoll/linux/epoll_pwait2.cpp
index 14b419399fe9b..4123157d29fff 100644
--- a/libc/src/sys/epoll/linux/epoll_pwait2.cpp
+++ b/libc/src/sys/epoll/linux/epoll_pwait2.cpp
@@ -25,10 +25,22 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(int, epoll_pwait2,
                    (int epfd, struct epoll_event *events, int maxevents,
                     const struct timespec *timeout, const sigset_t *sigmask)) {
+#ifdef SYS_epoll_pwait2
   int ret = LIBC_NAMESPACE::syscall_impl<int>(
       SYS_epoll_pwait2, epfd, reinterpret_cast<long>(events), maxevents,
       reinterpret_cast<long>(timeout), reinterpret_cast<long>(sigmask),
       NSIG / 8);
+#elif defined(SYS_epoll_pwait)
+  // Convert nanoseconds to milliseconds, rounding up if there are remaining
+  // nanoseconds
+  long timeout_ms = static_cast<long>(timeout->tv_sec * 1000 +
+                                      (timeout->tv_nsec + 999999) / 1000000);
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(
+      SYS_epoll_pwait, epfd, reinterpret_cast<long>(events), maxevents,
+      timeout_ms, reinterpret_cast<long>(sigmask), NSIG / 8);
+#else
+#error "epoll_pwait and epoll_pwait2 syscalls not available."
+#endif
 
   // A negative return value indicates an error with the magnitude of the
   // value being the error code.

From 1f00c4244602e66f98a608bd20d6fb62f11079de Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 20 Jul 2024 21:41:03 +0100
Subject: [PATCH 307/486] [VPlan] Assert masked interleave accesses are allowed
 if needed (NFC)

Add assertion at interleave group construction.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6b050c138552b..86e4149cf27a6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8983,6 +8983,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
 
     bool NeedsMaskForGaps =
         IG->requiresScalarEpilogue() && !CM.isScalarEpilogueAllowed();
+    assert((!NeedsMaskForGaps || useMaskedInterleavedAccesses(CM.TTI)) &&
+           "masked interleaved groups are not allowed.");
     auto *VPIG = new VPInterleaveRecipe(IG, Recipe->getAddr(), StoredValues,
                                         Recipe->getMask(), NeedsMaskForGaps);
     VPIG->insertBefore(Recipe);

From 28045ceab08d41a8a42d93ebc445e8fe906f884c Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 14:13:02 -0700
Subject: [PATCH 308/486] [ELF] Support (TYPE=<value>) beside output section
 address

Support `preinit_array . (TYPE=SHT_PREINIT_ARRAY) : { QUAD(16) }`

Follow-up to https://reviews.llvm.org/D118840

peek2() could be eliminated by a future change.
---
 lld/ELF/ScriptParser.cpp                      | 28 ++++++++++---------
 .../ELF/linkerscript/custom-section-type.s    |  2 +-
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index 47a94c29ea496..92f4bb41cd68f 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -92,7 +92,7 @@ class ScriptParser final : ScriptLexer {
   SymbolAssignment *readSymbolAssignment(StringRef name);
   ByteCommand *readByteCommand(StringRef tok);
   std::array<uint8_t, 4> readFill();
-  bool readSectionDirective(OutputSection *cmd, StringRef tok1, StringRef tok2);
+  bool readSectionDirective(OutputSection *cmd, StringRef tok2);
   void readSectionAddressType(OutputSection *cmd);
   OutputDesc *readOverlaySectionDescription();
   OutputDesc *readOutputSectionDescription(StringRef outSec);
@@ -875,9 +875,7 @@ constexpr std::pair<const char *, unsigned> typeMap[] = {
 // "(TYPE=<value>)".
 // Tok1 and Tok2 are next 2 tokens peeked. See comment for
 // readSectionAddressType below.
-bool ScriptParser::readSectionDirective(OutputSection *cmd, StringRef tok1, StringRef tok2) {
-  if (tok1 != "(")
-    return false;
+bool ScriptParser::readSectionDirective(OutputSection *cmd, StringRef tok2) {
   if (tok2 != "NOLOAD" && tok2 != "COPY" && tok2 != "INFO" &&
       tok2 != "OVERLAY" && tok2 != "TYPE")
     return false;
@@ -921,16 +919,20 @@ bool ScriptParser::readSectionDirective(OutputSection *cmd, StringRef tok1, Stri
 // https://sourceware.org/binutils/docs/ld/Output-Section-Address.html
 // https://sourceware.org/binutils/docs/ld/Output-Section-Type.html
 void ScriptParser::readSectionAddressType(OutputSection *cmd) {
-  // Temporarily set inExpr to support TYPE=<value> without spaces.
-  bool saved = std::exchange(inExpr, true);
-  bool isDirective = readSectionDirective(cmd, peek(), peek2());
-  inExpr = saved;
-  if (isDirective)
-    return;
-
+  if (peek() == "(") {
+    // Temporarily set inExpr to support TYPE=<value> without spaces.
+    SaveAndRestore saved(inExpr, true);
+    if (readSectionDirective(cmd, peek2()))
+      return;
+  }
   cmd->addrExpr = readExpr();
-  if (peek() == "(" && !readSectionDirective(cmd, "(", peek2()))
-    setError("unknown section directive: " + peek2());
+
+  if (peek() == "(") {
+    SaveAndRestore saved(inExpr, true);
+    StringRef tok2 = peek2();
+    if (!readSectionDirective(cmd, tok2))
+      setError("unknown section directive: " + tok2);
+  }
 }
 
 static Expr checkAlignment(Expr e, std::string &loc) {
diff --git a/lld/test/ELF/linkerscript/custom-section-type.s b/lld/test/ELF/linkerscript/custom-section-type.s
index 8ca0a4db325bd..2add3a52f8117 100644
--- a/lld/test/ELF/linkerscript/custom-section-type.s
+++ b/lld/test/ELF/linkerscript/custom-section-type.s
@@ -67,7 +67,7 @@ SECTIONS {
   nobits ( TYPE=SHT_NOBITS) : { BYTE(8) }
   init_array (TYPE=SHT_INIT_ARRAY ) : { QUAD(myinit) }
   fini_array (TYPE=SHT_FINI_ARRAY) : { QUAD(15) }
-  preinit_array (TYPE=SHT_PREINIT_ARRAY) : { QUAD(16) }
+  preinit_array . (TYPE=SHT_PREINIT_ARRAY) : { QUAD(16) }
   group (TYPE=17) : { LONG(17) }
   expr (TYPE=0x41+1) : { BYTE(0x42) *(expr) }
 }

From 52a46bc31de238f9d5ee3a8be2388781b2af56e4 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Sat, 20 Jul 2024 14:22:05 -0700
Subject: [PATCH 309/486] [SandboxIR] Implement StoreInst (#99707)

This patch adds the SandboxIR StoreInst instruction which mirrors
llvm::StoreInst.
---
 llvm/include/llvm/SandboxIR/SandboxIR.h       | 52 ++++++++++++++++--
 .../llvm/SandboxIR/SandboxIRValues.def        |  1 +
 llvm/lib/SandboxIR/SandboxIR.cpp              | 54 +++++++++++++++++++
 llvm/unittests/SandboxIR/SandboxIRTest.cpp    | 33 ++++++++++++
 4 files changed, 135 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index f168fdf8b1056..dfffe5c96f1cf 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -76,6 +76,7 @@ class Context;
 class Function;
 class Instruction;
 class LoadInst;
+class StoreInst;
 class User;
 class Value;
 
@@ -172,10 +173,11 @@ class Value {
   /// order.
   llvm::Value *Val = nullptr;
 
-  friend class Context;  // For getting `Val`.
-  friend class User;     // For getting `Val`.
-  friend class Use;      // For getting `Val`.
-  friend class LoadInst; // For getting `Val`.
+  friend class Context;   // For getting `Val`.
+  friend class User;      // For getting `Val`.
+  friend class Use;       // For getting `Val`.
+  friend class LoadInst;  // For getting `Val`.
+  friend class StoreInst; // For getting `Val`.
 
   /// All values point to the context.
   Context &Ctx;
@@ -495,7 +497,8 @@ class Instruction : public sandboxir::User {
   /// A SandboxIR Instruction may map to multiple LLVM IR Instruction. This
   /// returns its topmost LLVM IR instruction.
   llvm::Instruction *getTopmostLLVMInstruction() const;
-  friend class LoadInst; // For getTopmostLLVMInstruction().
+  friend class LoadInst;  // For getTopmostLLVMInstruction().
+  friend class StoreInst; // For getTopmostLLVMInstruction().
 
   /// \Returns the LLVM IR Instructions that this SandboxIR maps to in program
   /// order.
@@ -599,6 +602,43 @@ class LoadInst final : public Instruction {
 #endif
 };
 
+class StoreInst final : public Instruction {
+  /// Use StoreInst::create().
+  StoreInst(llvm::StoreInst *SI, Context &Ctx)
+      : Instruction(ClassID::Store, Opcode::Store, SI, Ctx) {}
+  friend Context; // for StoreInst()
+  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
+    return getOperandUseDefault(OpIdx, Verify);
+  }
+  SmallVector<llvm::Instruction *, 1> getLLVMInstrs() const final {
+    return {cast<llvm::Instruction>(Val)};
+  }
+
+public:
+  unsigned getUseOperandNo(const Use &Use) const final {
+    return getUseOperandNoDefault(Use);
+  }
+  unsigned getNumOfIRInstrs() const final { return 1u; }
+  static StoreInst *create(Value *V, Value *Ptr, MaybeAlign Align,
+                           Instruction *InsertBefore, Context &Ctx);
+  static StoreInst *create(Value *V, Value *Ptr, MaybeAlign Align,
+                           BasicBlock *InsertAtEnd, Context &Ctx);
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From);
+  Value *getValueOperand() const;
+  Value *getPointerOperand() const;
+  Align getAlign() const { return cast<llvm::StoreInst>(Val)->getAlign(); }
+  bool isSimple() const { return cast<llvm::StoreInst>(Val)->isSimple(); }
+  bool isUnordered() const { return cast<llvm::StoreInst>(Val)->isUnordered(); }
+#ifndef NDEBUG
+  void verify() const final {
+    assert(isa<llvm::StoreInst>(Val) && "Expected StoreInst!");
+  }
+  void dump(raw_ostream &OS) const override;
+  LLVM_DUMP_METHOD void dump() const override;
+#endif
+};
+
 /// An LLLVM Instruction that has no SandboxIR equivalent class gets mapped to
 /// an OpaqueInstr.
 class OpaqueInst : public sandboxir::Instruction {
@@ -734,6 +774,8 @@ class Context {
 
   LoadInst *createLoadInst(llvm::LoadInst *LI);
   friend LoadInst; // For createLoadInst()
+  StoreInst *createStoreInst(llvm::StoreInst *SI);
+  friend StoreInst; // For createStoreInst()
 
 public:
   Context(LLVMContext &LLVMCtx)
diff --git a/llvm/include/llvm/SandboxIR/SandboxIRValues.def b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
index e1ed3cdac6bba..90365ca7a1c45 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIRValues.def
+++ b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
@@ -26,6 +26,7 @@ DEF_USER(Constant, Constant)
 //       ClassID, Opcode(s),  Class
 DEF_INSTR(Opaque, OP(Opaque), OpaqueInst)
 DEF_INSTR(Load, OP(Load), LoadInst)
+DEF_INSTR(Store, OP(Store), StoreInst)
 
 #ifdef DEF_VALUE
 #undef DEF_VALUE
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index d49ec0c91e599..209b677bafbb5 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -496,6 +496,50 @@ void LoadInst::dump() const {
   dump(dbgs());
   dbgs() << "\n";
 }
+#endif // NDEBUG
+StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
+                             Instruction *InsertBefore, Context &Ctx) {
+  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(BeforeIR);
+  auto *NewSI =
+      Builder.CreateAlignedStore(V->Val, Ptr->Val, Align, /*isVolatile=*/false);
+  auto *NewSBI = Ctx.createStoreInst(NewSI);
+  return NewSBI;
+}
+StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
+                             BasicBlock *InsertAtEnd, Context &Ctx) {
+  auto *InsertAtEndIR = cast<llvm::BasicBlock>(InsertAtEnd->Val);
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(InsertAtEndIR);
+  auto *NewSI =
+      Builder.CreateAlignedStore(V->Val, Ptr->Val, Align, /*isVolatile=*/false);
+  auto *NewSBI = Ctx.createStoreInst(NewSI);
+  return NewSBI;
+}
+
+bool StoreInst::classof(const Value *From) {
+  return From->getSubclassID() == ClassID::Store;
+}
+
+Value *StoreInst::getValueOperand() const {
+  return Ctx.getValue(cast<llvm::StoreInst>(Val)->getValueOperand());
+}
+
+Value *StoreInst::getPointerOperand() const {
+  return Ctx.getValue(cast<llvm::StoreInst>(Val)->getPointerOperand());
+}
+
+#ifndef NDEBUG
+void StoreInst::dump(raw_ostream &OS) const {
+  dumpCommonPrefix(OS);
+  dumpCommonSuffix(OS);
+}
+
+void StoreInst::dump() const {
+  dump(dbgs());
+  dbgs() << "\n";
+}
 
 void OpaqueInst::dump(raw_ostream &OS) const {
   dumpCommonPrefix(OS);
@@ -619,6 +663,11 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
     It->second = std::unique_ptr<LoadInst>(new LoadInst(LLVMLd, *this));
     return It->second.get();
   }
+  case llvm::Instruction::Store: {
+    auto *LLVMSt = cast<llvm::StoreInst>(LLVMV);
+    It->second = std::unique_ptr<StoreInst>(new StoreInst(LLVMSt, *this));
+    return It->second.get();
+  }
   default:
     break;
   }
@@ -642,6 +691,11 @@ LoadInst *Context::createLoadInst(llvm::LoadInst *LI) {
   return cast<LoadInst>(registerValue(std::move(NewPtr)));
 }
 
+StoreInst *Context::createStoreInst(llvm::StoreInst *SI) {
+  auto NewPtr = std::unique_ptr<StoreInst>(new StoreInst(SI, *this));
+  return cast<StoreInst>(registerValue(std::move(NewPtr)));
+}
+
 Value *Context::getValue(llvm::Value *V) const {
   auto It = LLVMValueToValueMap.find(V);
   if (It != LLVMValueToValueMap.end())
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 04beb429502bc..054a81e9cf308 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -591,3 +591,36 @@ define void @foo(ptr %arg0, ptr %arg1) {
   EXPECT_EQ(NewLd->getAlign(), 8);
   EXPECT_EQ(NewLd->getName(), "NewLd");
 }
+
+TEST_F(SandboxIRTest, StoreInst) {
+  parseIR(C, R"IR(
+define void @foo(i8 %val, ptr %ptr) {
+  store i8 %val, ptr %ptr, align 64
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  sandboxir::Function *F = Ctx.createFunction(LLVMF);
+  auto *Val = F->getArg(0);
+  auto *Ptr = F->getArg(1);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *St = cast<sandboxir::StoreInst>(&*It++);
+  auto *Ret = &*It++;
+
+  // Check that the StoreInst has been created correctly.
+  // Check getPointerOperand()
+  EXPECT_EQ(St->getValueOperand(), Val);
+  EXPECT_EQ(St->getPointerOperand(), Ptr);
+  // Check getAlign()
+  EXPECT_EQ(St->getAlign(), 64);
+  // Check create(InsertBefore)
+  sandboxir::StoreInst *NewSt =
+      sandboxir::StoreInst::create(Val, Ptr, Align(8),
+                                   /*InsertBefore=*/Ret, Ctx);
+  EXPECT_EQ(NewSt->getType(), St->getType());
+  EXPECT_EQ(NewSt->getValueOperand(), Val);
+  EXPECT_EQ(NewSt->getPointerOperand(), Ptr);
+  EXPECT_EQ(NewSt->getAlign(), 8);
+}

From a23efcc703ce2a4ac534af4fcc2b38765cac9f8f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 20 Jul 2024 22:23:01 +0100
Subject: [PATCH 310/486] [VPlan] Move VPInterleaveRecipe::execute to
 VPlanRecipes.cpp (NFC).

Move ::exeute and ::print to VPlanRecipes.cpp in line with other recipe
definitions.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 383 ------------------
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 367 +++++++++++++++++
 2 files changed, 367 insertions(+), 383 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 86e4149cf27a6..6d28b8fabe42e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -441,37 +441,6 @@ static std::optional<unsigned> getSmallBestKnownTC(ScalarEvolution &SE,
   return std::nullopt;
 }
 
-/// Return a vector containing interleaved elements from multiple
-/// smaller input vectors.
-static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
-                                const Twine &Name) {
-  unsigned Factor = Vals.size();
-  assert(Factor > 1 && "Tried to interleave invalid number of vectors");
-
-  VectorType *VecTy = cast<VectorType>(Vals[0]->getType());
-#ifndef NDEBUG
-  for (Value *Val : Vals)
-    assert(Val->getType() == VecTy && "Tried to interleave mismatched types");
-#endif
-
-  // Scalable vectors cannot use arbitrary shufflevectors (only splats), so
-  // must use intrinsics to interleave.
-  if (VecTy->isScalableTy()) {
-    VectorType *WideVecTy = VectorType::getDoubleElementsVectorType(VecTy);
-    return Builder.CreateIntrinsic(WideVecTy, Intrinsic::vector_interleave2,
-                                   Vals,
-                                   /*FMFSource=*/nullptr, Name);
-  }
-
-  // Fixed length. Start by concatenating all vectors into a wide vector.
-  Value *WideVec = concatenateVectors(Builder, Vals);
-
-  // Interleave the elements into the wide vector.
-  const unsigned NumElts = VecTy->getElementCount().getFixedValue();
-  return Builder.CreateShuffleVector(
-      WideVec, createInterleaveMask(NumElts, Factor), Name);
-}
-
 namespace {
 // Forward declare GeneratedRTChecks.
 class GeneratedRTChecks;
@@ -553,16 +522,6 @@ class InnerLoopVectorizer {
                             const VPIteration &Instance,
                             VPTransformState &State);
 
-  /// Try to vectorize interleaved access group \p Group with the base address
-  /// given in \p Addr, optionally masking the vector operations if \p
-  /// BlockInMask is non-null. Use \p State to translate given VPValues to IR
-  /// values in the vectorized loop.
-  void vectorizeInterleaveGroup(const InterleaveGroup<Instruction> *Group,
-                                ArrayRef<VPValue *> VPDefs,
-                                VPTransformState &State, VPValue *Addr,
-                                ArrayRef<VPValue *> StoredValues,
-                                VPValue *BlockInMask, bool NeedsMaskForGaps);
-
   /// Fix the non-induction PHIs in \p Plan.
   void fixNonInductionPHIs(VPlan &Plan, VPTransformState &State);
 
@@ -611,11 +570,6 @@ class InnerLoopVectorizer {
   /// Returns (and creates if needed) the trip count of the widened loop.
   Value *getOrCreateVectorTripCount(BasicBlock *InsertBlock);
 
-  /// Returns a bitcasted value to the requested vector type.
-  /// Also handles bitcasts of vector<float> <-> vector<pointer> types.
-  Value *createBitOrPointerCast(Value *V, VectorType *DstVTy,
-                                const DataLayout &DL);
-
   /// Emit a bypass check to see if the vector trip count is zero, including if
   /// it overflows.
   void emitIterationCountCheck(BasicBlock *Bypass);
@@ -2393,275 +2347,6 @@ static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
   return TTI.enableMaskedInterleavedAccessVectorization();
 }
 
-// Try to vectorize the interleave group that \p Instr belongs to.
-//
-// E.g. Translate following interleaved load group (factor = 3):
-//   for (i = 0; i < N; i+=3) {
-//     R = Pic[i];             // Member of index 0
-//     G = Pic[i+1];           // Member of index 1
-//     B = Pic[i+2];           // Member of index 2
-//     ... // do something to R, G, B
-//   }
-// To:
-//   %wide.vec = load <12 x i32>                       ; Read 4 tuples of R,G,B
-//   %R.vec = shuffle %wide.vec, poison, <0, 3, 6, 9>   ; R elements
-//   %G.vec = shuffle %wide.vec, poison, <1, 4, 7, 10>  ; G elements
-//   %B.vec = shuffle %wide.vec, poison, <2, 5, 8, 11>  ; B elements
-//
-// Or translate following interleaved store group (factor = 3):
-//   for (i = 0; i < N; i+=3) {
-//     ... do something to R, G, B
-//     Pic[i]   = R;           // Member of index 0
-//     Pic[i+1] = G;           // Member of index 1
-//     Pic[i+2] = B;           // Member of index 2
-//   }
-// To:
-//   %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
-//   %B_U.vec = shuffle %B.vec, poison, <0, 1, 2, 3, u, u, u, u>
-//   %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
-//        <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>    ; Interleave R,G,B elements
-//   store <12 x i32> %interleaved.vec              ; Write 4 tuples of R,G,B
-void InnerLoopVectorizer::vectorizeInterleaveGroup(
-    const InterleaveGroup<Instruction> *Group, ArrayRef<VPValue *> VPDefs,
-    VPTransformState &State, VPValue *Addr, ArrayRef<VPValue *> StoredValues,
-    VPValue *BlockInMask, bool NeedsMaskForGaps) {
-  Instruction *Instr = Group->getInsertPos();
-  const DataLayout &DL = Instr->getDataLayout();
-
-  // Prepare for the vector type of the interleaved load/store.
-  Type *ScalarTy = getLoadStoreType(Instr);
-  unsigned InterleaveFactor = Group->getFactor();
-  auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor);
-
-  // Prepare for the new pointers.
-  SmallVector<Value *, 2> AddrParts;
-  unsigned Index = Group->getIndex(Instr);
-
-  // TODO: extend the masked interleaved-group support to reversed access.
-  assert((!BlockInMask || !Group->isReverse()) &&
-         "Reversed masked interleave-group not supported.");
-
-  Value *Idx;
-  // If the group is reverse, adjust the index to refer to the last vector lane
-  // instead of the first. We adjust the index from the first vector lane,
-  // rather than directly getting the pointer for lane VF - 1, because the
-  // pointer operand of the interleaved access is supposed to be uniform. For
-  // uniform instructions, we're only required to generate a value for the
-  // first vector lane in each unroll iteration.
-  if (Group->isReverse()) {
-    Value *RuntimeVF = getRuntimeVF(Builder, Builder.getInt32Ty(), State.VF);
-    Idx = Builder.CreateSub(RuntimeVF, Builder.getInt32(1));
-    Idx = Builder.CreateMul(Idx, Builder.getInt32(Group->getFactor()));
-    Idx = Builder.CreateAdd(Idx, Builder.getInt32(Index));
-    Idx = Builder.CreateNeg(Idx);
-  } else
-    Idx = Builder.getInt32(-Index);
-
-  for (unsigned Part = 0; Part < State.UF; Part++) {
-    Value *AddrPart = State.get(Addr, VPIteration(Part, 0));
-    if (auto *I = dyn_cast<Instruction>(AddrPart))
-      State.setDebugLocFrom(I->getDebugLoc());
-
-    // Notice current instruction could be any index. Need to adjust the address
-    // to the member of index 0.
-    //
-    // E.g.  a = A[i+1];     // Member of index 1 (Current instruction)
-    //       b = A[i];       // Member of index 0
-    // Current pointer is pointed to A[i+1], adjust it to A[i].
-    //
-    // E.g.  A[i+1] = a;     // Member of index 1
-    //       A[i]   = b;     // Member of index 0
-    //       A[i+2] = c;     // Member of index 2 (Current instruction)
-    // Current pointer is pointed to A[i+2], adjust it to A[i].
-
-    bool InBounds = false;
-    if (auto *gep = dyn_cast<GetElementPtrInst>(AddrPart->stripPointerCasts()))
-      InBounds = gep->isInBounds();
-    AddrPart = Builder.CreateGEP(ScalarTy, AddrPart, Idx, "", InBounds);
-    AddrParts.push_back(AddrPart);
-  }
-
-  State.setDebugLocFrom(Instr->getDebugLoc());
-  Value *PoisonVec = PoisonValue::get(VecTy);
-
-  auto CreateGroupMask = [this, &BlockInMask, &State, &InterleaveFactor](
-                             unsigned Part, Value *MaskForGaps) -> Value * {
-    if (State.VF.isScalable()) {
-      assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");
-      assert(InterleaveFactor == 2 &&
-             "Unsupported deinterleave factor for scalable vectors");
-      auto *BlockInMaskPart = State.get(BlockInMask, Part);
-      SmallVector<Value *, 2> Ops = {BlockInMaskPart, BlockInMaskPart};
-      auto *MaskTy = VectorType::get(Builder.getInt1Ty(),
-                                     State.VF.getKnownMinValue() * 2, true);
-      return Builder.CreateIntrinsic(MaskTy, Intrinsic::vector_interleave2, Ops,
-                                     /*FMFSource=*/nullptr, "interleaved.mask");
-    }
-
-    if (!BlockInMask)
-      return MaskForGaps;
-
-    Value *BlockInMaskPart = State.get(BlockInMask, Part);
-    Value *ShuffledMask = Builder.CreateShuffleVector(
-        BlockInMaskPart,
-        createReplicatedMask(InterleaveFactor, State.VF.getKnownMinValue()),
-        "interleaved.mask");
-    return MaskForGaps ? Builder.CreateBinOp(Instruction::And, ShuffledMask,
-                                             MaskForGaps)
-                       : ShuffledMask;
-  };
-
-  // Vectorize the interleaved load group.
-  if (isa<LoadInst>(Instr)) {
-    Value *MaskForGaps = nullptr;
-    if (NeedsMaskForGaps) {
-      MaskForGaps =
-          createBitMaskForGaps(Builder, State.VF.getKnownMinValue(), *Group);
-      assert(MaskForGaps && "Mask for Gaps is required but it is null");
-    }
-
-    // For each unroll part, create a wide load for the group.
-    SmallVector<Value *, 2> NewLoads;
-    for (unsigned Part = 0; Part < State.UF; Part++) {
-      Instruction *NewLoad;
-      if (BlockInMask || MaskForGaps) {
-        assert(useMaskedInterleavedAccesses(*TTI) &&
-               "masked interleaved groups are not allowed.");
-        Value *GroupMask = CreateGroupMask(Part, MaskForGaps);
-        NewLoad =
-            Builder.CreateMaskedLoad(VecTy, AddrParts[Part], Group->getAlign(),
-                                     GroupMask, PoisonVec, "wide.masked.vec");
-      }
-      else
-        NewLoad = Builder.CreateAlignedLoad(VecTy, AddrParts[Part],
-                                            Group->getAlign(), "wide.vec");
-      Group->addMetadata(NewLoad);
-      NewLoads.push_back(NewLoad);
-    }
-
-    if (VecTy->isScalableTy()) {
-      assert(InterleaveFactor == 2 &&
-             "Unsupported deinterleave factor for scalable vectors");
-
-      for (unsigned Part = 0; Part < State.UF; ++Part) {
-        // Scalable vectors cannot use arbitrary shufflevectors (only splats),
-        // so must use intrinsics to deinterleave.
-        Value *DI = Builder.CreateIntrinsic(
-            Intrinsic::vector_deinterleave2, VecTy, NewLoads[Part],
-            /*FMFSource=*/nullptr, "strided.vec");
-        unsigned J = 0;
-        for (unsigned I = 0; I < InterleaveFactor; ++I) {
-          Instruction *Member = Group->getMember(I);
-
-          if (!Member)
-            continue;
-
-          Value *StridedVec = Builder.CreateExtractValue(DI, I);
-          // If this member has different type, cast the result type.
-          if (Member->getType() != ScalarTy) {
-            VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
-            StridedVec = createBitOrPointerCast(StridedVec, OtherVTy, DL);
-          }
-
-          if (Group->isReverse())
-            StridedVec = Builder.CreateVectorReverse(StridedVec, "reverse");
-
-          State.set(VPDefs[J], StridedVec, Part);
-          ++J;
-        }
-      }
-
-      return;
-    }
-
-    // For each member in the group, shuffle out the appropriate data from the
-    // wide loads.
-    unsigned J = 0;
-    for (unsigned I = 0; I < InterleaveFactor; ++I) {
-      Instruction *Member = Group->getMember(I);
-
-      // Skip the gaps in the group.
-      if (!Member)
-        continue;
-
-      auto StrideMask =
-          createStrideMask(I, InterleaveFactor, State.VF.getKnownMinValue());
-      for (unsigned Part = 0; Part < State.UF; Part++) {
-        Value *StridedVec = Builder.CreateShuffleVector(
-            NewLoads[Part], StrideMask, "strided.vec");
-
-        // If this member has different type, cast the result type.
-        if (Member->getType() != ScalarTy) {
-          assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
-          VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
-          StridedVec = createBitOrPointerCast(StridedVec, OtherVTy, DL);
-        }
-
-        if (Group->isReverse())
-          StridedVec = Builder.CreateVectorReverse(StridedVec, "reverse");
-
-        State.set(VPDefs[J], StridedVec, Part);
-      }
-      ++J;
-    }
-    return;
-  }
-
-  // The sub vector type for current instruction.
-  auto *SubVT = VectorType::get(ScalarTy, State.VF);
-
-  // Vectorize the interleaved store group.
-  Value *MaskForGaps =
-      createBitMaskForGaps(Builder, State.VF.getKnownMinValue(), *Group);
-  assert((!MaskForGaps || useMaskedInterleavedAccesses(*TTI)) &&
-         "masked interleaved groups are not allowed.");
-  assert((!MaskForGaps || !State.VF.isScalable()) &&
-         "masking gaps for scalable vectors is not yet supported.");
-  for (unsigned Part = 0; Part < State.UF; Part++) {
-    // Collect the stored vector from each member.
-    SmallVector<Value *, 4> StoredVecs;
-    unsigned StoredIdx = 0;
-    for (unsigned i = 0; i < InterleaveFactor; i++) {
-      assert((Group->getMember(i) || MaskForGaps) &&
-             "Fail to get a member from an interleaved store group");
-      Instruction *Member = Group->getMember(i);
-
-      // Skip the gaps in the group.
-      if (!Member) {
-        Value *Undef = PoisonValue::get(SubVT);
-        StoredVecs.push_back(Undef);
-        continue;
-      }
-
-      Value *StoredVec = State.get(StoredValues[StoredIdx], Part);
-      ++StoredIdx;
-
-      if (Group->isReverse())
-        StoredVec = Builder.CreateVectorReverse(StoredVec, "reverse");
-
-      // If this member has different type, cast it to a unified type.
-
-      if (StoredVec->getType() != SubVT)
-        StoredVec = createBitOrPointerCast(StoredVec, SubVT, DL);
-
-      StoredVecs.push_back(StoredVec);
-    }
-
-    // Interleave all the smaller vectors into one wider vector.
-    Value *IVec = interleaveVectors(Builder, StoredVecs, "interleaved.vec");
-    Instruction *NewStoreInstr;
-    if (BlockInMask || MaskForGaps) {
-      Value *GroupMask = CreateGroupMask(Part, MaskForGaps);
-      NewStoreInstr = Builder.CreateMaskedStore(IVec, AddrParts[Part],
-                                                Group->getAlign(), GroupMask);
-    } else
-      NewStoreInstr =
-          Builder.CreateAlignedStore(IVec, AddrParts[Part], Group->getAlign());
-
-    Group->addMetadata(NewStoreInstr);
-  }
-}
-
 void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
                                                VPReplicateRecipe *RepRecipe,
                                                const VPIteration &Instance,
@@ -2769,36 +2454,6 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) {
   return VectorTripCount;
 }
 
-Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy,
-                                                   const DataLayout &DL) {
-  // Verify that V is a vector type with same number of elements as DstVTy.
-  auto VF = DstVTy->getElementCount();
-  auto *SrcVecTy = cast<VectorType>(V->getType());
-  assert(VF == SrcVecTy->getElementCount() && "Vector dimensions do not match");
-  Type *SrcElemTy = SrcVecTy->getElementType();
-  Type *DstElemTy = DstVTy->getElementType();
-  assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&
-         "Vector elements must have same size");
-
-  // Do a direct cast if element types are castable.
-  if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) {
-    return Builder.CreateBitOrPointerCast(V, DstVTy);
-  }
-  // V cannot be directly casted to desired vector type.
-  // May happen when V is a floating point vector but DstVTy is a vector of
-  // pointers or vice-versa. Handle this using a two-step bitcast using an
-  // intermediate Integer type for the bitcast i.e. Ptr <-> Int <-> Float.
-  assert((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) &&
-         "Only one type should be a pointer type");
-  assert((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) &&
-         "Only one type should be a floating point type");
-  Type *IntTy =
-      IntegerType::getIntNTy(V->getContext(), DL.getTypeSizeInBits(SrcElemTy));
-  auto *VecIntTy = VectorType::get(IntTy, VF);
-  Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);
-  return Builder.CreateBitOrPointerCast(CastVal, DstVTy);
-}
-
 void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
   Value *Count = getTripCount();
   // Reuse existing vector loop preheader for TC checks.
@@ -9396,37 +9051,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
   VPlanTransforms::clearReductionWrapFlags(*Plan);
 }
 
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
-                               VPSlotTracker &SlotTracker) const {
-  O << Indent << "INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";
-  IG->getInsertPos()->printAsOperand(O, false);
-  O << ", ";
-  getAddr()->printAsOperand(O, SlotTracker);
-  VPValue *Mask = getMask();
-  if (Mask) {
-    O << ", ";
-    Mask->printAsOperand(O, SlotTracker);
-  }
-
-  unsigned OpIdx = 0;
-  for (unsigned i = 0; i < IG->getFactor(); ++i) {
-    if (!IG->getMember(i))
-      continue;
-    if (getNumStoreOperands() > 0) {
-      O << "\n" << Indent << "  store ";
-      getOperand(1 + OpIdx)->printAsOperand(O, SlotTracker);
-      O << " to index " << i;
-    } else {
-      O << "\n" << Indent << "  ";
-      getVPValue(OpIdx)->printAsOperand(O, SlotTracker);
-      O << " = load from index " << i;
-    }
-    ++OpIdx;
-  }
-}
-#endif
-
 void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
   assert(IndDesc.getKind() == InductionDescriptor::IK_PtrInduction &&
          "Not a pointer induction according to InductionDescriptor!");
@@ -9510,13 +9134,6 @@ void VPDerivedIVRecipe::execute(VPTransformState &State) {
   State.set(this, DerivedIV, VPIteration(0, 0));
 }
 
-void VPInterleaveRecipe::execute(VPTransformState &State) {
-  assert(!State.Instance && "Interleave group being replicated.");
-  State.ILV->vectorizeInterleaveGroup(IG, definedValues(), State, getAddr(),
-                                      getStoredValues(), getMask(),
-                                      NeedsMaskForGaps);
-}
-
 void VPReplicateRecipe::execute(VPTransformState &State) {
   Instruction *UI = getUnderlyingInstr();
   if (State.Instance) { // Generate a single instance.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 4b1ac79bbfdd4..1b787d0490672 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2037,6 +2037,373 @@ void VPWidenStoreEVLRecipe::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
+static Value *createBitOrPointerCast(IRBuilderBase &Builder, Value *V,
+                                     VectorType *DstVTy, const DataLayout &DL) {
+  // Verify that V is a vector type with same number of elements as DstVTy.
+  auto VF = DstVTy->getElementCount();
+  auto *SrcVecTy = cast<VectorType>(V->getType());
+  assert(VF == SrcVecTy->getElementCount() && "Vector dimensions do not match");
+  Type *SrcElemTy = SrcVecTy->getElementType();
+  Type *DstElemTy = DstVTy->getElementType();
+  assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&
+         "Vector elements must have same size");
+
+  // Do a direct cast if element types are castable.
+  if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) {
+    return Builder.CreateBitOrPointerCast(V, DstVTy);
+  }
+  // V cannot be directly casted to desired vector type.
+  // May happen when V is a floating point vector but DstVTy is a vector of
+  // pointers or vice-versa. Handle this using a two-step bitcast using an
+  // intermediate Integer type for the bitcast i.e. Ptr <-> Int <-> Float.
+  assert((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) &&
+         "Only one type should be a pointer type");
+  assert((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) &&
+         "Only one type should be a floating point type");
+  Type *IntTy =
+      IntegerType::getIntNTy(V->getContext(), DL.getTypeSizeInBits(SrcElemTy));
+  auto *VecIntTy = VectorType::get(IntTy, VF);
+  Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);
+  return Builder.CreateBitOrPointerCast(CastVal, DstVTy);
+}
+
+/// Return a vector containing interleaved elements from multiple
+/// smaller input vectors.
+static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
+                                const Twine &Name) {
+  unsigned Factor = Vals.size();
+  assert(Factor > 1 && "Tried to interleave invalid number of vectors");
+
+  VectorType *VecTy = cast<VectorType>(Vals[0]->getType());
+#ifndef NDEBUG
+  for (Value *Val : Vals)
+    assert(Val->getType() == VecTy && "Tried to interleave mismatched types");
+#endif
+
+  // Scalable vectors cannot use arbitrary shufflevectors (only splats), so
+  // must use intrinsics to interleave.
+  if (VecTy->isScalableTy()) {
+    VectorType *WideVecTy = VectorType::getDoubleElementsVectorType(VecTy);
+    return Builder.CreateIntrinsic(WideVecTy, Intrinsic::vector_interleave2,
+                                   Vals,
+                                   /*FMFSource=*/nullptr, Name);
+  }
+
+  // Fixed length. Start by concatenating all vectors into a wide vector.
+  Value *WideVec = concatenateVectors(Builder, Vals);
+
+  // Interleave the elements into the wide vector.
+  const unsigned NumElts = VecTy->getElementCount().getFixedValue();
+  return Builder.CreateShuffleVector(
+      WideVec, createInterleaveMask(NumElts, Factor), Name);
+}
+
+// Try to vectorize the interleave group that \p Instr belongs to.
+//
+// E.g. Translate following interleaved load group (factor = 3):
+//   for (i = 0; i < N; i+=3) {
+//     R = Pic[i];             // Member of index 0
+//     G = Pic[i+1];           // Member of index 1
+//     B = Pic[i+2];           // Member of index 2
+//     ... // do something to R, G, B
+//   }
+// To:
+//   %wide.vec = load <12 x i32>                       ; Read 4 tuples of R,G,B
+//   %R.vec = shuffle %wide.vec, poison, <0, 3, 6, 9>   ; R elements
+//   %G.vec = shuffle %wide.vec, poison, <1, 4, 7, 10>  ; G elements
+//   %B.vec = shuffle %wide.vec, poison, <2, 5, 8, 11>  ; B elements
+//
+// Or translate following interleaved store group (factor = 3):
+//   for (i = 0; i < N; i+=3) {
+//     ... do something to R, G, B
+//     Pic[i]   = R;           // Member of index 0
+//     Pic[i+1] = G;           // Member of index 1
+//     Pic[i+2] = B;           // Member of index 2
+//   }
+// To:
+//   %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
+//   %B_U.vec = shuffle %B.vec, poison, <0, 1, 2, 3, u, u, u, u>
+//   %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
+//        <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>    ; Interleave R,G,B elements
+//   store <12 x i32> %interleaved.vec              ; Write 4 tuples of R,G,B
+void VPInterleaveRecipe::execute(VPTransformState &State) {
+  assert(!State.Instance && "Interleave group being replicated.");
+  const InterleaveGroup<Instruction> *Group = IG;
+  Instruction *Instr = Group->getInsertPos();
+
+  // Prepare for the vector type of the interleaved load/store.
+  Type *ScalarTy = getLoadStoreType(Instr);
+  unsigned InterleaveFactor = Group->getFactor();
+  auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor);
+
+  // Prepare for the new pointers.
+  SmallVector<Value *, 2> AddrParts;
+  unsigned Index = Group->getIndex(Instr);
+
+  // TODO: extend the masked interleaved-group support to reversed access.
+  VPValue *BlockInMask = getMask();
+  assert((!BlockInMask || !Group->isReverse()) &&
+         "Reversed masked interleave-group not supported.");
+
+  Value *Idx;
+  // If the group is reverse, adjust the index to refer to the last vector lane
+  // instead of the first. We adjust the index from the first vector lane,
+  // rather than directly getting the pointer for lane VF - 1, because the
+  // pointer operand of the interleaved access is supposed to be uniform. For
+  // uniform instructions, we're only required to generate a value for the
+  // first vector lane in each unroll iteration.
+  if (Group->isReverse()) {
+    Value *RuntimeVF =
+        getRuntimeVF(State.Builder, State.Builder.getInt32Ty(), State.VF);
+    Idx = State.Builder.CreateSub(RuntimeVF, State.Builder.getInt32(1));
+    Idx = State.Builder.CreateMul(Idx,
+                                  State.Builder.getInt32(Group->getFactor()));
+    Idx = State.Builder.CreateAdd(Idx, State.Builder.getInt32(Index));
+    Idx = State.Builder.CreateNeg(Idx);
+  } else
+    Idx = State.Builder.getInt32(-Index);
+
+  VPValue *Addr = getAddr();
+  for (unsigned Part = 0; Part < State.UF; Part++) {
+    Value *AddrPart = State.get(Addr, VPIteration(Part, 0));
+    if (auto *I = dyn_cast<Instruction>(AddrPart))
+      State.setDebugLocFrom(I->getDebugLoc());
+
+    // Notice current instruction could be any index. Need to adjust the address
+    // to the member of index 0.
+    //
+    // E.g.  a = A[i+1];     // Member of index 1 (Current instruction)
+    //       b = A[i];       // Member of index 0
+    // Current pointer is pointed to A[i+1], adjust it to A[i].
+    //
+    // E.g.  A[i+1] = a;     // Member of index 1
+    //       A[i]   = b;     // Member of index 0
+    //       A[i+2] = c;     // Member of index 2 (Current instruction)
+    // Current pointer is pointed to A[i+2], adjust it to A[i].
+
+    bool InBounds = false;
+    if (auto *gep = dyn_cast<GetElementPtrInst>(AddrPart->stripPointerCasts()))
+      InBounds = gep->isInBounds();
+    AddrPart = State.Builder.CreateGEP(ScalarTy, AddrPart, Idx, "", InBounds);
+    AddrParts.push_back(AddrPart);
+  }
+
+  State.setDebugLocFrom(Instr->getDebugLoc());
+  Value *PoisonVec = PoisonValue::get(VecTy);
+
+  auto CreateGroupMask = [&BlockInMask, &State, &InterleaveFactor](
+                             unsigned Part, Value *MaskForGaps) -> Value * {
+    if (State.VF.isScalable()) {
+      assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");
+      assert(InterleaveFactor == 2 &&
+             "Unsupported deinterleave factor for scalable vectors");
+      auto *BlockInMaskPart = State.get(BlockInMask, Part);
+      SmallVector<Value *, 2> Ops = {BlockInMaskPart, BlockInMaskPart};
+      auto *MaskTy = VectorType::get(State.Builder.getInt1Ty(),
+                                     State.VF.getKnownMinValue() * 2, true);
+      return State.Builder.CreateIntrinsic(
+          MaskTy, Intrinsic::vector_interleave2, Ops,
+          /*FMFSource=*/nullptr, "interleaved.mask");
+    }
+
+    if (!BlockInMask)
+      return MaskForGaps;
+
+    Value *BlockInMaskPart = State.get(BlockInMask, Part);
+    Value *ShuffledMask = State.Builder.CreateShuffleVector(
+        BlockInMaskPart,
+        createReplicatedMask(InterleaveFactor, State.VF.getKnownMinValue()),
+        "interleaved.mask");
+    return MaskForGaps ? State.Builder.CreateBinOp(Instruction::And,
+                                                   ShuffledMask, MaskForGaps)
+                       : ShuffledMask;
+  };
+
+  const DataLayout &DL = Instr->getDataLayout();
+  // Vectorize the interleaved load group.
+  if (isa<LoadInst>(Instr)) {
+    Value *MaskForGaps = nullptr;
+    if (NeedsMaskForGaps) {
+      MaskForGaps = createBitMaskForGaps(State.Builder,
+                                         State.VF.getKnownMinValue(), *Group);
+      assert(MaskForGaps && "Mask for Gaps is required but it is null");
+    }
+
+    // For each unroll part, create a wide load for the group.
+    SmallVector<Value *, 2> NewLoads;
+    for (unsigned Part = 0; Part < State.UF; Part++) {
+      Instruction *NewLoad;
+      if (BlockInMask || MaskForGaps) {
+        Value *GroupMask = CreateGroupMask(Part, MaskForGaps);
+        NewLoad = State.Builder.CreateMaskedLoad(VecTy, AddrParts[Part],
+                                                 Group->getAlign(), GroupMask,
+                                                 PoisonVec, "wide.masked.vec");
+      } else
+        NewLoad = State.Builder.CreateAlignedLoad(
+            VecTy, AddrParts[Part], Group->getAlign(), "wide.vec");
+      Group->addMetadata(NewLoad);
+      NewLoads.push_back(NewLoad);
+    }
+
+    ArrayRef<VPValue *> VPDefs = definedValues();
+    const DataLayout &DL = State.CFG.PrevBB->getDataLayout();
+    if (VecTy->isScalableTy()) {
+      assert(InterleaveFactor == 2 &&
+             "Unsupported deinterleave factor for scalable vectors");
+
+      for (unsigned Part = 0; Part < State.UF; ++Part) {
+        // Scalable vectors cannot use arbitrary shufflevectors (only splats),
+        // so must use intrinsics to deinterleave.
+        Value *DI = State.Builder.CreateIntrinsic(
+            Intrinsic::vector_deinterleave2, VecTy, NewLoads[Part],
+            /*FMFSource=*/nullptr, "strided.vec");
+        unsigned J = 0;
+        for (unsigned I = 0; I < InterleaveFactor; ++I) {
+          Instruction *Member = Group->getMember(I);
+
+          if (!Member)
+            continue;
+
+          Value *StridedVec = State.Builder.CreateExtractValue(DI, I);
+          // If this member has different type, cast the result type.
+          if (Member->getType() != ScalarTy) {
+            VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
+            StridedVec =
+                createBitOrPointerCast(State.Builder, StridedVec, OtherVTy, DL);
+          }
+
+          if (Group->isReverse())
+            StridedVec =
+                State.Builder.CreateVectorReverse(StridedVec, "reverse");
+
+          State.set(VPDefs[J], StridedVec, Part);
+          ++J;
+        }
+      }
+
+      return;
+    }
+
+    // For each member in the group, shuffle out the appropriate data from the
+    // wide loads.
+    unsigned J = 0;
+    for (unsigned I = 0; I < InterleaveFactor; ++I) {
+      Instruction *Member = Group->getMember(I);
+
+      // Skip the gaps in the group.
+      if (!Member)
+        continue;
+
+      auto StrideMask =
+          createStrideMask(I, InterleaveFactor, State.VF.getKnownMinValue());
+      for (unsigned Part = 0; Part < State.UF; Part++) {
+        Value *StridedVec = State.Builder.CreateShuffleVector(
+            NewLoads[Part], StrideMask, "strided.vec");
+
+        // If this member has different type, cast the result type.
+        if (Member->getType() != ScalarTy) {
+          assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
+          VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
+          StridedVec =
+              createBitOrPointerCast(State.Builder, StridedVec, OtherVTy, DL);
+        }
+
+        if (Group->isReverse())
+          StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");
+
+        State.set(VPDefs[J], StridedVec, Part);
+      }
+      ++J;
+    }
+    return;
+  }
+
+  // The sub vector type for current instruction.
+  auto *SubVT = VectorType::get(ScalarTy, State.VF);
+
+  // Vectorize the interleaved store group.
+  Value *MaskForGaps =
+      createBitMaskForGaps(State.Builder, State.VF.getKnownMinValue(), *Group);
+  assert((!MaskForGaps || !State.VF.isScalable()) &&
+         "masking gaps for scalable vectors is not yet supported.");
+  ArrayRef<VPValue *> StoredValues = getStoredValues();
+  for (unsigned Part = 0; Part < State.UF; Part++) {
+    // Collect the stored vector from each member.
+    SmallVector<Value *, 4> StoredVecs;
+    unsigned StoredIdx = 0;
+    for (unsigned i = 0; i < InterleaveFactor; i++) {
+      assert((Group->getMember(i) || MaskForGaps) &&
+             "Fail to get a member from an interleaved store group");
+      Instruction *Member = Group->getMember(i);
+
+      // Skip the gaps in the group.
+      if (!Member) {
+        Value *Undef = PoisonValue::get(SubVT);
+        StoredVecs.push_back(Undef);
+        continue;
+      }
+
+      Value *StoredVec = State.get(StoredValues[StoredIdx], Part);
+      ++StoredIdx;
+
+      if (Group->isReverse())
+        StoredVec = State.Builder.CreateVectorReverse(StoredVec, "reverse");
+
+      // If this member has different type, cast it to a unified type.
+
+      if (StoredVec->getType() != SubVT)
+        StoredVec = createBitOrPointerCast(State.Builder, StoredVec, SubVT, DL);
+
+      StoredVecs.push_back(StoredVec);
+    }
+
+    // Interleave all the smaller vectors into one wider vector.
+    Value *IVec =
+        interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
+    Instruction *NewStoreInstr;
+    if (BlockInMask || MaskForGaps) {
+      Value *GroupMask = CreateGroupMask(Part, MaskForGaps);
+      NewStoreInstr = State.Builder.CreateMaskedStore(
+          IVec, AddrParts[Part], Group->getAlign(), GroupMask);
+    } else
+      NewStoreInstr = State.Builder.CreateAlignedStore(IVec, AddrParts[Part],
+                                                       Group->getAlign());
+
+    Group->addMetadata(NewStoreInstr);
+  }
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
+                               VPSlotTracker &SlotTracker) const {
+  O << Indent << "INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";
+  IG->getInsertPos()->printAsOperand(O, false);
+  O << ", ";
+  getAddr()->printAsOperand(O, SlotTracker);
+  VPValue *Mask = getMask();
+  if (Mask) {
+    O << ", ";
+    Mask->printAsOperand(O, SlotTracker);
+  }
+
+  unsigned OpIdx = 0;
+  for (unsigned i = 0; i < IG->getFactor(); ++i) {
+    if (!IG->getMember(i))
+      continue;
+    if (getNumStoreOperands() > 0) {
+      O << "\n" << Indent << "  store ";
+      getOperand(1 + OpIdx)->printAsOperand(O, SlotTracker);
+      O << " to index " << i;
+    } else {
+      O << "\n" << Indent << "  ";
+      getVPValue(OpIdx)->printAsOperand(O, SlotTracker);
+      O << " = load from index " << i;
+    }
+    ++OpIdx;
+  }
+}
+#endif
+
 void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) {
   Value *Start = getStartValue()->getLiveInIRValue();
   PHINode *EntryPart = PHINode::Create(Start->getType(), 2, "index");

From c93554b82aa77cf5fc43fb01a87498e0302f7b36 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 14:23:54 -0700
Subject: [PATCH 311/486] [ELF] Simplify ScriptLexer::consume. NFC

---
 lld/ELF/ScriptLexer.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lld/ELF/ScriptLexer.cpp b/lld/ELF/ScriptLexer.cpp
index 14f39ed10e17c..d5ffe8c4dfd8c 100644
--- a/lld/ELF/ScriptLexer.cpp
+++ b/lld/ELF/ScriptLexer.cpp
@@ -282,10 +282,9 @@ StringRef ScriptLexer::peek2() {
 }
 
 bool ScriptLexer::consume(StringRef tok) {
-  if (peek() == tok) {
-    skip();
+  if (next() == tok)
     return true;
-  }
+  --pos;
   return false;
 }
 

From efa833dd0f47a4f65df8c64dc145e06d8fb286f2 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 14:36:55 -0700
Subject: [PATCH 312/486] [ELF] Simplify readExpr. NFC

---
 lld/ELF/ScriptParser.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index 92f4bb41cd68f..f20de5e2fb4fb 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -1182,10 +1182,8 @@ SymbolAssignment *ScriptParser::readSymbolAssignment(StringRef name) {
 Expr ScriptParser::readExpr() {
   // Our lexer is context-aware. Set the in-expression bit so that
   // they apply different tokenization rules.
-  bool orig = inExpr;
-  inExpr = true;
+  SaveAndRestore saved(inExpr, true);
   Expr e = readExpr1(readPrimary(), 0);
-  inExpr = orig;
   return e;
 }
 
@@ -1251,9 +1249,9 @@ Expr ScriptParser::readExpr1(Expr lhs, int minPrec) {
     StringRef op1 = peek();
     if (precedence(op1) < minPrec)
       break;
-    if (consume("?"))
-      return readTernary(lhs);
     skip();
+    if (op1 == "?")
+      return readTernary(lhs);
     Expr rhs = readPrimary();
 
     // Evaluate the remaining part of the expression first if the

From 2572a76f230575611678e787fd9ccf6838a4cfe7 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Sat, 20 Jul 2024 14:57:23 -0700
Subject: [PATCH 313/486] Revert "Replace distutils.version with looseversion
 since the former was deprecated in python 3.10 and removed in 3.12." (#99786)

Reverts llvm/llvm-project#99549 because it breaks a bunch of build bots.
---
 cross-project-tests/lit.cfg.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cross-project-tests/lit.cfg.py b/cross-project-tests/lit.cfg.py
index 47062866fbc26..774c4eaf4d976 100644
--- a/cross-project-tests/lit.cfg.py
+++ b/cross-project-tests/lit.cfg.py
@@ -4,7 +4,8 @@
 import subprocess
 import sys
 
-from looseversion import LooseVersion
+# TODO: LooseVersion is undocumented; use something else.
+from distutils.version import LooseVersion
 
 import lit.formats
 import lit.util

From 16c24a850d6c60f02bc897e159d24a86aae09555 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Sat, 20 Jul 2024 19:24:10 -0300
Subject: [PATCH 314/486] [libc] Disable bind test for riscv

Currently it's returning ENXIO on bind and the build bots are failing
---
 libc/config/linux/riscv/entrypoints.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 026100736bb67..54a382eccb546 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -843,7 +843,6 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.sys.select.select
 
     # sys/socket.h entrypoints
-    libc.src.sys.socket.bind
     libc.src.sys.socket.socket
   )
 endif()

From ae2012d701827f224df04d0ae57e89472c1322d2 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Sun, 21 Jul 2024 00:30:31 +0200
Subject: [PATCH 315/486] [libc] Disable epoll_create fail test when
 SYS_epoll_create1 is used internally (#99785)

The fail test case only makes sense if SYS_epoll_create is used internally to implement epoll_create, since the only argument to epoll_create (size) is dropped if SYS_epoll_create1 is used.
---
 libc/test/src/sys/epoll/linux/epoll_create_test.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libc/test/src/sys/epoll/linux/epoll_create_test.cpp b/libc/test/src/sys/epoll/linux/epoll_create_test.cpp
index fdcdcf8eb4271..9c4bad10c8384 100644
--- a/libc/test/src/sys/epoll/linux/epoll_create_test.cpp
+++ b/libc/test/src/sys/epoll/linux/epoll_create_test.cpp
@@ -10,6 +10,7 @@
 #include "src/unistd/close.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
+#include <sys/syscall.h> // For syscall numbers.
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
 
@@ -21,6 +22,8 @@ TEST(LlvmLibcEpollCreateTest, Basic) {
   ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
 }
 
+#ifdef SYS_epoll_create
 TEST(LlvmLibcEpollCreateTest, Fails) {
   ASSERT_THAT(LIBC_NAMESPACE::epoll_create(0), Fails(EINVAL));
 }
+#endif

From 7cbd89921bed66e2422ce0b387fa8070b78152e1 Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Sat, 20 Jul 2024 23:51:04 +0100
Subject: [PATCH 316/486] [compiler-rt] lsan remove unneeded comma for
 pthread_atfork declaration. (#99788)

---
 compiler-rt/lib/lsan/lsan_interceptors.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cpp b/compiler-rt/lib/lsan/lsan_interceptors.cpp
index 6df4b6865b379..b569c337e9764 100644
--- a/compiler-rt/lib/lsan/lsan_interceptors.cpp
+++ b/compiler-rt/lib/lsan/lsan_interceptors.cpp
@@ -389,7 +389,7 @@ INTERCEPTOR(int, atexit, void (*f)()) {
 extern "C" {
 extern int _pthread_atfork(void (*prepare)(), void (*parent)(),
                            void (*child)());
-};
+}
 
 INTERCEPTOR(int, pthread_atfork, void (*prepare)(), void (*parent)(),
             void (*child)()) {

From ba9b5ff4f745b2c604e13656cdca0e66835dc99a Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 16:10:57 -0700
Subject: [PATCH 317/486] [ELF,test] Fix RUN line issue in defsym.s

---
 lld/test/ELF/defsym.s | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lld/test/ELF/defsym.s b/lld/test/ELF/defsym.s
index 0168ce854cc15..fed937ffc1c9c 100644
--- a/lld/test/ELF/defsym.s
+++ b/lld/test/ELF/defsym.s
@@ -11,8 +11,7 @@
 
 ## Check we are reporting the error correctly and don't crash
 ## when handling the second --defsym.
-# RUN: not ld.lld -o /dev/null %t.o --defsym ERR+ \
-#        --defsym foo2=foo1 2>&1 | FileCheck %s --check-prefix=ERR
+# RUN: not ld.lld -o /dev/null %t.o --defsym ERR+ --defsym foo2=foo1 2>&1 | FileCheck %s --check-prefix=ERR
 # ERR: error: --defsym: syntax error: ERR+
 
 # CHECK-DAG: 0000000000000123     0 NOTYPE  GLOBAL DEFAULT   ABS foo1
@@ -27,7 +26,7 @@
 # RUN: ld.lld -o %t %t.o --defsym=foo2=1
 # RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=ABS
 
-# ABS: 0000000000000123     0 NOTYPE  GLOBAL DEFAULT   ABS foo2
+# ABS: 0000000000000001     0 NOTYPE  GLOBAL DEFAULT   ABS foo2
 
 # RUN: ld.lld -o %t %t.o --defsym=foo2=foo1+5
 # RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=EXPR

From b828c13f3ceb336bf517fab0223b966ccace100e Mon Sep 17 00:00:00 2001
From: Hongyu Chen <hongyc4@uci.edu>
Date: Sat, 20 Jul 2024 16:35:38 -0700
Subject: [PATCH 318/486]  [ELF] Delete peek2 in Lexer (#99790)

Thanks to Fangrui's change

https://github.com/llvm/llvm-project/commit/28045ceab08d41a8a42d93ebc445e8fe906f884c
so peek2 can be removed.
---
 lld/ELF/ScriptLexer.cpp  |  9 ---------
 lld/ELF/ScriptLexer.h    |  1 -
 lld/ELF/ScriptParser.cpp | 28 ++++++++++++++--------------
 3 files changed, 14 insertions(+), 24 deletions(-)

diff --git a/lld/ELF/ScriptLexer.cpp b/lld/ELF/ScriptLexer.cpp
index d5ffe8c4dfd8c..c8c02ab0f3e09 100644
--- a/lld/ELF/ScriptLexer.cpp
+++ b/lld/ELF/ScriptLexer.cpp
@@ -272,15 +272,6 @@ StringRef ScriptLexer::peek() {
   return tok;
 }
 
-StringRef ScriptLexer::peek2() {
-  skip();
-  StringRef tok = next();
-  if (errorCount())
-    return "";
-  pos = pos - 2;
-  return tok;
-}
-
 bool ScriptLexer::consume(StringRef tok) {
   if (next() == tok)
     return true;
diff --git a/lld/ELF/ScriptLexer.h b/lld/ELF/ScriptLexer.h
index 7919e493fa28b..d5393818ed553 100644
--- a/lld/ELF/ScriptLexer.h
+++ b/lld/ELF/ScriptLexer.h
@@ -26,7 +26,6 @@ class ScriptLexer {
   bool atEOF();
   StringRef next();
   StringRef peek();
-  StringRef peek2();
   void skip();
   bool consume(StringRef tok);
   void expect(StringRef expect);
diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index f20de5e2fb4fb..49aa7e6374905 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -92,7 +92,7 @@ class ScriptParser final : ScriptLexer {
   SymbolAssignment *readSymbolAssignment(StringRef name);
   ByteCommand *readByteCommand(StringRef tok);
   std::array<uint8_t, 4> readFill();
-  bool readSectionDirective(OutputSection *cmd, StringRef tok2);
+  bool readSectionDirective(OutputSection *cmd, StringRef tok);
   void readSectionAddressType(OutputSection *cmd);
   OutputDesc *readOverlaySectionDescription();
   OutputDesc *readOutputSectionDescription(StringRef outSec);
@@ -873,14 +873,11 @@ constexpr std::pair<const char *, unsigned> typeMap[] = {
 // Tries to read the special directive for an output section definition which
 // can be one of following: "(NOLOAD)", "(COPY)", "(INFO)", "(OVERLAY)", and
 // "(TYPE=<value>)".
-// Tok1 and Tok2 are next 2 tokens peeked. See comment for
-// readSectionAddressType below.
-bool ScriptParser::readSectionDirective(OutputSection *cmd, StringRef tok2) {
-  if (tok2 != "NOLOAD" && tok2 != "COPY" && tok2 != "INFO" &&
-      tok2 != "OVERLAY" && tok2 != "TYPE")
+bool ScriptParser::readSectionDirective(OutputSection *cmd, StringRef tok) {
+  if (tok != "NOLOAD" && tok != "COPY" && tok != "INFO" && tok != "OVERLAY" &&
+      tok != "TYPE")
     return false;
 
-  expect("(");
   if (consume("NOLOAD")) {
     cmd->type = SHT_NOBITS;
     cmd->typeIsSet = true;
@@ -919,19 +916,22 @@ bool ScriptParser::readSectionDirective(OutputSection *cmd, StringRef tok2) {
 // https://sourceware.org/binutils/docs/ld/Output-Section-Address.html
 // https://sourceware.org/binutils/docs/ld/Output-Section-Type.html
 void ScriptParser::readSectionAddressType(OutputSection *cmd) {
-  if (peek() == "(") {
+  if (consume("(")) {
     // Temporarily set inExpr to support TYPE=<value> without spaces.
     SaveAndRestore saved(inExpr, true);
-    if (readSectionDirective(cmd, peek2()))
+    if (readSectionDirective(cmd, peek()))
       return;
+    cmd->addrExpr = readExpr();
+    expect(")");
+  } else {
+    cmd->addrExpr = readExpr();
   }
-  cmd->addrExpr = readExpr();
 
-  if (peek() == "(") {
+  if (consume("(")) {
     SaveAndRestore saved(inExpr, true);
-    StringRef tok2 = peek2();
-    if (!readSectionDirective(cmd, tok2))
-      setError("unknown section directive: " + tok2);
+    StringRef tok = peek();
+    if (!readSectionDirective(cmd, tok))
+      setError("unknown section directive: " + tok);
   }
 }
 

From 1d5d18924d185a4267462479307f1ff9911cb112 Mon Sep 17 00:00:00 2001
From: hev <wangrui@loongson.cn>
Date: Sun, 21 Jul 2024 10:17:20 +0800
Subject: [PATCH 319/486] Revert "[LoongArch] Remove spurious mask operations
 from andn->icmp on 16 and 8 bit values" (#99792)

Reverts llvm/llvm-project#99272
---
 .../LoongArch/LoongArchISelLowering.cpp       | 162 ------------------
 llvm/test/CodeGen/LoongArch/andn-icmp.ll      |  56 ++++--
 2 files changed, 40 insertions(+), 178 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 6072e5e244263..ba6be85c7f2e8 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -335,7 +335,6 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
   setTargetDAGCombine(ISD::AND);
   setTargetDAGCombine(ISD::OR);
   setTargetDAGCombine(ISD::SRL);
-  setTargetDAGCombine(ISD::SETCC);
 
   // Set DAG combine for 'LSX' feature.
 
@@ -2529,165 +2528,6 @@ static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
-static bool checkValueWidth(SDValue V, ISD::LoadExtType &ExtType) {
-  ExtType = ISD::NON_EXTLOAD;
-
-  switch (V.getNode()->getOpcode()) {
-  case ISD::LOAD: {
-    LoadSDNode *LoadNode = cast<LoadSDNode>(V.getNode());
-    if ((LoadNode->getMemoryVT() == MVT::i8) ||
-        (LoadNode->getMemoryVT() == MVT::i16)) {
-      ExtType = LoadNode->getExtensionType();
-      return true;
-    }
-    return false;
-  }
-  case ISD::AssertSext: {
-    VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
-    if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
-      ExtType = ISD::SEXTLOAD;
-      return true;
-    }
-    return false;
-  }
-  case ISD::AssertZext: {
-    VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
-    if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
-      ExtType = ISD::ZEXTLOAD;
-      return true;
-    }
-    return false;
-  }
-  default:
-    return false;
-  }
-
-  return false;
-}
-
-// Eliminate redundant truncation and zero-extension nodes.
-// * Case 1:
-//  +------------+ +------------+ +------------+
-//  |   Input1   | |   Input2   | |     CC     |
-//  +------------+ +------------+ +------------+
-//         |              |              |
-//         V              V              +----+
-//  +------------+ +------------+             |
-//  |  TRUNCATE  | |  TRUNCATE  |             |
-//  +------------+ +------------+             |
-//         |              |                   |
-//         V              V                   |
-//  +------------+ +------------+             |
-//  |  ZERO_EXT  | |  ZERO_EXT  |             |
-//  +------------+ +------------+             |
-//         |              |                   |
-//         |              +-------------+     |
-//         V              V             |     |
-//        +----------------+            |     |
-//        |      AND       |            |     |
-//        +----------------+            |     |
-//                |                     |     |
-//                +---------------+     |     |
-//                                |     |     |
-//                                V     V     V
-//                               +-------------+
-//                               |     CMP     |
-//                               +-------------+
-// * Case 2:
-//  +------------+ +------------+ +-------------+ +------------+ +------------+
-//  |   Input1   | |   Input2   | | Constant -1 | | Constant 0 | |     CC     |
-//  +------------+ +------------+ +-------------+ +------------+ +------------+
-//         |              |             |               |               |
-//         V              |             |               |               |
-//  +------------+        |             |               |               |
-//  |     XOR    |<---------------------+               |               |
-//  +------------+        |                             |               |
-//         |              |                             |               |
-//         V              V             +---------------+               |
-//  +------------+ +------------+       |                               |
-//  |  TRUNCATE  | |  TRUNCATE  |       |     +-------------------------+
-//  +------------+ +------------+       |     |
-//         |              |             |     |
-//         V              V             |     |
-//  +------------+ +------------+       |     |
-//  |  ZERO_EXT  | |  ZERO_EXT  |       |     |
-//  +------------+ +------------+       |     |
-//         |              |             |     |
-//         V              V             |     |
-//        +----------------+            |     |
-//        |      AND       |            |     |
-//        +----------------+            |     |
-//                |                     |     |
-//                +---------------+     |     |
-//                                |     |     |
-//                                V     V     V
-//                               +-------------+
-//                               |     CMP     |
-//                               +-------------+
-static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
-                                   TargetLowering::DAGCombinerInfo &DCI,
-                                   const LoongArchSubtarget &Subtarget) {
-  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
-
-  SDNode *AndNode = N->getOperand(0).getNode();
-  if (AndNode->getOpcode() != ISD::AND)
-    return SDValue();
-
-  SDValue AndInputValue2 = AndNode->getOperand(1);
-  if (AndInputValue2.getOpcode() != ISD::ZERO_EXTEND)
-    return SDValue();
-
-  SDValue CmpInputValue = N->getOperand(1);
-  SDValue AndInputValue1 = AndNode->getOperand(0);
-  if (AndInputValue1.getOpcode() == ISD::XOR) {
-    if (CC != ISD::SETEQ && CC != ISD::SETNE)
-      return SDValue();
-    ConstantSDNode *CN = dyn_cast<ConstantSDNode>(AndInputValue1.getOperand(1));
-    if (!CN || CN->getSExtValue() != -1)
-      return SDValue();
-    CN = dyn_cast<ConstantSDNode>(CmpInputValue);
-    if (!CN || CN->getSExtValue() != 0)
-      return SDValue();
-    AndInputValue1 = AndInputValue1.getOperand(0);
-    if (AndInputValue1.getOpcode() != ISD::ZERO_EXTEND)
-      return SDValue();
-  } else if (AndInputValue1.getOpcode() == ISD::ZERO_EXTEND) {
-    if (AndInputValue2 != CmpInputValue)
-      return SDValue();
-  } else {
-    return SDValue();
-  }
-
-  SDValue TruncValue1 = AndInputValue1.getNode()->getOperand(0);
-  if (TruncValue1.getOpcode() != ISD::TRUNCATE)
-    return SDValue();
-
-  SDValue TruncValue2 = AndInputValue2.getNode()->getOperand(0);
-  if (TruncValue2.getOpcode() != ISD::TRUNCATE)
-    return SDValue();
-
-  SDValue TruncInputValue1 = TruncValue1.getNode()->getOperand(0);
-  SDValue TruncInputValue2 = TruncValue2.getNode()->getOperand(0);
-  ISD::LoadExtType ExtType1;
-  ISD::LoadExtType ExtType2;
-
-  if (!checkValueWidth(TruncInputValue1, ExtType1) ||
-      !checkValueWidth(TruncInputValue2, ExtType2))
-    return SDValue();
-
-  if ((ExtType2 != ISD::ZEXTLOAD) &&
-      ((ExtType2 != ISD::SEXTLOAD) && (ExtType1 != ISD::SEXTLOAD)))
-    return SDValue();
-
-  // These truncation and zero-extension nodes are not necessary, remove them.
-  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N), AndNode->getValueType(0),
-                               TruncInputValue1, TruncInputValue2);
-  SDValue NewSetCC =
-      DAG.getSetCC(SDLoc(N), N->getValueType(0), NewAnd, TruncInputValue2, CC);
-  DAG.ReplaceAllUsesWith(N, NewSetCC.getNode());
-  return SDValue(N, 0);
-}
-
 // Combine (loongarch_bitrev_w (loongarch_revb_2w X)) to loongarch_bitrev_4b.
 static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG,
                                       TargetLowering::DAGCombinerInfo &DCI,
@@ -3315,8 +3155,6 @@ SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N,
     return performANDCombine(N, DAG, DCI, Subtarget);
   case ISD::OR:
     return performORCombine(N, DAG, DCI, Subtarget);
-  case ISD::SETCC:
-    return performSETCCCombine(N, DAG, DCI, Subtarget);
   case ISD::SRL:
     return performSRLCombine(N, DAG, DCI, Subtarget);
   case LoongArchISD::BITREV_W:
diff --git a/llvm/test/CodeGen/LoongArch/andn-icmp.ll b/llvm/test/CodeGen/LoongArch/andn-icmp.ll
index 6d07e7a947297..4fc3c8df4664c 100644
--- a/llvm/test/CodeGen/LoongArch/andn-icmp.ll
+++ b/llvm/test/CodeGen/LoongArch/andn-icmp.ll
@@ -6,12 +6,14 @@ define i1 @andn_icmp_eq_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_eq_i8:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    andn $a0, $a1, $a0
+; LA32-NEXT:    andi $a0, $a0, 255
 ; LA32-NEXT:    sltui $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_eq_i8:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    andn $a0, $a1, $a0
+; LA64-NEXT:    andi $a0, $a0, 255
 ; LA64-NEXT:    sltui $a0, $a0, 1
 ; LA64-NEXT:    ret
   %and = and i8 %a, %b
@@ -23,12 +25,14 @@ define i1 @andn_icmp_eq_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_eq_i16:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    andn $a0, $a1, $a0
+; LA32-NEXT:    bstrpick.w $a0, $a0, 15, 0
 ; LA32-NEXT:    sltui $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_eq_i16:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    andn $a0, $a1, $a0
+; LA64-NEXT:    bstrpick.d $a0, $a0, 15, 0
 ; LA64-NEXT:    sltui $a0, $a0, 1
 ; LA64-NEXT:    ret
   %and = and i16 %a, %b
@@ -76,12 +80,14 @@ define i1 @andn_icmp_ne_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ne_i8:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    andn $a0, $a1, $a0
+; LA32-NEXT:    andi $a0, $a0, 255
 ; LA32-NEXT:    sltu $a0, $zero, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ne_i8:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    andn $a0, $a1, $a0
+; LA64-NEXT:    andi $a0, $a0, 255
 ; LA64-NEXT:    sltu $a0, $zero, $a0
 ; LA64-NEXT:    ret
   %and = and i8 %a, %b
@@ -93,12 +99,14 @@ define i1 @andn_icmp_ne_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ne_i16:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    andn $a0, $a1, $a0
+; LA32-NEXT:    bstrpick.w $a0, $a0, 15, 0
 ; LA32-NEXT:    sltu $a0, $zero, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ne_i16:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    andn $a0, $a1, $a0
+; LA64-NEXT:    bstrpick.d $a0, $a0, 15, 0
 ; LA64-NEXT:    sltu $a0, $zero, $a0
 ; LA64-NEXT:    ret
   %and = and i16 %a, %b
@@ -145,13 +153,15 @@ define i1 @andn_icmp_ne_i64(i64 %a, i64 %b) nounwind {
 define i1 @andn_icmp_ult_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ult_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    and $a0, $a1, $a0
 ; LA32-NEXT:    sltu $a0, $a0, $a1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ult_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    and $a0, $a1, $a0
 ; LA64-NEXT:    sltu $a0, $a0, $a1
 ; LA64-NEXT:    ret
   %and = and i8 %a, %b
@@ -162,13 +172,15 @@ define i1 @andn_icmp_ult_i8(i8 signext %a, i8 signext %b) nounwind {
 define i1 @andn_icmp_ult_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ult_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    and $a0, $a1, $a0
 ; LA32-NEXT:    sltu $a0, $a0, $a1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ult_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    and $a0, $a1, $a0
 ; LA64-NEXT:    sltu $a0, $a0, $a1
 ; LA64-NEXT:    ret
   %and = and i16 %a, %b
@@ -179,14 +191,16 @@ define i1 @andn_icmp_ult_i16(i16 signext %a, i16 signext %b) nounwind {
 define i1 @andn_icmp_uge_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_uge_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    and $a0, $a1, $a0
 ; LA32-NEXT:    sltu $a0, $a0, $a1
 ; LA32-NEXT:    xori $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_uge_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    and $a0, $a1, $a0
 ; LA64-NEXT:    sltu $a0, $a0, $a1
 ; LA64-NEXT:    xori $a0, $a0, 1
 ; LA64-NEXT:    ret
@@ -198,14 +212,16 @@ define i1 @andn_icmp_uge_i8(i8 signext %a, i8 signext %b) nounwind {
 define i1 @andn_icmp_uge_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_uge_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    and $a0, $a1, $a0
 ; LA32-NEXT:    sltu $a0, $a0, $a1
 ; LA32-NEXT:    xori $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_uge_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    and $a0, $a1, $a0
 ; LA64-NEXT:    sltu $a0, $a0, $a1
 ; LA64-NEXT:    xori $a0, $a0, 1
 ; LA64-NEXT:    ret
@@ -217,13 +233,15 @@ define i1 @andn_icmp_uge_i16(i16 signext %a, i16 signext %b) nounwind {
 define i1 @andn_icmp_ugt_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ugt_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    and $a0, $a1, $a0
 ; LA32-NEXT:    sltu $a0, $a1, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ugt_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    and $a0, $a1, $a0
 ; LA64-NEXT:    sltu $a0, $a1, $a0
 ; LA64-NEXT:    ret
   %and = and i8 %a, %b
@@ -234,13 +252,15 @@ define i1 @andn_icmp_ugt_i8(i8 signext %a, i8 signext %b) nounwind {
 define i1 @andn_icmp_ugt_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ugt_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    and $a0, $a1, $a0
 ; LA32-NEXT:    sltu $a0, $a1, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ugt_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    and $a0, $a1, $a0
 ; LA64-NEXT:    sltu $a0, $a1, $a0
 ; LA64-NEXT:    ret
   %and = and i16 %a, %b
@@ -251,14 +271,16 @@ define i1 @andn_icmp_ugt_i16(i16 signext %a, i16 signext %b) nounwind {
 define i1 @andn_icmp_ule_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ule_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    and $a0, $a1, $a0
 ; LA32-NEXT:    sltu $a0, $a1, $a0
 ; LA32-NEXT:    xori $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ule_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    and $a0, $a1, $a0
 ; LA64-NEXT:    sltu $a0, $a1, $a0
 ; LA64-NEXT:    xori $a0, $a0, 1
 ; LA64-NEXT:    ret
@@ -270,14 +292,16 @@ define i1 @andn_icmp_ule_i8(i8 signext %a, i8 signext %b) nounwind {
 define i1 @andn_icmp_ule_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ule_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    and $a0, $a1, $a0
 ; LA32-NEXT:    sltu $a0, $a1, $a0
 ; LA32-NEXT:    xori $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ule_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    and $a0, $a1, $a0
 ; LA64-NEXT:    sltu $a0, $a1, $a0
 ; LA64-NEXT:    xori $a0, $a0, 1
 ; LA64-NEXT:    ret

From c8e69fa4a0bd158fbd63cdb4794dd1232184f155 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Sat, 20 Jul 2024 22:35:04 -0500
Subject: [PATCH 320/486] [libc] Fix GPU 'printf' on strings with padding

Summary:
We get the `strlen` to know how much memory to allocate here, but it
wasn't taking into account if the padding was larger than the string
itself. This patch sets it to an empty string so we always add the
minimum size. This implementation is slightly wasteful with memory, but
I am not concerned with a few extra bytes here and there for some memory
that gets immediately free'd.
---
 libc/utils/gpu/server/rpc_server.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libc/utils/gpu/server/rpc_server.cpp b/libc/utils/gpu/server/rpc_server.cpp
index 119539e3cad44..ed23d22f0bc36 100644
--- a/libc/utils/gpu/server/rpc_server.cpp
+++ b/libc/utils/gpu/server/rpc_server.cpp
@@ -108,6 +108,10 @@ void handle_printf(rpc::Server::Port &port) {
       if (cur_section.has_conv && cur_section.conv_name == 's' &&
           cur_section.conv_val_ptr) {
         strs_to_copy[lane].emplace_back(cur_section.conv_val_ptr);
+        // Get the minimum size of the string in the case of padding.
+        char c = '\0';
+        cur_section.conv_val_ptr = &c;
+        convert(&writer, cur_section);
       } else if (cur_section.has_conv) {
         // Ignore conversion errors for the first pass.
         convert(&writer, cur_section);

From d69eb7b7ffb33d0be716759a38a235868671705d Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 20:59:26 -0700
Subject: [PATCH 321/486] [MC] Move createMC{Object,Asm}Streamer to .cpp

Currently, the template arguments are incomplete types and unique_ptr&&
has to be used. Moving the implementation to .cpp allows us to
use complete types and unique_ptr.

In addition, add a createMCObjectStreamer overload without unused bool
parameters. The existing createMCObjectStreamer overload, with unused
and confusing bool parameters, will be deprecated.
---
 llvm/include/llvm/MC/TargetRegistry.h | 67 +++------------------
 llvm/lib/MC/TargetRegistry.cpp        | 84 +++++++++++++++++++++++++++
 2 files changed, 91 insertions(+), 60 deletions(-)

diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index bb4566aa65e03..e09318ba18f5a 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -525,63 +525,17 @@ class Target {
   /// \param TAB The target assembler backend object. Takes ownership.
   /// \param OW The stream object.
   /// \param Emitter The target independent assembler object.Takes ownership.
+  MCStreamer *createMCObjectStreamer(const Triple &T, MCContext &Ctx,
+                                     std::unique_ptr<MCAsmBackend> TAB,
+                                     std::unique_ptr<MCObjectWriter> OW,
+                                     std::unique_ptr<MCCodeEmitter> Emitter,
+                                     const MCSubtargetInfo &STI) const;
   MCStreamer *createMCObjectStreamer(const Triple &T, MCContext &Ctx,
                                      std::unique_ptr<MCAsmBackend> &&TAB,
                                      std::unique_ptr<MCObjectWriter> &&OW,
                                      std::unique_ptr<MCCodeEmitter> &&Emitter,
                                      const MCSubtargetInfo &STI, bool, bool,
-                                     bool) const {
-    MCStreamer *S = nullptr;
-    switch (T.getObjectFormat()) {
-    case Triple::UnknownObjectFormat:
-      llvm_unreachable("Unknown object format");
-    case Triple::COFF:
-      assert((T.isOSWindows() || T.isUEFI()) &&
-             "only Windows and UEFI COFF are supported");
-      S = COFFStreamerCtorFn(Ctx, std::move(TAB), std::move(OW),
-                             std::move(Emitter));
-      break;
-    case Triple::MachO:
-      if (MachOStreamerCtorFn)
-        S = MachOStreamerCtorFn(Ctx, std::move(TAB), std::move(OW),
-                                std::move(Emitter));
-      else
-        S = createMachOStreamer(Ctx, std::move(TAB), std::move(OW),
-                                std::move(Emitter), false);
-      break;
-    case Triple::ELF:
-      if (ELFStreamerCtorFn)
-        S = ELFStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW),
-                              std::move(Emitter));
-      else
-        S = createELFStreamer(Ctx, std::move(TAB), std::move(OW),
-                              std::move(Emitter));
-      break;
-    case Triple::Wasm:
-      S = createWasmStreamer(Ctx, std::move(TAB), std::move(OW),
-                             std::move(Emitter));
-      break;
-    case Triple::GOFF:
-      S = createGOFFStreamer(Ctx, std::move(TAB), std::move(OW),
-                             std::move(Emitter));
-      break;
-    case Triple::XCOFF:
-      S = XCOFFStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW),
-                              std::move(Emitter));
-      break;
-    case Triple::SPIRV:
-      S = createSPIRVStreamer(Ctx, std::move(TAB), std::move(OW),
-                              std::move(Emitter));
-      break;
-    case Triple::DXContainer:
-      S = createDXContainerStreamer(Ctx, std::move(TAB), std::move(OW),
-                                    std::move(Emitter));
-      break;
-    }
-    if (ObjectTargetStreamerCtorFn)
-      ObjectTargetStreamerCtorFn(*S, STI);
-    return S;
-  }
+                                     bool) const;
 
   MCStreamer *createAsmStreamer(MCContext &Ctx,
                                 std::unique_ptr<formatted_raw_ostream> OS,
@@ -589,14 +543,7 @@ class Target {
                                 MCInstPrinter *InstPrint,
                                 std::unique_ptr<MCCodeEmitter> &&CE,
                                 std::unique_ptr<MCAsmBackend> &&TAB,
-                                bool ShowInst) const {
-    formatted_raw_ostream &OSRef = *OS;
-    MCStreamer *S = llvm::createAsmStreamer(
-        Ctx, std::move(OS), IsVerboseAsm, UseDwarfDirectory, InstPrint,
-        std::move(CE), std::move(TAB), ShowInst);
-    createAsmTargetStreamer(*S, OSRef, InstPrint, IsVerboseAsm);
-    return S;
-  }
+                                bool ShowInst) const;
 
   MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
                                             formatted_raw_ostream &OS,
diff --git a/llvm/lib/MC/TargetRegistry.cpp b/llvm/lib/MC/TargetRegistry.cpp
index 0aa48916c7d25..75272b445db65 100644
--- a/llvm/lib/MC/TargetRegistry.cpp
+++ b/llvm/lib/MC/TargetRegistry.cpp
@@ -9,6 +9,10 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/MC/MCAsmBackend.h"
+#include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCObjectStreamer.h"
+#include "llvm/MC/MCObjectWriter.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <vector>
@@ -17,6 +21,86 @@ using namespace llvm;
 // Clients are responsible for avoid race conditions in registration.
 static Target *FirstTarget = nullptr;
 
+MCStreamer *Target::createMCObjectStreamer(
+    const Triple &T, MCContext &Ctx, std::unique_ptr<MCAsmBackend> TAB,
+    std::unique_ptr<MCObjectWriter> OW, std::unique_ptr<MCCodeEmitter> Emitter,
+    const MCSubtargetInfo &STI) const {
+  MCStreamer *S = nullptr;
+  switch (T.getObjectFormat()) {
+  case Triple::UnknownObjectFormat:
+    llvm_unreachable("Unknown object format");
+  case Triple::COFF:
+    assert((T.isOSWindows() || T.isUEFI()) &&
+           "only Windows and UEFI COFF are supported");
+    S = COFFStreamerCtorFn(Ctx, std::move(TAB), std::move(OW),
+                           std::move(Emitter));
+    break;
+  case Triple::MachO:
+    if (MachOStreamerCtorFn)
+      S = MachOStreamerCtorFn(Ctx, std::move(TAB), std::move(OW),
+                              std::move(Emitter));
+    else
+      S = createMachOStreamer(Ctx, std::move(TAB), std::move(OW),
+                              std::move(Emitter), false);
+    break;
+  case Triple::ELF:
+    if (ELFStreamerCtorFn)
+      S = ELFStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW),
+                            std::move(Emitter));
+    else
+      S = createELFStreamer(Ctx, std::move(TAB), std::move(OW),
+                            std::move(Emitter));
+    break;
+  case Triple::Wasm:
+    S = createWasmStreamer(Ctx, std::move(TAB), std::move(OW),
+                           std::move(Emitter));
+    break;
+  case Triple::GOFF:
+    S = createGOFFStreamer(Ctx, std::move(TAB), std::move(OW),
+                           std::move(Emitter));
+    break;
+  case Triple::XCOFF:
+    S = XCOFFStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW),
+                            std::move(Emitter));
+    break;
+  case Triple::SPIRV:
+    S = createSPIRVStreamer(Ctx, std::move(TAB), std::move(OW),
+                            std::move(Emitter));
+    break;
+  case Triple::DXContainer:
+    S = createDXContainerStreamer(Ctx, std::move(TAB), std::move(OW),
+                                  std::move(Emitter));
+    break;
+  }
+  if (ObjectTargetStreamerCtorFn)
+    ObjectTargetStreamerCtorFn(*S, STI);
+  return S;
+}
+
+MCStreamer *Target::createMCObjectStreamer(
+    const Triple &T, MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB,
+    std::unique_ptr<MCObjectWriter> &&OW,
+    std::unique_ptr<MCCodeEmitter> &&Emitter, const MCSubtargetInfo &STI, bool,
+    bool, bool) const {
+  return createMCObjectStreamer(T, Ctx, std::move(TAB), std::move(OW),
+                                std::move(Emitter), STI);
+}
+
+MCStreamer *Target::createAsmStreamer(MCContext &Ctx,
+                                      std::unique_ptr<formatted_raw_ostream> OS,
+                                      bool IsVerboseAsm, bool UseDwarfDirectory,
+                                      MCInstPrinter *InstPrint,
+                                      std::unique_ptr<MCCodeEmitter> &&CE,
+                                      std::unique_ptr<MCAsmBackend> &&TAB,
+                                      bool ShowInst) const {
+  formatted_raw_ostream &OSRef = *OS;
+  MCStreamer *S = llvm::createAsmStreamer(
+      Ctx, std::move(OS), IsVerboseAsm, UseDwarfDirectory, InstPrint,
+      std::move(CE), std::move(TAB), ShowInst);
+  createAsmTargetStreamer(*S, OSRef, InstPrint, IsVerboseAsm);
+  return S;
+}
+
 iterator_range<TargetRegistry::iterator> TargetRegistry::targets() {
   return make_range(iterator(FirstTarget), iterator());
 }

From b8220b986dcc8c5d0c44a125642009d8175fc11d Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 21:27:36 -0700
Subject: [PATCH 322/486] [MC] Remove unused bool arguments from
 createMCObjectStreamer callers

---
 clang/tools/driver/cc1as_main.cpp                     |  4 +---
 llvm/lib/CodeGen/LLVMTargetMachine.cpp                | 11 +++++------
 llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp        |  3 +--
 llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp    |  3 +--
 llvm/tools/llvm-dwp/llvm-dwp.cpp                      |  5 ++---
 .../llvm-mc-assemble-fuzzer.cpp                       |  5 ++---
 llvm/tools/llvm-mc/llvm-mc.cpp                        |  4 +---
 llvm/tools/llvm-ml/llvm-ml.cpp                        |  5 ++---
 .../DebugInfo/DWARF/DWARFExpressionCopyBytesTest.cpp  |  5 +----
 llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp     |  3 +--
 llvm/unittests/MC/DwarfLineTableHeaders.cpp           |  5 +----
 mlir/lib/Target/LLVM/ROCDL/Target.cpp                 |  3 +--
 12 files changed, 19 insertions(+), 37 deletions(-)

diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index ec93f092713f5..2d5acbecb123a 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -571,9 +571,7 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
 
     Triple T(Opts.Triple);
     Str.reset(TheTarget->createMCObjectStreamer(
-        T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI,
-        Opts.RelaxAll, Opts.IncrementalLinkerCompatible,
-        /*DWARFMustBeAtTheEnd*/ true));
+        T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI));
     Str.get()->initSections(Opts.NoExecStack, *STI);
   }
 
diff --git a/llvm/lib/CodeGen/LLVMTargetMachine.cpp b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
index 3092177e37e98..562821df06651 100644
--- a/llvm/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
@@ -208,8 +208,7 @@ Expected<std::unique_ptr<MCStreamer>> LLVMTargetMachine::createMCStreamer(
         T, Context, std::unique_ptr<MCAsmBackend>(MAB),
         DwoOut ? MAB->createDwoObjectWriter(Out, *DwoOut)
                : MAB->createObjectWriter(Out),
-        std::unique_ptr<MCCodeEmitter>(MCE), STI, /*ignore=*/false, false,
-        false));
+        std::unique_ptr<MCCodeEmitter>(MCE), STI));
     break;
   }
   case CodeGenFileType::Null:
@@ -275,15 +274,15 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, MCContext *&Ctx,
   const MCRegisterInfo &MRI = *getMCRegisterInfo();
   std::unique_ptr<MCCodeEmitter> MCE(
       getTarget().createMCCodeEmitter(*getMCInstrInfo(), *Ctx));
-  std::unique_ptr<MCAsmBackend> MAB(
-      getTarget().createMCAsmBackend(STI, MRI, Options.MCOptions));
+  MCAsmBackend *MAB =
+      getTarget().createMCAsmBackend(STI, MRI, Options.MCOptions);
   if (!MCE || !MAB)
     return true;
 
   const Triple &T = getTargetTriple();
   std::unique_ptr<MCStreamer> AsmStreamer(getTarget().createMCObjectStreamer(
-      T, *Ctx, std::move(MAB), MAB->createObjectWriter(Out), std::move(MCE),
-      STI, /*ignore=*/false, false, false));
+      T, *Ctx, std::unique_ptr<MCAsmBackend>(MAB), MAB->createObjectWriter(Out),
+      std::move(MCE), STI));
 
   // Create the AsmPrinter, which takes ownership of AsmStreamer if successful.
   FunctionPass *Printer =
diff --git a/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp b/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
index 74df2eb9d68ae..55a0ee81ad4bc 100644
--- a/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
+++ b/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
@@ -110,8 +110,7 @@ Error DwarfStreamer::init(Triple TheTriple,
     MS = TheTarget->createMCObjectStreamer(
         TheTriple, *MC, std::unique_ptr<MCAsmBackend>(MAB),
         MAB->createObjectWriter(OutFile), std::unique_ptr<MCCodeEmitter>(MCE),
-        *MSTI, MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible,
-        /*DWARFMustBeAtTheEnd*/ false);
+        *MSTI);
     break;
   }
   }
diff --git a/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp b/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp
index b9edcb63a3401..dd379de68e4c4 100644
--- a/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp
+++ b/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp
@@ -89,8 +89,7 @@ Error DwarfEmitterImpl::init(Triple TheTriple,
     MS = TheTarget->createMCObjectStreamer(
         TheTriple, *MC, std::unique_ptr<MCAsmBackend>(MAB),
         MAB->createObjectWriter(OutFile), std::unique_ptr<MCCodeEmitter>(MCE),
-        *MSTI, MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible,
-        /*DWARFMustBeAtTheEnd*/ false);
+        *MSTI);
     break;
   }
   }
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index 18f4f1a0eb9fb..60a89cb13c57a 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -266,9 +266,8 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
 
   std::unique_ptr<MCStreamer> MS(TheTarget->createMCObjectStreamer(
       *ErrOrTriple, MC, std::unique_ptr<MCAsmBackend>(MAB),
-      MAB->createObjectWriter(*OS), std::unique_ptr<MCCodeEmitter>(MCE), *MSTI,
-      MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible,
-      /*DWARFMustBeAtTheEnd*/ false));
+      MAB->createObjectWriter(*OS), std::unique_ptr<MCCodeEmitter>(MCE),
+      *MSTI));
   if (!MS)
     return error("no object streamer for target " + TripleName, Context);
 
diff --git a/llvm/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp b/llvm/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp
index 6ec19a367d58b..ef9d0f37198ac 100644
--- a/llvm/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp
+++ b/llvm/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp
@@ -233,9 +233,8 @@ int AssembleOneInput(const uint8_t *Data, size_t Size) {
     MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions);
     Str.reset(TheTarget->createMCObjectStreamer(
         TheTriple, Ctx, std::unique_ptr<MCAsmBackend>(MAB),
-        MAB->createObjectWriter(*OS), std::unique_ptr<MCCodeEmitter>(CE), *STI,
-        MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible,
-        /*DWARFMustBeAtTheEnd*/ false));
+        MAB->createObjectWriter(*OS), std::unique_ptr<MCCodeEmitter>(CE),
+        *STI));
   }
   const int Res = AssembleInput(ProgName, TheTarget, SrcMgr, Ctx, *Str, *MAI, *STI,
       *MCII, MCOptions);
diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp
index de999a48d5753..c46e4f087d42b 100644
--- a/llvm/tools/llvm-mc/llvm-mc.cpp
+++ b/llvm/tools/llvm-mc/llvm-mc.cpp
@@ -555,9 +555,7 @@ int main(int argc, char **argv) {
         TheTriple, Ctx, std::unique_ptr<MCAsmBackend>(MAB),
         DwoOut ? MAB->createDwoObjectWriter(*OS, DwoOut->os())
                : MAB->createObjectWriter(*OS),
-        std::unique_ptr<MCCodeEmitter>(CE), *STI, MCOptions.MCRelaxAll,
-        MCOptions.MCIncrementalLinkerCompatible,
-        /*DWARFMustBeAtTheEnd*/ false));
+        std::unique_ptr<MCCodeEmitter>(CE), *STI));
     if (NoExecStack)
       Str->initSections(true, *STI);
   }
diff --git a/llvm/tools/llvm-ml/llvm-ml.cpp b/llvm/tools/llvm-ml/llvm-ml.cpp
index 24643bd4296be..829347dafee98 100644
--- a/llvm/tools/llvm-ml/llvm-ml.cpp
+++ b/llvm/tools/llvm-ml/llvm-ml.cpp
@@ -402,9 +402,8 @@ int llvm_ml_main(int Argc, char **Argv, const llvm::ToolContext &) {
     MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions);
     Str.reset(TheTarget->createMCObjectStreamer(
         TheTriple, Ctx, std::unique_ptr<MCAsmBackend>(MAB),
-        MAB->createObjectWriter(*OS), std::unique_ptr<MCCodeEmitter>(CE), *STI,
-        MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible,
-        /*DWARFMustBeAtTheEnd*/ false));
+        MAB->createObjectWriter(*OS), std::unique_ptr<MCCodeEmitter>(CE),
+        *STI));
   } else {
     llvm_unreachable("Invalid file type!");
   }
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCopyBytesTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCopyBytesTest.cpp
index 43fdf5d3d6f31..ec9c0dddcbc0c 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCopyBytesTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCopyBytesTest.cpp
@@ -112,10 +112,7 @@ DWARFExpressionCopyBytesTest::createStreamer(raw_pwrite_stream &OS) {
   std::unique_ptr<MCObjectWriter> OW = MAB->createObjectWriter(OS);
   Res.Streamer.reset(TheTarget->createMCObjectStreamer(
       Triple(TripleName), *Res.Ctx, std::unique_ptr<MCAsmBackend>(MAB),
-      std::move(OW), std::unique_ptr<MCCodeEmitter>(MCE), *STI,
-      /* RelaxAll */ false,
-      /* IncrementalLinkerCompatible */ false,
-      /* DWARFMustBeAtTheEnd */ false));
+      std::move(OW), std::unique_ptr<MCCodeEmitter>(MCE), *STI));
   return Res;
 }
 
diff --git a/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp b/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
index ad5e51b7efb83..2cbd4cc55df95 100644
--- a/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
@@ -503,8 +503,7 @@ llvm::Error dwarfgen::Generator::init(Triple TheTriple, uint16_t V) {
   MS = TheTarget->createMCObjectStreamer(
       TheTriple, *MC, std::unique_ptr<MCAsmBackend>(MAB),
       MAB->createObjectWriter(*Stream), std::unique_ptr<MCCodeEmitter>(MCE),
-      *MSTI, MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible,
-      /*DWARFMustBeAtTheEnd*/ false);
+      *MSTI);
   if (!MS)
     return make_error<StringError>("no object streamer for target " +
                                        TripleName,
diff --git a/llvm/unittests/MC/DwarfLineTableHeaders.cpp b/llvm/unittests/MC/DwarfLineTableHeaders.cpp
index d8a657ed5048e..1fad1ba6ce638 100644
--- a/llvm/unittests/MC/DwarfLineTableHeaders.cpp
+++ b/llvm/unittests/MC/DwarfLineTableHeaders.cpp
@@ -83,10 +83,7 @@ class DwarfLineTableHeaders : public ::testing::Test {
     std::unique_ptr<MCObjectWriter> OW = MAB->createObjectWriter(OS);
     Res.Streamer.reset(TheTarget->createMCObjectStreamer(
         Triple(TripleName), *Res.Ctx, std::unique_ptr<MCAsmBackend>(MAB),
-        std::move(OW), std::unique_ptr<MCCodeEmitter>(MCE), *STI,
-        /* RelaxAll */ false,
-        /* IncrementalLinkerCompatible */ false,
-        /* DWARFMustBeAtTheEnd */ false));
+        std::move(OW), std::unique_ptr<MCCodeEmitter>(MCE), *STI));
     return Res;
   }
 
diff --git a/mlir/lib/Target/LLVM/ROCDL/Target.cpp b/mlir/lib/Target/LLVM/ROCDL/Target.cpp
index 70d6bcd76285a..4d23f987eb05e 100644
--- a/mlir/lib/Target/LLVM/ROCDL/Target.cpp
+++ b/mlir/lib/Target/LLVM/ROCDL/Target.cpp
@@ -324,8 +324,7 @@ SerializeGPUModuleBase::assembleIsa(StringRef isa) {
   mcStreamer.reset(target->createMCObjectStreamer(
       triple, ctx, std::unique_ptr<llvm::MCAsmBackend>(mab),
       mab->createObjectWriter(os), std::unique_ptr<llvm::MCCodeEmitter>(ce),
-      *sti, mcOptions.MCRelaxAll, mcOptions.MCIncrementalLinkerCompatible,
-      /*DWARFMustBeAtTheEnd*/ false));
+      *sti));
 
   std::unique_ptr<llvm::MCAsmParser> parser(
       createMCAsmParser(srcMgr, ctx, *mcStreamer, *mai));

From 6c9086d13fa7e1069e75ed2d139aae30ee3863c8 Mon Sep 17 00:00:00 2001
From: antangelo <contact@antangelo.com>
Date: Sun, 21 Jul 2024 00:29:18 -0400
Subject: [PATCH 323/486] [AArch64] Support varargs for preserve_nonecc
 (#99434)

Adds varargs support for preserve_none by falling back to C argument
passing for the target platform for varargs functions.

Fixes #95093
---
 .../AArch64/AArch64CallingConvention.td       |  73 ++++++-----
 .../Target/AArch64/AArch64FrameLowering.cpp   |  17 ++-
 .../Target/AArch64/AArch64ISelLowering.cpp    |  39 +++++-
 .../Target/AArch64/AArch64RegisterInfo.cpp    |   7 +-
 llvm/lib/Target/AArch64/AArch64Subtarget.h    |   4 +-
 .../AArch64/GISel/AArch64CallLowering.cpp     |  12 +-
 .../GISel/AArch64InstructionSelector.cpp      |   2 +-
 .../AArch64/preserve_nonecc_varargs_aapcs.ll  | 123 ++++++++++++++++++
 .../AArch64/preserve_nonecc_varargs_darwin.ll |  67 ++++++++++
 .../AArch64/preserve_nonecc_varargs_win64.ll  |  69 ++++++++++
 10 files changed, 359 insertions(+), 54 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_aapcs.ll
 create mode 100644 llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_darwin.ll
 create mode 100644 llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_win64.ll

diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.td b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
index 6f885f4588c4b..2bbb4997d56a5 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.td
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
@@ -17,6 +17,11 @@ class CCIfBigEndian<CCAction A> :
 class CCIfILP32<CCAction A> :
   CCIf<"State.getMachineFunction().getDataLayout().getPointerSize() == 4", A>;
 
+/// CCIfSubtarget - Match if the current subtarget has a feature F.
+class CCIfSubtarget<string F, CCAction A>
+    : CCIf<!strconcat("State.getMachineFunction()"
+                      ".getSubtarget<AArch64Subtarget>().", F),
+           A>;
 
 //===----------------------------------------------------------------------===//
 // ARM AAPCS64 Calling Convention
@@ -496,36 +501,44 @@ def CC_AArch64_GHC : CallingConv<[
 
 let Entry = 1 in
 def CC_AArch64_Preserve_None : CallingConv<[
-    // We can pass arguments in all general registers, except:
-    // - X8, used for sret
-    // - X16/X17, used by the linker as IP0/IP1
-    // - X18, the platform register
-    // - X19, the base pointer
-    // - X29, the frame pointer
-    // - X30, the link register
-    // General registers are not preserved with the exception of
-    // FP, LR, and X18
-    // Non-volatile registers are used first, so functions may call
-    // normal functions without saving and reloading arguments.
-    // X9 is assigned last as it is used in FrameLowering as the first
-    // choice for a scratch register.
-    CCIfType<[i32], CCAssignToReg<[W20, W21, W22, W23,
-                                   W24, W25, W26, W27, W28,
-                                   W0, W1, W2, W3, W4, W5,
-                                   W6, W7, W10, W11,
-                                   W12, W13, W14, W9]>>,
-    CCIfType<[i64], CCAssignToReg<[X20, X21, X22, X23,
-                                   X24, X25, X26, X27, X28,
-                                   X0, X1, X2, X3, X4, X5,
-                                   X6, X7, X10, X11,
-                                   X12, X13, X14, X9]>>,
-
-    // Windows uses X15 for stack allocation
-    CCIf<"!State.getMachineFunction().getSubtarget<AArch64Subtarget>().isTargetWindows()",
-        CCIfType<[i32], CCAssignToReg<[W15]>>>,
-    CCIf<"!State.getMachineFunction().getSubtarget<AArch64Subtarget>().isTargetWindows()",
-        CCIfType<[i64], CCAssignToReg<[X15]>>>,
-    CCDelegateTo<CC_AArch64_AAPCS>
+  // VarArgs are only supported using the C calling convention.
+  // This handles the non-variadic parameter case. Variadic parameters
+  // are handled in CCAssignFnForCall.
+  CCIfVarArg<CCIfSubtarget<"isTargetDarwin()", CCDelegateTo<CC_AArch64_DarwinPCS>>>,
+  CCIfVarArg<CCIfSubtarget<"isTargetWindows()", CCDelegateTo<CC_AArch64_Win64PCS>>>,
+  CCIfVarArg<CCDelegateTo<CC_AArch64_AAPCS>>,
+
+  // We can pass arguments in all general registers, except:
+  // - X8, used for sret
+  // - X16/X17, used by the linker as IP0/IP1
+  // - X18, the platform register
+  // - X19, the base pointer
+  // - X29, the frame pointer
+  // - X30, the link register
+  // General registers are not preserved with the exception of
+  // FP, LR, and X18
+  // Non-volatile registers are used first, so functions may call
+  // normal functions without saving and reloading arguments.
+  // X9 is assigned last as it is used in FrameLowering as the first
+  // choice for a scratch register.
+  CCIfType<[i32], CCAssignToReg<[W20, W21, W22, W23,
+                                 W24, W25, W26, W27, W28,
+                                 W0, W1, W2, W3, W4, W5,
+                                 W6, W7, W10, W11,
+                                 W12, W13, W14, W9]>>,
+  CCIfType<[i64], CCAssignToReg<[X20, X21, X22, X23,
+                                 X24, X25, X26, X27, X28,
+                                 X0, X1, X2, X3, X4, X5,
+                                 X6, X7, X10, X11,
+                                 X12, X13, X14, X9]>>,
+
+  // Windows uses X15 for stack allocation
+  CCIf<"!State.getMachineFunction().getSubtarget<AArch64Subtarget>().isTargetWindows()",
+    CCIfType<[i32], CCAssignToReg<[W15]>>>,
+  CCIf<"!State.getMachineFunction().getSubtarget<AArch64Subtarget>().isTargetWindows()",
+    CCIfType<[i64], CCAssignToReg<[X15]>>>,
+
+  CCDelegateTo<CC_AArch64_AAPCS>
 ]>;
 
 // The order of the callee-saves in this file is important, because the
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 8c58f0bdf2f53..b1b83e27c5592 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1900,8 +1900,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
     return;
   }
 
-  bool IsWin64 =
-      Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
+  bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg());
   unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
 
   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
@@ -2307,8 +2306,8 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
   // How much of the stack used by incoming arguments this function is expected
   // to restore in this particular epilogue.
   int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB);
-  bool IsWin64 =
-      Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
+  bool IsWin64 = Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv(),
+                                              MF.getFunction().isVarArg());
   unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
 
   int64_t AfterCSRPopSize = ArgumentStackToRestore;
@@ -2614,8 +2613,8 @@ static StackOffset getFPOffset(const MachineFunction &MF,
                                int64_t ObjectOffset) {
   const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
   const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
-  bool IsWin64 =
-      Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
+  const Function &F = MF.getFunction();
+  bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg());
   unsigned FixedObject =
       getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
   int64_t CalleeSaveSize = AFI->getCalleeSavedStackSize(MF.getFrameInfo());
@@ -2721,9 +2720,9 @@ StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
         // via the frame pointer, so we have to use the FP in the parent
         // function.
         (void) Subtarget;
-        assert(
-            Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
-            "Funclets should only be present on Win64");
+        assert(Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv(),
+                                            MF.getFunction().isVarArg()) &&
+               "Funclets should only be present on Win64");
         UseFP = true;
       } else {
         // We have the choice between FP and (SP or BP).
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 84de1ee8f8923..bf205b1706a6c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -7109,7 +7109,13 @@ CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
   case CallingConv::GHC:
     return CC_AArch64_GHC;
   case CallingConv::PreserveNone:
-    return CC_AArch64_Preserve_None;
+    // The VarArg implementation makes assumptions about register
+    // argument passing that do not hold for preserve_none, so we
+    // instead fall back to C argument passing.
+    // The non-vararg case is handled in the CC function itself.
+    if (!IsVarArg)
+      return CC_AArch64_Preserve_None;
+    [[fallthrough]];
   case CallingConv::C:
   case CallingConv::Fast:
   case CallingConv::PreserveMost:
@@ -7182,7 +7188,8 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
   MachineFunction &MF = DAG.getMachineFunction();
   const Function &F = MF.getFunction();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  bool IsWin64 = Subtarget->isCallingConvWin64(F.getCallingConv());
+  bool IsWin64 =
+      Subtarget->isCallingConvWin64(F.getCallingConv(), F.isVarArg());
   bool StackViaX4 = CallConv == CallingConv::ARM64EC_Thunk_X64 ||
                     (isVarArg && Subtarget->isWindowsArm64EC());
   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
@@ -7634,7 +7641,9 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
   auto PtrVT = getPointerTy(DAG.getDataLayout());
-  bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
+  Function &F = MF.getFunction();
+  bool IsWin64 =
+      Subtarget->isCallingConvWin64(F.getCallingConv(), F.isVarArg());
 
   SmallVector<SDValue, 8> MemOps;
 
@@ -7805,6 +7814,21 @@ static bool mayTailCallThisCC(CallingConv::ID CC) {
   }
 }
 
+/// Return true if the call convention supports varargs
+/// Currently only those that pass varargs like the C
+/// calling convention does are eligible
+/// Calling conventions listed in this function must also
+/// be properly handled in AArch64Subtarget::isCallingConvWin64
+static bool callConvSupportsVarArgs(CallingConv::ID CC) {
+  switch (CC) {
+  case CallingConv::C:
+  case CallingConv::PreserveNone:
+    return true;
+  default:
+    return false;
+  }
+}
+
 static void analyzeCallOperands(const AArch64TargetLowering &TLI,
                                 const AArch64Subtarget *Subtarget,
                                 const TargetLowering::CallLoweringInfo &CLI,
@@ -7813,7 +7837,7 @@ static void analyzeCallOperands(const AArch64TargetLowering &TLI,
   CallingConv::ID CalleeCC = CLI.CallConv;
   bool IsVarArg = CLI.IsVarArg;
   const SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
-  bool IsCalleeWin64 = Subtarget->isCallingConvWin64(CalleeCC);
+  bool IsCalleeWin64 = Subtarget->isCallingConvWin64(CalleeCC, IsVarArg);
 
   // For Arm64EC thunks, allocate 32 extra bytes at the bottom of the stack
   // for the shadow store.
@@ -7941,8 +7965,8 @@ bool AArch64TargetLowering::isEligibleForTailCallOptimization(
 
   // I want anyone implementing a new calling convention to think long and hard
   // about this assert.
-  assert((!IsVarArg || CalleeCC == CallingConv::C) &&
-         "Unexpected variadic calling convention");
+  if (IsVarArg && !callConvSupportsVarArgs(CalleeCC))
+    report_fatal_error("Unsupported variadic calling convention");
 
   LLVMContext &C = *DAG.getContext();
   // Check that the call results are passed in the same way.
@@ -10872,8 +10896,9 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
 SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
                                             SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
+  Function &F = MF.getFunction();
 
-  if (Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv()))
+  if (Subtarget->isCallingConvWin64(F.getCallingConv(), F.isVarArg()))
     return LowerWin64_VASTART(Op, DAG);
   else if (Subtarget->isTargetDarwin())
     return LowerDarwin_VASTART(Op, DAG);
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 1e069f4790c53..435cc18cdea62 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -611,7 +611,8 @@ bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF,
                                              MCRegister Reg) const {
   CallingConv::ID CC = MF.getFunction().getCallingConv();
   const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
-  bool IsVarArg = STI.isCallingConvWin64(MF.getFunction().getCallingConv());
+  bool IsVarArg = STI.isCallingConvWin64(MF.getFunction().getCallingConv(),
+                                         MF.getFunction().isVarArg());
 
   auto HasReg = [](ArrayRef<MCRegister> RegList, MCRegister Reg) {
     return llvm::is_contained(RegList, Reg);
@@ -623,7 +624,9 @@ bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF,
   case CallingConv::GHC:
     return HasReg(CC_AArch64_GHC_ArgRegs, Reg);
   case CallingConv::PreserveNone:
-    return HasReg(CC_AArch64_Preserve_None_ArgRegs, Reg);
+    if (!MF.getFunction().isVarArg())
+      return HasReg(CC_AArch64_Preserve_None_ArgRegs, Reg);
+    [[fallthrough]];
   case CallingConv::C:
   case CallingConv::Fast:
   case CallingConv::PreserveMost:
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 4b840b24ba134..12c3d25d32ee7 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -322,13 +322,15 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
 
   std::unique_ptr<PBQPRAConstraint> getCustomPBQPConstraints() const override;
 
-  bool isCallingConvWin64(CallingConv::ID CC) const {
+  bool isCallingConvWin64(CallingConv::ID CC, bool IsVarArg) const {
     switch (CC) {
     case CallingConv::C:
     case CallingConv::Fast:
     case CallingConv::Swift:
     case CallingConv::SwiftTail:
       return isTargetWindows();
+    case CallingConv::PreserveNone:
+      return IsVarArg && isTargetWindows();
     case CallingConv::Win64:
       return true;
     default:
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index 5206ba46260ed..b4d2a3388c1df 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -117,7 +117,9 @@ struct AArch64OutgoingValueAssigner
                  CCValAssign::LocInfo LocInfo,
                  const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
                  CCState &State) override {
-    bool IsCalleeWin = Subtarget.isCallingConvWin64(State.getCallingConv());
+    const Function &F = State.getMachineFunction().getFunction();
+    bool IsCalleeWin =
+        Subtarget.isCallingConvWin64(State.getCallingConv(), F.isVarArg());
     bool UseVarArgsCCForFixed = IsCalleeWin && State.isVarArg();
 
     bool Res;
@@ -557,8 +559,8 @@ void AArch64CallLowering::saveVarArgRegisters(
   MachineFrameInfo &MFI = MF.getFrameInfo();
   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
   auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
-  bool IsWin64CC =
-      Subtarget.isCallingConvWin64(CCInfo.getCallingConv());
+  bool IsWin64CC = Subtarget.isCallingConvWin64(CCInfo.getCallingConv(),
+                                                MF.getFunction().isVarArg());
   const LLT p0 = LLT::pointer(0, 64);
   const LLT s64 = LLT::scalar(64);
 
@@ -653,7 +655,9 @@ bool AArch64CallLowering::lowerFormalArguments(
       F.getCallingConv() == CallingConv::ARM64EC_Thunk_X64)
     return false;
 
-  bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv()) && !Subtarget.isWindowsArm64EC();
+  bool IsWin64 =
+      Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg()) &&
+      !Subtarget.isWindowsArm64EC();
 
   SmallVector<ArgInfo, 8> SplitArgs;
   SmallVector<std::pair<Register, Register>> BoolArgs;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 0d3f6d9e353ba..009928a8a7488 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2006,7 +2006,7 @@ bool AArch64InstructionSelector::selectVaStartDarwin(
 
   int FrameIdx = FuncInfo->getVarArgsStackIndex();
   if (MF.getSubtarget<AArch64Subtarget>().isCallingConvWin64(
-          MF.getFunction().getCallingConv())) {
+          MF.getFunction().getCallingConv(), MF.getFunction().isVarArg())) {
     FrameIdx = FuncInfo->getVarArgsGPRSize() > 0
                    ? FuncInfo->getVarArgsGPRIndex()
                    : FuncInfo->getVarArgsStackIndex();
diff --git a/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_aapcs.ll b/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_aapcs.ll
new file mode 100644
index 0000000000000..48898719f40ce
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_aapcs.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s
+
+%va_list = type { ptr, ptr, ptr, i32, i32 }
+
+define preserve_nonecc i32 @callee(i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, ...) nounwind noinline ssp {
+; CHECK-LABEL: callee:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #192
+; CHECK-NEXT:    mov x8, #-24 // =0xffffffffffffffe8
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    add x10, sp, #136
+; CHECK-NEXT:    movk x8, #65408, lsl #32
+; CHECK-NEXT:    add x9, x9, #128
+; CHECK-NEXT:    stp x6, x7, [sp, #144]
+; CHECK-NEXT:    stp x9, x8, [sp, #176]
+; CHECK-NEXT:    add x9, x10, #24
+; CHECK-NEXT:    add x10, sp, #192
+; CHECK-NEXT:    mov w8, #-24 // =0xffffffe8
+; CHECK-NEXT:    str x5, [sp, #136]
+; CHECK-NEXT:    stp q0, q1, [sp]
+; CHECK-NEXT:    stp q2, q3, [sp, #32]
+; CHECK-NEXT:    stp q4, q5, [sp, #64]
+; CHECK-NEXT:    stp q6, q7, [sp, #96]
+; CHECK-NEXT:    stp x10, x9, [sp, #160]
+; CHECK-NEXT:    tbz w8, #31, .LBB0_3
+; CHECK-NEXT:  // %bb.1: // %maybe_reg
+; CHECK-NEXT:    add w9, w8, #8
+; CHECK-NEXT:    cmp w9, #0
+; CHECK-NEXT:    str w9, [sp, #184]
+; CHECK-NEXT:    b.gt .LBB0_3
+; CHECK-NEXT:  // %bb.2: // %in_reg
+; CHECK-NEXT:    ldr x9, [sp, #168]
+; CHECK-NEXT:    add x8, x9, w8, sxtw
+; CHECK-NEXT:    b .LBB0_4
+; CHECK-NEXT:  .LBB0_3: // %on_stack
+; CHECK-NEXT:    ldr x8, [sp, #160]
+; CHECK-NEXT:    add x9, x8, #8
+; CHECK-NEXT:    str x9, [sp, #160]
+; CHECK-NEXT:  .LBB0_4: // %end
+; CHECK-NEXT:    ldr w0, [x8]
+; CHECK-NEXT:    add sp, sp, #192
+; CHECK-NEXT:    ret
+entry:
+  %args = alloca %va_list, align 8
+  call void @llvm.va_start(ptr %args)
+  %gr_offs_p = getelementptr inbounds %va_list, ptr %args, i32 0, i32 3
+  %gr_offs = load i32, ptr %gr_offs_p, align 8
+  %0 = icmp sge i32 %gr_offs, 0
+  br i1 %0, label %on_stack, label %maybe_reg
+
+maybe_reg:
+  %new_reg_offs = add i32 %gr_offs, 8
+  store i32 %new_reg_offs, ptr %gr_offs_p, align 8
+  %inreg = icmp sle i32 %new_reg_offs, 0
+  br i1 %inreg, label %in_reg, label %on_stack
+
+in_reg:
+  %reg_top_p = getelementptr inbounds %va_list, ptr %args, i32 0, i32 1
+  %reg_top = load ptr, ptr %reg_top_p, align 8
+  %reg = getelementptr inbounds i8, ptr %reg_top, i32 %gr_offs
+  br label %end
+
+on_stack:
+  %stack_p = getelementptr inbounds %va_list, ptr %args, i32 0, i32 0
+  %stack = load ptr, ptr %stack_p, align 8
+  %new_stack = getelementptr inbounds i8, ptr %stack, i64 8
+  store ptr %new_stack, ptr %stack_p, align 8
+  br label %end
+
+end:
+  %p = phi ptr [ %reg, %in_reg ], [ %stack, %on_stack ]
+  %10 = load i32, ptr %p, align 8
+  call void @llvm.va_end.p0(ptr %args)
+  ret i32 %10
+}
+
+declare void @llvm.va_start(ptr) nounwind
+declare void @llvm.va_end(ptr) nounwind
+
+define i32 @caller() nounwind ssp {
+; CHECK-LABEL: caller:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #176
+; CHECK-NEXT:    mov w8, #10 // =0xa
+; CHECK-NEXT:    mov w9, #9 // =0x9
+; CHECK-NEXT:    mov w0, #1 // =0x1
+; CHECK-NEXT:    mov w1, #2 // =0x2
+; CHECK-NEXT:    mov w2, #3 // =0x3
+; CHECK-NEXT:    mov w3, #4 // =0x4
+; CHECK-NEXT:    mov w4, #5 // =0x5
+; CHECK-NEXT:    mov w5, #6 // =0x6
+; CHECK-NEXT:    mov w6, #7 // =0x7
+; CHECK-NEXT:    mov w7, #8 // =0x8
+; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x28, x27, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x26, x25, [sp, #112] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x24, x23, [sp, #128] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #144] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #160] // 16-byte Folded Spill
+; CHECK-NEXT:    str w8, [sp, #8]
+; CHECK-NEXT:    str w9, [sp]
+; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    ldp x20, x19, [sp, #160] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x22, x21, [sp, #144] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x24, x23, [sp, #128] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x26, x25, [sp, #112] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x28, x27, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #176
+; CHECK-NEXT:    ret
+  %r = tail call preserve_nonecc i32 (i32, i32, i32, i32, i32, ...) @callee(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10)
+  ret i32 %r
+}
+
diff --git a/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_darwin.ll b/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_darwin.ll
new file mode 100644
index 0000000000000..e227f14542cc1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_darwin.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-apple-darwin < %s | FileCheck %s
+
+define preserve_nonecc i32 @callee(i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, ...) nounwind noinline ssp {
+; CHECK-LABEL: callee:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    add x8, sp, #16
+; CHECK-NEXT:    ldr w0, [sp, #16]
+; CHECK-NEXT:    orr x8, x8, #0x8
+; CHECK-NEXT:    str x8, [sp, #8]
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ret
+  %args = alloca ptr, align 8
+  call void @llvm.va_start(ptr %args)
+  %10 = va_arg ptr %args, i32
+  call void @llvm.va_end(ptr %args)
+  ret i32 %10
+}
+
+declare void @llvm.va_start(ptr) nounwind
+declare void @llvm.va_end(ptr) nounwind
+
+define i32 @caller() nounwind ssp {
+; CHECK-LABEL: caller:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub sp, sp, #208
+; CHECK-NEXT:    mov w8, #10 ; =0xa
+; CHECK-NEXT:    mov w9, #9 ; =0x9
+; CHECK-NEXT:    mov w0, #1 ; =0x1
+; CHECK-NEXT:    stp x9, x8, [sp, #24]
+; CHECK-NEXT:    mov w8, #8 ; =0x8
+; CHECK-NEXT:    mov w9, #6 ; =0x6
+; CHECK-NEXT:    str x8, [sp, #16]
+; CHECK-NEXT:    mov w8, #7 ; =0x7
+; CHECK-NEXT:    mov w1, #2 ; =0x2
+; CHECK-NEXT:    mov w2, #3 ; =0x3
+; CHECK-NEXT:    mov w3, #4 ; =0x4
+; CHECK-NEXT:    mov w4, #5 ; =0x5
+; CHECK-NEXT:    stp d15, d14, [sp, #48] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #64] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #80] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #96] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x28, x27, [sp, #112] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x26, x25, [sp, #128] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x24, x23, [sp, #144] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #160] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #176] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #192] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x9, x8, [sp]
+; CHECK-NEXT:    bl _callee
+; CHECK-NEXT:    ldp x29, x30, [sp, #192] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #176] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x22, x21, [sp, #160] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x24, x23, [sp, #144] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x26, x25, [sp, #128] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x28, x27, [sp, #112] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp d9, d8, [sp, #96] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #80] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #64] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp, #48] ; 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #208
+; CHECK-NEXT:    ret
+  %r = tail call preserve_nonecc i32 (i32, i32, i32, i32, i32, ...) @callee(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10)
+  ret i32 %r
+}
+
diff --git a/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_win64.ll b/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_win64.ll
new file mode 100644
index 0000000000000..83dd240a6540f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_win64.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-pc-windows < %s | FileCheck %s
+
+define preserve_nonecc i32 @callee(i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, ...) nounwind noinline ssp {
+; CHECK-LABEL: callee:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    mov x0, x5
+; CHECK-NEXT:    add x8, sp, #24
+; CHECK-NEXT:    stp x6, x7, [sp, #32]
+; CHECK-NEXT:    str x5, [sp, #24]
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    str x8, [sp, #8]
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+  %args = alloca ptr, align 8
+  call void @llvm.va_start(ptr %args)
+  %p = load ptr, ptr %args, align 8
+  %10 = load i32, ptr %p, align 8
+  call void @llvm.va_end(ptr %args)
+  ret i32 %10
+}
+
+declare void @llvm.va_start(ptr) nounwind
+declare void @llvm.va_end(ptr) nounwind
+
+define i32 @caller() nounwind ssp {
+; CHECK-LABEL: caller:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #176
+; CHECK-NEXT:    mov w8, #10 // =0xa
+; CHECK-NEXT:    mov w9, #9 // =0x9
+; CHECK-NEXT:    mov w0, #1 // =0x1
+; CHECK-NEXT:    mov w1, #2 // =0x2
+; CHECK-NEXT:    mov w2, #3 // =0x3
+; CHECK-NEXT:    mov w3, #4 // =0x4
+; CHECK-NEXT:    mov w4, #5 // =0x5
+; CHECK-NEXT:    mov w5, #6 // =0x6
+; CHECK-NEXT:    mov w6, #7 // =0x7
+; CHECK-NEXT:    mov w7, #8 // =0x8
+; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x28, x27, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x26, x25, [sp, #112] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x24, x23, [sp, #128] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #144] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #160] // 16-byte Folded Spill
+; CHECK-NEXT:    str w8, [sp, #8]
+; CHECK-NEXT:    str w9, [sp]
+; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    ldp x20, x19, [sp, #160] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x22, x21, [sp, #144] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x24, x23, [sp, #128] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x26, x25, [sp, #112] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x28, x27, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #176
+; CHECK-NEXT:    ret
+  %r = tail call preserve_nonecc i32 (i32, i32, i32, i32, i32, ...) @callee(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10)
+  ret i32 %r
+}
+

From 86e21e1af24f35bab4c34a3e0b7b5846126c3e9f Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 21:30:49 -0700
Subject: [PATCH 324/486] [BOLT] Remove unused bool arguments from
 createMCObjectStreamer callers

---
 bolt/include/bolt/Core/BinaryContext.h | 5 +----
 bolt/lib/Core/BinaryContext.cpp        | 5 +----
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index de9ba09a5bb49..b3cf9f834cc08 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -1452,10 +1452,7 @@ class BinaryContext {
     std::unique_ptr<MCObjectWriter> OW = MAB->createObjectWriter(OS);
     std::unique_ptr<MCStreamer> Streamer(TheTarget->createMCObjectStreamer(
         *TheTriple, *Ctx, std::unique_ptr<MCAsmBackend>(MAB), std::move(OW),
-        std::unique_ptr<MCCodeEmitter>(MCE), *STI,
-        /* RelaxAll */ false,
-        /* IncrementalLinkerCompatible */ false,
-        /* DWARFMustBeAtTheEnd */ false));
+        std::unique_ptr<MCCodeEmitter>(MCE), *STI));
     return Streamer;
   }
 
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 035f68e39751b..83a5484f097ef 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -2367,10 +2367,7 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
   std::unique_ptr<MCObjectWriter> OW = MAB->createObjectWriter(VecOS);
   std::unique_ptr<MCStreamer> Streamer(TheTarget->createMCObjectStreamer(
       *TheTriple, *LocalCtx, std::unique_ptr<MCAsmBackend>(MAB), std::move(OW),
-      std::unique_ptr<MCCodeEmitter>(MCEInstance.MCE.release()), *STI,
-      /*RelaxAll=*/false,
-      /*IncrementalLinkerCompatible=*/false,
-      /*DWARFMustBeAtTheEnd=*/false));
+      std::unique_ptr<MCCodeEmitter>(MCEInstance.MCE.release()), *STI));
 
   Streamer->initSections(false, *STI);
 

From f1422a86c4a812a7ccd744082741841e596ccea0 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 21:46:07 -0700
Subject: [PATCH 325/486] [MC] Deprecate createMCObjectStreamer with 3 unused
 trailing bool

---
 llvm/include/llvm/MC/TargetRegistry.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index e09318ba18f5a..69658141355f5 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -23,6 +23,7 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/TargetParser/Triple.h"
@@ -530,6 +531,7 @@ class Target {
                                      std::unique_ptr<MCObjectWriter> OW,
                                      std::unique_ptr<MCCodeEmitter> Emitter,
                                      const MCSubtargetInfo &STI) const;
+  LLVM_DEPRECATED("Use the overload without the 3 trailing bool", "")
   MCStreamer *createMCObjectStreamer(const Triple &T, MCContext &Ctx,
                                      std::unique_ptr<MCAsmBackend> &&TAB,
                                      std::unique_ptr<MCObjectWriter> &&OW,

From aefe411dae156350721268161e6cac36a8d29333 Mon Sep 17 00:00:00 2001
From: WANG Rui <wangrui@loongson.cn>
Date: Sun, 21 Jul 2024 13:41:43 +0800
Subject: [PATCH 326/486] [LoongArch] Add a test for spurious mask removal. NFC

Link: https://github.com/llvm/llvm-project/pull/99272#issuecomment-2241348794
---
 llvm/test/CodeGen/LoongArch/andn-icmp.ll | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/llvm/test/CodeGen/LoongArch/andn-icmp.ll b/llvm/test/CodeGen/LoongArch/andn-icmp.ll
index 4fc3c8df4664c..447f3ac5c34fd 100644
--- a/llvm/test/CodeGen/LoongArch/andn-icmp.ll
+++ b/llvm/test/CodeGen/LoongArch/andn-icmp.ll
@@ -601,3 +601,25 @@ define i1 @andn_icmp_ult_i8_nn(i8 %a, i8 %b) nounwind {
   %cmp = icmp ult i8 %and, %b
   ret i1 %cmp
 }
+
+define i1 @andn_icmp_eq_i8_i32(i8 signext %a, i8 signext %b) nounwind {
+; LA32-LABEL: andn_icmp_eq_i8_i32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    andn $a0, $a1, $a0
+; LA32-NEXT:    andi $a0, $a0, 255
+; LA32-NEXT:    sltui $a0, $a0, 1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: andn_icmp_eq_i8_i32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andn $a0, $a1, $a0
+; LA64-NEXT:    andi $a0, $a0, 255
+; LA64-NEXT:    sltui $a0, $a0, 1
+; LA64-NEXT:    ret
+  %x = zext i8 %a to i32
+  %y = zext i8 %b to i32
+  %not = xor i32 %x, -1
+  %and = and i32 %not, %y
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}

From 52e79ed1e078f69103dcd9234c2494e7dd64b5f7 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 22:57:01 -0700
Subject: [PATCH 327/486] [MC] MCAsmStreamer: use MCTargetOptions

Some bool parameters duplicate MCTargetOptions and might cause
inconsistency/confusion.
---
 clang/tools/driver/cc1as_main.cpp             |  3 ++
 llvm/lib/CodeGen/LLVMTargetMachine.cpp        | 17 +--------
 .../lib/DWARFLinker/Classic/DWARFStreamer.cpp |  2 +
 .../DWARFLinker/Parallel/DWARFEmitterImpl.cpp |  2 +
 llvm/lib/MC/MCAsmStreamer.cpp                 | 37 ++++++++++++++-----
 llvm/tools/llvm-mc/llvm-mc.cpp                |  3 ++
 llvm/tools/llvm-ml/llvm-ml.cpp                |  1 +
 7 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index 2d5acbecb123a..15d1e0c2f2f2d 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -531,6 +531,9 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
   MCOptions.MCNoWarn = Opts.NoWarn;
   MCOptions.MCFatalWarnings = Opts.FatalWarnings;
   MCOptions.MCNoTypeCheck = Opts.NoTypeCheck;
+  MCOptions.ShowMCInst = Opts.ShowInst;
+  MCOptions.AsmVerbose = true;
+  MCOptions.MCUseDwarfDirectory = MCTargetOptions::EnableDwarfDirectory;
   MCOptions.ABIName = Opts.TargetABI;
 
   // FIXME: There is a bit of code duplication with addPassesToEmitFile.
diff --git a/llvm/lib/CodeGen/LLVMTargetMachine.cpp b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
index 562821df06651..819187c129c3a 100644
--- a/llvm/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
@@ -167,25 +167,12 @@ Expected<std::unique_ptr<MCStreamer>> LLVMTargetMachine::createMCStreamer(
     if (Options.MCOptions.ShowMCEncoding)
       MCE.reset(getTarget().createMCCodeEmitter(MII, Context));
 
-    bool UseDwarfDirectory = false;
-    switch (Options.MCOptions.MCUseDwarfDirectory) {
-    case MCTargetOptions::DisableDwarfDirectory:
-      UseDwarfDirectory = false;
-      break;
-    case MCTargetOptions::EnableDwarfDirectory:
-      UseDwarfDirectory = true;
-      break;
-    case MCTargetOptions::DefaultDwarfDirectory:
-      UseDwarfDirectory = MAI.enableDwarfFileDirectoryDefault();
-      break;
-    }
-
     std::unique_ptr<MCAsmBackend> MAB(
         getTarget().createMCAsmBackend(STI, MRI, Options.MCOptions));
     auto FOut = std::make_unique<formatted_raw_ostream>(Out);
     MCStreamer *S = getTarget().createAsmStreamer(
-        Context, std::move(FOut), Options.MCOptions.AsmVerbose,
-        UseDwarfDirectory, InstPrinter, std::move(MCE), std::move(MAB),
+        Context, std::move(FOut), Options.MCOptions.AsmVerbose, true,
+        InstPrinter, std::move(MCE), std::move(MAB),
         Options.MCOptions.ShowMCInst);
     AsmStreamer.reset(S);
     break;
diff --git a/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp b/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
index 55a0ee81ad4bc..45a62daffb033 100644
--- a/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
+++ b/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
@@ -62,6 +62,8 @@ Error DwarfStreamer::init(Triple TheTriple,
                              TripleName.c_str());
 
   MCTargetOptions MCOptions = mc::InitMCTargetOptionsFromFlags();
+  MCOptions.AsmVerbose = true;
+  MCOptions.MCUseDwarfDirectory = MCTargetOptions::EnableDwarfDirectory;
   MAI.reset(TheTarget->createMCAsmInfo(*MRI, TripleName, MCOptions));
   if (!MAI)
     return createStringError(std::errc::invalid_argument,
diff --git a/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp b/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp
index dd379de68e4c4..f790766348b53 100644
--- a/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp
+++ b/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp
@@ -41,6 +41,8 @@ Error DwarfEmitterImpl::init(Triple TheTriple,
                              TripleName.c_str());
 
   MCTargetOptions MCOptions = mc::InitMCTargetOptionsFromFlags();
+  MCOptions.AsmVerbose = true;
+  MCOptions.MCUseDwarfDirectory = MCTargetOptions::EnableDwarfDirectory;
   MAI.reset(TheTarget->createMCAsmInfo(*MRI, TripleName, MCOptions));
   if (!MAI)
     return createStringError(std::errc::invalid_argument,
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 24209e456b5e2..b4122881bd45a 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -55,9 +55,9 @@ class MCAsmStreamer final : public MCStreamer {
   raw_svector_ostream CommentStream;
   raw_null_ostream NullStream;
 
-  unsigned IsVerboseAsm : 1;
-  unsigned ShowInst : 1;
-  unsigned UseDwarfDirectory : 1;
+  bool IsVerboseAsm = false;
+  bool ShowInst = false;
+  bool UseDwarfDirectory = false;
 
   void EmitRegisterName(int64_t Register);
   void PrintQuotedString(StringRef Data, raw_ostream &OS) const;
@@ -72,24 +72,41 @@ class MCAsmStreamer final : public MCStreamer {
 
 public:
   MCAsmStreamer(MCContext &Context, std::unique_ptr<formatted_raw_ostream> os,
-                bool isVerboseAsm, bool useDwarfDirectory,
-                MCInstPrinter *printer, std::unique_ptr<MCCodeEmitter> emitter,
-                std::unique_ptr<MCAsmBackend> asmbackend, bool showInst)
+                bool, bool, MCInstPrinter *printer,
+                std::unique_ptr<MCCodeEmitter> emitter,
+                std::unique_ptr<MCAsmBackend> asmbackend, bool)
       : MCStreamer(Context), OSOwner(std::move(os)), OS(*OSOwner),
         MAI(Context.getAsmInfo()), InstPrinter(printer),
         Assembler(std::make_unique<MCAssembler>(
             Context, std::move(asmbackend), std::move(emitter),
             (asmbackend) ? asmbackend->createObjectWriter(NullStream)
                          : nullptr)),
-        CommentStream(CommentToEmit), IsVerboseAsm(isVerboseAsm),
-        ShowInst(showInst), UseDwarfDirectory(useDwarfDirectory) {
+        CommentStream(CommentToEmit) {
     assert(InstPrinter);
-    if (IsVerboseAsm)
-        InstPrinter->setCommentStream(CommentStream);
     if (Assembler->getBackendPtr())
       setAllowAutoPadding(Assembler->getBackend().allowAutoPadding());
 
     Context.setUseNamesOnTempLabels(true);
+
+    auto *TO = Context.getTargetOptions();
+    if (!TO)
+      return;
+    IsVerboseAsm = TO->AsmVerbose;
+    if (IsVerboseAsm)
+      InstPrinter->setCommentStream(CommentStream);
+    ShowInst = TO->ShowMCInst;
+    switch (TO->MCUseDwarfDirectory) {
+    case MCTargetOptions::DisableDwarfDirectory:
+      UseDwarfDirectory = false;
+      break;
+    case MCTargetOptions::EnableDwarfDirectory:
+      UseDwarfDirectory = true;
+      break;
+    case MCTargetOptions::DefaultDwarfDirectory:
+      UseDwarfDirectory =
+          Context.getAsmInfo()->enableDwarfFileDirectoryDefault();
+      break;
+    }
   }
 
   MCAssembler &getAssembler() { return *Assembler; }
diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp
index c46e4f087d42b..0f1e330541441 100644
--- a/llvm/tools/llvm-mc/llvm-mc.cpp
+++ b/llvm/tools/llvm-mc/llvm-mc.cpp
@@ -356,6 +356,9 @@ int main(int argc, char **argv) {
   cl::ParseCommandLineOptions(argc, argv, "llvm machine code playground\n");
   MCTargetOptions MCOptions = mc::InitMCTargetOptionsFromFlags();
   MCOptions.CompressDebugSections = CompressDebugSections.getValue();
+  MCOptions.ShowMCInst = ShowInst;
+  MCOptions.AsmVerbose = true;
+  MCOptions.MCUseDwarfDirectory = MCTargetOptions::EnableDwarfDirectory;
 
   setDwarfDebugFlags(argc, argv);
   setDwarfDebugProducer();
diff --git a/llvm/tools/llvm-ml/llvm-ml.cpp b/llvm/tools/llvm-ml/llvm-ml.cpp
index 829347dafee98..bcfec97019c1c 100644
--- a/llvm/tools/llvm-ml/llvm-ml.cpp
+++ b/llvm/tools/llvm-ml/llvm-ml.cpp
@@ -264,6 +264,7 @@ int llvm_ml_main(int Argc, char **Argv, const llvm::ToolContext &) {
   MCOptions.AssemblyLanguage = "masm";
   MCOptions.MCFatalWarnings = InputArgs.hasArg(OPT_fatal_warnings);
   MCOptions.MCSaveTempLabels = InputArgs.hasArg(OPT_save_temp_labels);
+  MCOptions.AsmVerbose = true;
 
   Triple TheTriple = GetTriple(ProgName, InputArgs);
   std::string Error;

From 9e69e6b8c32a3a06ba9e8eafcba3045974ef34be Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 20 Jul 2024 23:28:31 -0700
Subject: [PATCH 328/486] [MC] createAsmStreamer: add overload without unused
 bool parameters

The bool parameters have been made ineffective in favor of
MCTargetOptions options to resolve inconsistency issues. New clients
should not pass the unused bool arguments. The existing overload will be
removed.
---
 llvm/include/llvm/MC/TargetRegistry.h | 13 ++++++++-----
 llvm/lib/MC/MCAsmStreamer.cpp         | 10 ++++------
 llvm/lib/MC/TargetRegistry.cpp        | 25 ++++++++++++++++++-------
 3 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index 69658141355f5..14ceb76d312c5 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -541,11 +541,14 @@ class Target {
 
   MCStreamer *createAsmStreamer(MCContext &Ctx,
                                 std::unique_ptr<formatted_raw_ostream> OS,
-                                bool IsVerboseAsm, bool UseDwarfDirectory,
-                                MCInstPrinter *InstPrint,
-                                std::unique_ptr<MCCodeEmitter> &&CE,
-                                std::unique_ptr<MCAsmBackend> &&TAB,
-                                bool ShowInst) const;
+                                MCInstPrinter *IP,
+                                std::unique_ptr<MCCodeEmitter> CE,
+                                std::unique_ptr<MCAsmBackend> TAB) const;
+  MCStreamer *
+  createAsmStreamer(MCContext &Ctx, std::unique_ptr<formatted_raw_ostream> OS,
+                    bool IsVerboseAsm, bool UseDwarfDirectory,
+                    MCInstPrinter *IP, std::unique_ptr<MCCodeEmitter> &&CE,
+                    std::unique_ptr<MCAsmBackend> &&TAB, bool ShowInst) const;
 
   MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
                                             formatted_raw_ostream &OS,
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index b4122881bd45a..db93a33bbe3f7 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -72,9 +72,8 @@ class MCAsmStreamer final : public MCStreamer {
 
 public:
   MCAsmStreamer(MCContext &Context, std::unique_ptr<formatted_raw_ostream> os,
-                bool, bool, MCInstPrinter *printer,
-                std::unique_ptr<MCCodeEmitter> emitter,
-                std::unique_ptr<MCAsmBackend> asmbackend, bool)
+                MCInstPrinter *printer, std::unique_ptr<MCCodeEmitter> emitter,
+                std::unique_ptr<MCAsmBackend> asmbackend)
       : MCStreamer(Context), OSOwner(std::move(os)), OS(*OSOwner),
         MAI(Context.getAsmInfo()), InstPrinter(printer),
         Assembler(std::make_unique<MCAssembler>(
@@ -2663,7 +2662,6 @@ MCStreamer *llvm::createAsmStreamer(MCContext &Context,
                                     std::unique_ptr<MCCodeEmitter> &&CE,
                                     std::unique_ptr<MCAsmBackend> &&MAB,
                                     bool ShowInst) {
-  return new MCAsmStreamer(Context, std::move(OS), isVerboseAsm,
-                           useDwarfDirectory, IP, std::move(CE), std::move(MAB),
-                           ShowInst);
+  return new MCAsmStreamer(Context, std::move(OS), IP, std::move(CE),
+                           std::move(MAB));
 }
diff --git a/llvm/lib/MC/TargetRegistry.cpp b/llvm/lib/MC/TargetRegistry.cpp
index 75272b445db65..4190117c1e264 100644
--- a/llvm/lib/MC/TargetRegistry.cpp
+++ b/llvm/lib/MC/TargetRegistry.cpp
@@ -11,8 +11,10 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <vector>
@@ -86,19 +88,28 @@ MCStreamer *Target::createMCObjectStreamer(
                                 std::move(Emitter), STI);
 }
 
+MCStreamer *Target::createAsmStreamer(MCContext &Ctx,
+                                      std::unique_ptr<formatted_raw_ostream> OS,
+                                      MCInstPrinter *IP,
+                                      std::unique_ptr<MCCodeEmitter> CE,
+                                      std::unique_ptr<MCAsmBackend> TAB) const {
+  formatted_raw_ostream &OSRef = *OS;
+  MCStreamer *S = llvm::createAsmStreamer(Ctx, std::move(OS), false, false, IP,
+                                          std::move(CE), std::move(TAB), false);
+  auto *TO = Ctx.getTargetOptions();
+  createAsmTargetStreamer(*S, OSRef, IP, TO && TO->AsmVerbose);
+  return S;
+}
+
 MCStreamer *Target::createAsmStreamer(MCContext &Ctx,
                                       std::unique_ptr<formatted_raw_ostream> OS,
                                       bool IsVerboseAsm, bool UseDwarfDirectory,
-                                      MCInstPrinter *InstPrint,
+                                      MCInstPrinter *IP,
                                       std::unique_ptr<MCCodeEmitter> &&CE,
                                       std::unique_ptr<MCAsmBackend> &&TAB,
                                       bool ShowInst) const {
-  formatted_raw_ostream &OSRef = *OS;
-  MCStreamer *S = llvm::createAsmStreamer(
-      Ctx, std::move(OS), IsVerboseAsm, UseDwarfDirectory, InstPrint,
-      std::move(CE), std::move(TAB), ShowInst);
-  createAsmTargetStreamer(*S, OSRef, InstPrint, IsVerboseAsm);
-  return S;
+  return createAsmStreamer(Ctx, std::move(OS), IP, std::move(CE),
+                           std::move(TAB));
 }
 
 iterator_range<TargetRegistry::iterator> TargetRegistry::targets() {

From 0caf0c93e759816663af52e8632d1c3953dbc715 Mon Sep 17 00:00:00 2001
From: Tim Creech <timothy.m.creech@intel.com>
Date: Sun, 21 Jul 2024 03:04:27 -0400
Subject: [PATCH 329/486] [llvm-profgen] Support creating profiles of arbitrary
 events (#99026)

This change introduces two options which may be used to create profiles
of arbitrary PMU events.

1. `--leading-ip-only` provides a simple sample-IP-based profile mode.
This is not useful for building a profile of execution frequency, but it
is useful for building new types of profiles.

   For example, to build a profile of unpredictable branches:

perf record -b -e branch-misses:upp -o perf.data ... llvm-profgen
--perfdata perf.data --leading-ip-only ...

2. `--perf-event=event` enables the creation of a profile concerned with
a specific event or set of events. The names given should match the
"event" field as emitted by perf-script(1).

This option has two spellings: `--perf-event` and `--perf-events`. The
plural spelling accepts a comma-separated list. The singular spelling
appends a single event name to the set of events which will be used.
This is meant to accommodate event names containing commas.

Combined, these options allow generating multiple kinds of profiles from
a single `perf record` collection. For example, to generate both
execution frequency and branch mispredict profiles:

perf record -c 1000003 -b -e
br_inst_retired.near_taken:upp,br_misp_retired.all_branches:upp ...
llvm-profgen --output execution.prof
--perf-event=br_inst_retired.near_taken:upp ...
llvm-profgen --leading-ip-only --output unpredictable.prof
--perf-event=br_misp_retired.all_branches:upp ...

These additions are in support of more general HWPGO[^1], allowing
feedback from a wider range of hardware events.

[^1]:
https://llvm.org/devmtg/2024-04/slides/TechnicalTalks/Xiao-EnablingHW-BasedPGO.pdf

---------

Co-authored-by: Tim Creech <tcreech@tcreech.com>
---
 .../tools/llvm-profgen/Inputs/cmov_3.perfbin  | Bin 0 -> 27192 bytes
 .../llvm-profgen/Inputs/cmov_3.perfscript     |  39 ++++++
 .../Inputs/ip-duplication.perfscript          |   2 +
 .../Inputs/noprobe-skid.perfscript            |   5 +
 .../tools/llvm-profgen/event-filtering.test   |  78 ++++++++++++
 .../llvm-profgen/iponly-nodupfactor.test      |  22 ++++
 llvm/test/tools/llvm-profgen/iponly.test      |  58 +++++++++
 llvm/tools/llvm-profgen/PerfReader.cpp        | 116 ++++++++++++++++--
 llvm/tools/llvm-profgen/ProfileGenerator.cpp  |  31 +++--
 9 files changed, 331 insertions(+), 20 deletions(-)
 create mode 100755 llvm/test/tools/llvm-profgen/Inputs/cmov_3.perfbin
 create mode 100644 llvm/test/tools/llvm-profgen/Inputs/cmov_3.perfscript
 create mode 100644 llvm/test/tools/llvm-profgen/Inputs/ip-duplication.perfscript
 create mode 100644 llvm/test/tools/llvm-profgen/Inputs/noprobe-skid.perfscript
 create mode 100644 llvm/test/tools/llvm-profgen/event-filtering.test
 create mode 100644 llvm/test/tools/llvm-profgen/iponly-nodupfactor.test
 create mode 100644 llvm/test/tools/llvm-profgen/iponly.test

diff --git a/llvm/test/tools/llvm-profgen/Inputs/cmov_3.perfbin b/llvm/test/tools/llvm-profgen/Inputs/cmov_3.perfbin
new file mode 100755
index 0000000000000000000000000000000000000000..7a1543041f8055c4dbc60c23645f91f680ba621b
GIT binary patch
literal 27192
zcmeHQeQX@Zb)Wm#6D9GRjzxXNtAw&CDYrb5lFC|Nd;HSTrX)t9WhrsII^HeGtB!Z<
z-JU|FK_i)t6Gn!jq)wx>MqmR)8>2~(KLW=HoS24^NNVdu4$#183r7j!Le7W7B#NuL
zuCMRS%scKb_Z*={3kcW&Zr{A$dv8A8%+BuY42K^Z7#<7;0*peC?PkawYt%U;u8cjC
z<gCPcSRK>Zd)NjR0VE7RolB647}1x4LO`@e>V<%!T`NX7)g5#l=NKbOB}9sLi$RxB
zL3<Rz5@n5oM>{T(Jd^b!`lKueQM(<_PZjAfRTmLScglhhwcDX@m?qTOZe2tq)umr1
zB_%s#(XLV2HA;P=S7m-g$)8w8{PoCs^n`TakQY=%27=13jJfTONV_9apD5`QCEp)<
z#Y+Eg@*9zMVVPi9`jO~PSw5na-}}Li`%}vQ5TRy1DD$h<4vhRSX+S32k<E;4-`bH)
zN3)q+X(l?;vpu?ft5(cwU0egpg?V6~+PnWg7Cgn~5tDA@ZGm!359Jt8M%f`wKa|~U
z*aC{p9?H9_;41-FJ@nhE=<lwAGw{b0R)0G)jESjy&L~>Rf@K&eND0AIGMmk(K#q`<
zPbD)sRxBiQX|{KGsJGAP(z>*5m3a47$S=q}6XMqZaBBdg_A9N)EJx4vN$l&N`Ag^q
zL3RLm3=4a7PO)#GKPlX>;nXH5Yt?nGvJ9@&76_;|ocbK%?KWHvEeQL;_kiyK-vhn}
zd=K~@@IByr!1ur{>4D!hy!(yB>@RB)XOvev7)yM7-U^nlBxb);bAi{VyzQp|mpA?l
zt_^qS5JP?vWv^Z>L)iE)$m6T`t5-PxL*((*`qg>hG5yY84b8pyU}Em4iP@hmjEoL+
z&Uao;T)e2ZKyvRo4(iv(wT3$$#xxn~Usz<%J)^@5cw!e`t(Ip{1B<x)IRwgA$Io48
zxTBx3MT2j1{nM{;GPjU;{<S@c=Wm1(f$t?=e9LO8q_9k;;I==tYZDZRl~&!Cn0w{g
zpF<74r>p`b@FAh;*8j!X#k|o!@%!V4<D-X0o;%eOVAo!Lt6WaZDL;nEyZ~Mhz>g}x
zvjTXb0{p!IzJtJKD9N=?A#i5>Ujs0FX4~UH;?LEBVED|&&jT~gL&bmB{tVU6Co-45
za^_zr6LbHNc>d?$f&YvKUd&up_cONXQtw><j_Z9F`*&Z5R2W2dUdQIU7QZRW8)$ih
zEN^hjWjAH{CR)BpmTz*)7ykPXmo~hk?_wC|rBK!||IRWrd$Dooboo(mal^+~KpSCy
z{b2lo_=9odg3}+J^G^Qs&PTfMVr*#c%11*(bC(9rgirO)g+Gxf&nI@FdrB|!1S<Z#
zPc|srBaIDTeW0!O^l)3PehO}eua36WUaJ$0b9<CG&VRfQ$*vnfzSRi1KoebuIFz80
zUSjUY!*j0>&%H4`_mgV|6utaH=@*^zP=Q8Sfe&4%+>@2N5G(Ho-vhn}d=K~@@IByr
z!1sXf0pA0@2Ye5_GY{Z;TVQQyM-L=}YiR2h%H@v&eE{ev&_{vdY=8JS<?`PE{kPwi
z%U6M}`(3$=^T*rZhavEhgDf!97+71otmYg{?Qs2W9SXqyPNe<I8WD@Z5A&e4v^EsM
z@BFXJWgRH1Zyc;|y{BR2higu;J!^Js?rwVz!dSj6{GI^Yhj8lHTi@7Q-`ZQR#_QYT
z^|4vy(eT;Or-PwF0C|<mKm*sgG2bu2?>f|>8z=@`Uw#&TD%8h9AWrG^^7Q)a_1Vay
z%GvOzLyrW5Mtuyjfy_j*ah@z%`N8*q?*ZQfz6X2{_#W^*;CsOLfbRj{1GkI^^d_B`
zwNX->T48Wg+>?o+RuGs}={!~^`Nv=}7sG0Ph0AW8$JR<7=V2Hqof;`$C*aC{<lnzl
z&LcDv(z%4@iS1#*;~W*kc`08ZSoQ%)alb2uRzcXiBqdVW3kvJU7=ru?tICqnd=~eI
zVpt^zTaft^J}-Hy7tU)jz-Cn*D1TTV<n=ix^<lw}^W<Nc&T{@CDVOH|Z-n-hyEEJ_
z+jmgX!;-r7_V)GNt+vAs=S^y7Cv5HQRJ&qbU9rxtKT_KdnrStWw4@;V$D5d4JEnDN
zF*cO5%xwF?O=>=8#z%(K{*k^dTRPyU_T{IhGg-5M33fxuF}0n?X0(`g6vDoJ{r#I5
zwl~i2vBU0-w*|VVJ7e3nGWLipFHseBFbF-RP|D>pxe0Y5m2wxHsa$bUH<`=j$1*uj
z-AaF#=}u=$6QG()PMLSB(h!rQekt<by!XKU>`w-u^F<taZ4P4pY;a&(V54>FxRJ^x
zi<6ACIAohs<HmF$Z<#48lh5HzME9f2XYjzm(V@Yktc+&2J8ZY`{n$s281X}g20-h;
z{zD*lm1h~2hwed<<_6y7U1{Z~GL|u3fLa^V`3$sa0qp2rkY$N9%Rf74a+G6(s9(WM
zW-}8x6jf`}W3o+&s*ZHp{5FmIZR+2ra8vNH$iGi<(8eLcAA2|ZV=s0DeqZ>{Q~w<a
zI|BsXA2$2%NaBT^soW{#mo~+J@0AYmpQrv@#&K`>&r|LW|9R^8=0NjOqN=0M`18}f
zx8VHroc|pO`uMH+j)dRB@Y^6BZ{n^4wd?0(`{MVkMP2XjC~h17JC*<KQOqIyvHeyY
z+vzz?RE05s2R>f-ZR)qFzwWbmKm6Z+PsW8M$68*pvvLj@t>by=&@-H<mky)nyrf<_
zw1)k-T+4nt9n$sExn1V5_HFW_Jf4(!)xnq(=FYBx80@!(-+H#d5yoBzIwDu-`(f2E
zNDOX;z7u|dn$R~uXH4pFrf6p#wlkk2<~gwWTVgH(^Urpsp;-qAPG3UFke!*83=6M@
z%!6x#VHN4cKzIYv?*^2sQllG8w_pKp58~f@W5Dg!-3Z+g(jeL=;m{9&IV~BGHU&1;
zpFUCFqJ-)jSFNpY3N*voWKaXVIr915)uH;kSMGo}u+SX%ptGe35)5mMqsm>Ycdp(8
zE3=BmT-xltsK>TLR)Ij^zv}Cd0ECb6Ku4)q=*Z+!*;3ks-R2ofOM!$Htu!2HrA?v^
zENWJKzgnqXF{`YId{L>5lpM4U_?3VwK$kyRx~#O+99Gsujwnqvmm)_$qqLm8Dem4`
zWfcpCgCA2AB#|#Gfym0^3ak07vU$bpN^|6kp>Zt|TDwzejNGHF2c2*vAxt7Wh0Qpa
zbSdm5CA6aEvQqZ}<+g~Wtg#gJn~GASDNQ`xvJ&D{i^L);wIKQgR-0jQd@NanCH!}~
zM6X7pxFVcYqvNGq=1)r|XA8Mx7SC2Yo}4hFxLjTYdD<K+O@MejA05vZqVRe-o&lW+
z5Q(@MEheX?vu1R{%$bFxWvaZH)My?fYV<%CsGcxW&|td`9o{z#+@zVw<x9orM6MJy
zXDl;Ugr#!GEM1zKPG<^e7nP0O0mqjmv*}Fncvputl`5EKYO-V6ER5f47IJ2`Z+e=^
zGtR_`W|>^ZVns7+Bnt(Qh}9<SB;FweJ*YLLFA;+GE9dmtg?J*`5l{VaWq*uQU#aYi
zapDn1`)8c^GUm1KBm^4?Xupb6e>og-r`sVO53v=M{V{f72v^?Z80f@nnd*txRmOX#
zem$e{)``PT301d4JRX9l2%RTr7lv39(>?W@EBlq4`Ynw1DLL`mnAg6U5bQRh^XQ!V
zFuT-sJH+E5*j*I!#9Kk#OTV()#F17Ac2!Lt^U|l?R}OXPRkZ8MjjwkCk_)kSFTrsv
z471<xeI<4w&Z8L&A@sZ^86IWfh4)IF-oZ(q@8^Pa-1vzq`hQUc|9TbtN)>#e3cegG
z>`@;6@U`8iD%wMcNqx5;^eXshmGt?^DjxZss-pjeD)`e?@b6c_e_jPISHW9@P-!S9
zeIwycKtD*_?T2BByW5wa9OF^nPgc=?stW%7D)?(v@MhfT<dJVj6?{bE?)rXM;`A;`
zes~;kulwci01v{httVXL;ZqX7;=-SmIPKS@^nVCAJY6c|1~wChmpQ(498=(B6Z570
zpF*3lCcp!1>3A3e+^hWCxqd54=sHgV--j{mkobrT*Cl?$g%3#lm<zAww|lvM6YJTa
z^Ze`=uW}yc`b)>{F~D)w;eK9}xc<^{`wZaVoh9KP0arc7^QQoB1l&EIe+|<KE`6>&
zT_yd?G9CBx;zxk1OG?AoD}XlzSFs-HPg40U;<&?5)j0$#4~JlrUo-Rv`@{kT0g2n+
z=Mi7daf~^`-XPYul})<xU61;<@c~#KegFl)e2>X~ey7B@fdZ!E_FJ6iTgU8^2{ai5
z9P8`WKZyFlrSBo%;Q7kT>;w<~?a(xN{%@*ab!vI0$)#{3IhHYUu-_791|FDU!ZEiV
zxHg%$yl~X7Ceuo}Z05L`Jz3T8?G!YAI&W#Zs`1;%HFE}DYdxyLi_3)ucZ8LkSX9Gz
zVY^oZ@AQbkoHbt5^TIv6U^7_i0hz;++S!b%$XvCy893bm=2j+U6ik>a*z4=Wo%MFg
zob`0dmi7pz+`}JE1)jiCzc`ha_710<CjwBxviBgTj>}(8Ay2^RHzS=DPJ>Aso(+vo
zoX(h{D2v^wni}3&a5EOI5*((LVrjErPGpL(&)Bf0U~e?+9xuYd)9Ji1k<E`Kvqsv=
z7m7x*G{aJ`k34HyW*T;zdukZt(AYSIOP+*sTZNNMY-ENB;?zknv2g}=3x}%sN-$*<
zI~iQM@X`&JbkPFK!GrOA1IEDqemExE=s&tYzHg`xVAw5h3?!s9(SMK`dxsD7#)pjq
zgM)_#Mvc*U@9+RvxX;nnbvi;Dj?FgAbka&PUU+f9_M#Y1*oLZtT%5uUjN`~|kQMXB
zBpk|}HJLGV0JPGXoKb?=zHK^-hk3iw^ln2)eymuOS@CJGVT|tUql%-?;2iwo$tf#2
z2GlADI!Q6UUmS)iKCr?xU4ywk%(^WY`ZyHt9g6a?24$1D4PHy1%t3mBS_P(|<b+wk
zsl9_SKnAyJ5)EW*ItzmvFOz1$_W-70^MfqU{i&IgvcV_QU`w2^5G^k(C=LlHr!w$3
zn7}a}E*j53BnEC_8g#%Z=vSg#W2H<sttBn1kQpmk&}lXP%>z2tI6P)VJUKR2Fi#K%
z-#<)>gU9U8s&^2_nIV6bmub*<3HpBw?inJzPg0ieg$VvF3_R|6=n3hP3SJeYa;5h|
z0=7fMZcpDi=>HI?q$nx9rx4r^k=NmGXDI&}ng0yD?_!|x!{eU^cm8nHAKw>D_UEPj
zd1-%6+Tq_rVBN`{{;uaRV5my=^nImW7L@)T2~>E%H0<F)VDN8&$ezB#JS6Su@2sf&
zq(}5P*yG<85l`Q1&Pn^2%*<VWD#J%0f~sUs-*x6)_LRT7{trw0UYQ{Mebj=qPe>f?
z;VpWJ|IPx2KEd+wAm9JsrH=>qmkXJ!5W;m1es+6R+N;uDuV{*hgeiIUn9H8tKaWYp
zt+F3bGGtHuU%Bk*`_yACLZk9`IMTpf{wJh;vy{{OI8z1S)+e6mm!v(lAI)FF(tf8%
zrg9}a0-lBl`j71CyO^#DjW-3UT&evDn1@I`{3>qRtpf)EKj}ko>l07(pFjuy&WY^l
zJ6p|7+P?(0*muaDzQ@u3O}UDdgFyX@@}+kC3CK`|?CHDRLI+ym+`}2jp6I`W*lrIK
z4`C2<5W&Ee?1=svFuOg?>tfQrTK<&a?`4`)ewz2_(!QM(C8c}`egh(yzf>ofk^QL(
zA|j}ZfoKRKIDS*0?~n5`!Amp%O9glT!*wG{i_$>)$$f&u^&ER}v20@%`!DPlR`=VD
zA?~*43s5c{WEZ6UT`mI`%l63p-TqrOA`<O!84zoybnn@VG4g)F&C5TYQ-K|YAGIGn
me^5+q57A0!{Gf;)p@CaUC|^=+f{2&><T0UfkITTtvi||LxjNba

literal 0
HcmV?d00001

diff --git a/llvm/test/tools/llvm-profgen/Inputs/cmov_3.perfscript b/llvm/test/tools/llvm-profgen/Inputs/cmov_3.perfscript
new file mode 100644
index 0000000000000..3d29d444d56bb
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/Inputs/cmov_3.perfscript
@@ -0,0 +1,39 @@
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/3//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/24//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/29//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//- 
+br_misp_retired.all_branches:upp:            4012fa 0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/24//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/4//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/3//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//- 
+br_misp_retired.all_branches:upp:            4012fa 0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/24//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/24//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/24//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/6//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/21//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/14//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/2//- 
+br_misp_retired.all_branches:upp:            4012fa 0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/26//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/5//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/24//-  0x401310/0x4012f0/P/-/-/2//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//- 
+br_misp_retired.all_branches:upp:            4012fa 0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/2//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/2//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/27//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/29//-  0x401310/0x4012f0/P/-/-/24//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/19//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/26//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/29//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/4//- 
+br_misp_retired.all_branches:upp:            4012fa 0x401310/0x4012f0/P/-/-/13//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/24//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/24//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/2//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//- 
+br_misp_retired.all_branches:upp:            4012fa 0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/20//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/34//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//- 
+  br_inst_retired.near_taken:upp:            4012fa 0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/24//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/17//-  0x401310/0x4012f0/P/-/-/9//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/24//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//- 
+br_misp_retired.all_branches:upp:            4012fa 0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/29//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/6//-  0x401310/0x4012f0/P/-/-/24//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/28//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/15//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/9//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/17//-  0x4012fa/0x4012ff/M/-/-/1//- 
+br_misp_retired.all_branches:upp:            4012fa 0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/24//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/4//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/19//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/10//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/25//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/7//-  0x4012fa/0x4012ff/M/-/-/1//- 
+br_misp_retired.all_branches:upp:            4012fa 0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/21//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/24//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/23//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/2//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/5//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/1//-  0x401310/0x4012f0/P/-/-/1//- 
+  br_inst_retired.near_taken:upp:            401310 0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/3//-  0x401310/0x4012f0/P/-/-/23//-  0x4012fa/0x4012ff/M/-/-/4//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/2//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/27//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/25//-  0x401310/0x4012f0/P/-/-/4//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/3//-  0x401310/0x4012f0/P/-/-/1//-  0x4012fa/0x4012ff/P/-/-/1//-  0x401310/0x4012f0/P/-/-/28//-  0x401310/0x4012f0/P/-/-/22//-  0x4012fa/0x4012ff/M/-/-/2//-  0x401310/0x4012f0/P/-/-/26//-  0x401310/0x4012f0/P/-/-/28//- 
diff --git a/llvm/test/tools/llvm-profgen/Inputs/ip-duplication.perfscript b/llvm/test/tools/llvm-profgen/Inputs/ip-duplication.perfscript
new file mode 100644
index 0000000000000..f0d4efcbe668e
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/Inputs/ip-duplication.perfscript
@@ -0,0 +1,2 @@
+           4006b7 0x4006b7/0x40068b/P/-/-/1  0x4006c8/0x4006b0/P/-/-/1  0x400689/0x4006b9/P/-/-/1  0x40066d/0x400686/P/-/-/2  0x4007a6/0x400650/P/-/-/9  0x4007ca/0x400790/P/-/-/8  0x4007d7/0x4007bd/P/-/-/1  0x400792/0x4007d7/P/-/-/1  0x4007b8/0x400790/P/-/-/2  0x4006a2/0x4007a8/P/-/-/3
+           40065d 40065d/0x40068f/M/-/-/1
diff --git a/llvm/test/tools/llvm-profgen/Inputs/noprobe-skid.perfscript b/llvm/test/tools/llvm-profgen/Inputs/noprobe-skid.perfscript
new file mode 100644
index 0000000000000..7c70a18452e57
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/Inputs/noprobe-skid.perfscript
@@ -0,0 +1,5 @@
+// Invalid perf line
+           40062f 0x40062f/0x4005b0/P/-/-/9  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/6  0x40062f/0x4005b0/P/-/-/16  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/6  0x40062f/0x4005b0/P/-/-/6  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005c8/0x4005dc/P/-/-/8  0x40062f/0x4005b0/P/-/-/9  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/10  0x40062f/0x4005b0/P/-/-/14  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/7  0x40062f/0x4005b0/P/-/-/8  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005c8/0x4005dc/P/-/-/7  0x40062f/0x4005b0/P/-/-/15  0x400645/0x4005ff/P/-/-/1
+           4005d7 0x4005d7/0x4005e5/P/-/-/8  0x40062f/0x4005b0/P/-/-/6  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/2  0x4005c8/0x4005dc/P/-/-/7  0x40062f/0x4005b0/P/-/-/11  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/8  0x40062f/0x4005b0/P/-/-/9  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/5  0x40062f/0x4005b0/P/-/-/11  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/2  0x4005c8/0x4005dc/P/-/-/7  0x40062f/0x4005b0/P/-/-/10  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/8  0x40062f/0x4005b0/P/-/-/9  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/13  0x40062f/0x4005b0/P/-/-/9
+           4005c8 0x4005c8/0x4005dc/P/-/-/11  0x40062f/0x4005b0/P/-/-/8  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/5  0x40062f/0x4005b0/P/-/-/6  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/12  0x40062f/0x4005b0/P/-/-/6  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/2  0x4005c8/0x4005dc/P/-/-/7  0x40062f/0x4005b0/P/-/-/10  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/8  0x40062f/0x4005b0/P/-/-/9  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/12  0x40062f/0x4005b0/P/-/-/6  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/2  0x4005c8/0x4005dc/P/-/-/8  0x40062f/0x4005b0/P/-/-/8
+           4005c5 0x4005c8/0x4005dc/P/-/-/11  0x40062f/0x4005b0/P/-/-/8  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/5  0x40062f/0x4005b0/P/-/-/6  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/12  0x40062f/0x4005b0/P/-/-/6  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/2  0x4005c8/0x4005dc/P/-/-/7  0x40062f/0x4005b0/P/-/-/10  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/8  0x40062f/0x4005b0/P/-/-/9  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/1  0x4005d7/0x4005e5/P/-/-/12  0x40062f/0x4005b0/P/-/-/6  0x400645/0x4005ff/P/-/-/1  0x400637/0x400645/P/-/-/1  0x4005e9/0x400634/P/-/-/2  0x4005c8/0x4005dc/P/-/-/8  0x40062f/0x4005b0/P/-/-/8
diff --git a/llvm/test/tools/llvm-profgen/event-filtering.test b/llvm/test/tools/llvm-profgen/event-filtering.test
new file mode 100644
index 0000000000000..ea486a8fa2f7b
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/event-filtering.test
@@ -0,0 +1,78 @@
+// RUN: llvm-profgen --format=text --perfscript=%S/Inputs/cmov_3.perfscript --binary=%S/Inputs/cmov_3.perfbin --output=%t --skip-symbolization --perf-event=br_inst_retired.near_taken:upp
+// RUN: FileCheck %s --input-file %t --check-prefix=CHECK-RAW-PROFILE
+// RUN: llvm-profgen --format=text --perfscript=%S/Inputs/cmov_3.perfscript --binary=%S/Inputs/cmov_3.perfbin --output=%t --perf-event=br_inst_retired.near_taken:upp
+// RUN: FileCheck %s --input-file %t --check-prefix=CHECK
+
+// RUN: llvm-profgen --format=text --perfscript=%S/Inputs/cmov_3.perfscript --binary=%S/Inputs/cmov_3.perfbin --output=%t --skip-symbolization --perf-event=br_misp_retired.all_branches:upp --leading-ip-only
+// RUN: FileCheck %s --input-file %t --check-prefix=UNPRED-RAW-PROFILE
+// RUN: llvm-profgen --format=text --perfscript=%S/Inputs/cmov_3.perfscript --binary=%S/Inputs/cmov_3.perfbin --output=%t --perf-event=br_misp_retired.all_branches:upp --leading-ip-only
+// RUN: FileCheck %s --input-file %t --check-prefix=UNPRED
+
+// Check that we can use perf event filtering to generate multiple types of
+// source-level profiles from a single perf profile. In this case, we generate
+// a typical execution frequency profile using br_inst_retired.near_taken LBRs,
+// and a branch mispredict profile using br_misp_retired.all_branches sample
+// IPs.
+
+// The source example below is based on perfKernelCpp/cmov_3, except a
+// misleading builtin is used to persuade the compiler not to use cmov, which
+// induces branch mispredicts.
+
+// CHECK: sel_arr:20229:0
+// CHECK:  3.1: 627
+// CHECK:  3.2: 627
+// CHECK:  4: 615
+// CHECK:  5: 627
+
+// UNPRED: sel_arr:18:0
+// UNPRED:  3.1: 0
+// UNPRED:  3.2: 0
+// UNPRED:  4: 9
+// UNPRED:  5: 0
+
+// CHECK-RAW-PROFILE:      3
+// CHECK-RAW-PROFILE-NEXT: 2f0-2fa:303
+// CHECK-RAW-PROFILE-NEXT: 2f0-310:312
+// CHECK-RAW-PROFILE-NEXT: 2ff-310:315
+
+// UNPRED-RAW-PROFILE:      1
+// UNPRED-RAW-PROFILE-NEXT: 2fa-2fa:9
+
+// original code:
+// clang -O2 -gline-tables-only -fdebug-info-for-profiling lit.c
+#include <stdlib.h>
+
+#define N 20000
+#define ITERS 10000
+
+static int *m_s1, *m_s2, *m_s3, *m_dst;
+
+void init(void) {
+    m_s1 = malloc(sizeof(int)*N);
+    m_s2 = malloc(sizeof(int)*N);
+    m_s3 = malloc(sizeof(int)*N);
+    m_dst = malloc(sizeof(int)*N);
+    srand(42);
+
+    for (int i = 0; i < N; i++) {
+        m_s1[i] = rand() % N;
+        m_s2[i] = 0;
+        m_s3[i] = 1;
+    }
+}
+
+void __attribute__((noinline)) sel_arr(int *dst, int *s1, int *s2, int *s3) {
+#pragma nounroll
+#pragma clang loop vectorize(disable) interleave(disable)
+    for (int i = 0; i < N; i++) {
+        int *p = __builtin_expect((s1[i] < 10035), 0) ? &s2[i] : &s3[i];
+        dst[i] = *p;
+    }
+}
+
+int main(void) {
+  init();
+  for(int i=0; i<ITERS; ++i)
+    sel_arr(m_dst, m_s1, m_s2, m_s3);
+  return 0;
+}
diff --git a/llvm/test/tools/llvm-profgen/iponly-nodupfactor.test b/llvm/test/tools/llvm-profgen/iponly-nodupfactor.test
new file mode 100644
index 0000000000000..006b1c42f3234
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/iponly-nodupfactor.test
@@ -0,0 +1,22 @@
+; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/ip-duplication.perfscript --binary=%S/Inputs/inline-noprobe2.perfbin --output=%t --use-offset=0 --leading-ip-only
+; RUN: FileCheck %s --input-file %t --check-prefix=CHECK
+
+; Test that we don't over-count samples for duplicated source code when
+; building an IP-based profile.
+
+; The inline-noprobe2.perfbin binary is used for this test because one of the
+; partition_pivot_last+3.1 debug locations has a duplication factor of 2
+; encoded into its discriminator. In IP-sample mode, a hit in one instruction
+; in the duplicated code does not imply a hit to the other duplicates.
+
+; The perfscript input includes 1 sample at a location with duplication factor
+; of 2, and another sample at the same source location but with no duplication
+; factor. These should be summed without duplication factors. Ensure we record
+; a count of 1+1=2 (and not 2+1=3) for the 3.1 location.
+
+;CHECK-LABEL: partition_pivot_last
+;CHECK-NEXT:  1: 0
+;CHECK-NEXT:  2: 0
+;CHECK-NEXT:  3: 0
+;CHECK-NEXT:  3.1: 2
+
diff --git a/llvm/test/tools/llvm-profgen/iponly.test b/llvm/test/tools/llvm-profgen/iponly.test
new file mode 100644
index 0000000000000..2e81798d7e6fe
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/iponly.test
@@ -0,0 +1,58 @@
+; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/noprobe-skid.perfscript --binary=%S/Inputs/noprobe.perfbin --output=%t --skip-symbolization --leading-ip-only
+; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-RAW-PROFILE
+; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/noprobe-skid.perfscript --binary=%S/Inputs/noprobe.perfbin --output=%t --leading-ip-only
+; RUN: FileCheck %s --input-file %t --check-prefix=CHECK
+
+; Here we check the ability to ignore LBRs, which is useful for generating
+; profiles where only the precise PMU sample IP is of interest. In general the
+; IPs need not identify a branch. In this case there are exactly 4 samples, so
+; we see only these 4 locations as "hot" and none of the LBR history.
+; Compare with noinline-noprobe.test, which includes LBR history.
+
+; Note that there are two different IPs (5c5 and 5c8) contributing to line
+; offset 1 in bar. This tests that sample counts corresponding to the same
+; debug location are summed into that location in the profile rather than the
+; maximum being taken, as happens with basic block execution count profiles.
+
+;CHECK: bar:14:0
+;CHECK:  0: 0
+;CHECK:  1: 2
+;CHECK:  2: 1
+;CHECK:  4: 0
+;CHECK:  5: 0
+;CHECK: foo:5:0
+;CHECK:  0: 0
+;CHECK:  1: 0
+;CHECK:  2: 0
+;CHECK:  3: 1
+;CHECK:  4: 0
+;CHECK:  5: 0
+
+CHECK-RAW-PROFILE:      4
+CHECK-RAW-PROFILE-NEXT: 5c5-5c5:1
+CHECK-RAW-PROFILE-NEXT: 5c8-5c8:1
+CHECK-RAW-PROFILE-NEXT: 5d7-5d7:1
+CHECK-RAW-PROFILE-NEXT: 62f-62f:1
+
+; original code:
+; clang -O3 -g -fdebug-info-for-profiling test.c -fno-inline -o a.out
+#include <stdio.h>
+
+int bar(int x, int y) {
+  if (x % 3) {
+    return x - y;
+  }
+  return x + y;
+}
+
+void foo() {
+  int s, i = 0;
+  while (i++ < 4000 * 4000)
+    if (i % 91) s = bar(i, s); else s += 30;
+  printf("sum is %d\n", s);
+}
+
+int main() {
+  foo();
+  return 0;
+}
diff --git a/llvm/tools/llvm-profgen/PerfReader.cpp b/llvm/tools/llvm-profgen/PerfReader.cpp
index 111c546f5329f..b4e4911fb8912 100644
--- a/llvm/tools/llvm-profgen/PerfReader.cpp
+++ b/llvm/tools/llvm-profgen/PerfReader.cpp
@@ -41,6 +41,17 @@ static cl::opt<bool>
                                 "and produce context-insensitive profile."));
 cl::opt<bool> ShowDetailedWarning("show-detailed-warning",
                                   cl::desc("Show detailed warning message."));
+cl::opt<bool>
+    LeadingIPOnly("leading-ip-only",
+                  cl::desc("Form a profile based only on sample IPs"));
+
+static cl::list<std::string> PerfEventFilter(
+    "perf-event",
+    cl::desc("Ignore samples not matching the given event names"));
+static cl::alias
+    PerfEventFilterPlural("perf-events", cl::CommaSeparated,
+                          cl::desc("Comma-delimited version of -perf-event"),
+                          cl::aliasopt(PerfEventFilter));
 
 extern cl::opt<std::string> PerfTraceFilename;
 extern cl::opt<bool> ShowDisassemblyOnly;
@@ -404,13 +415,18 @@ PerfScriptReader::convertPerfDataToTrace(ProfiledBinary *Binary, bool SkipPID,
     }
   }
 
+  // If filtering by events was requested, additionally request the "event"
+  // field.
+  const std::string FieldList =
+      PerfEventFilter.empty() ? "ip,brstack" : "event,ip,brstack";
+
   // Run perf script again to retrieve events for PIDs collected above
   SmallVector<StringRef, 8> ScriptSampleArgs;
   ScriptSampleArgs.push_back(PerfPath);
   ScriptSampleArgs.push_back("script");
   ScriptSampleArgs.push_back("--show-mmap-events");
   ScriptSampleArgs.push_back("-F");
-  ScriptSampleArgs.push_back("ip,brstack");
+  ScriptSampleArgs.push_back(FieldList);
   ScriptSampleArgs.push_back("-i");
   ScriptSampleArgs.push_back(PerfData);
   if (!PIDs.empty()) {
@@ -575,14 +591,54 @@ bool PerfScriptReader::extractLBRStack(TraceStream &TraceIt,
 
   // Skip the leading instruction pointer.
   size_t Index = 0;
+
+  StringRef EventName;
+  // Skip a perf event name. This may or may not exist.
+  if (Records.size() > Index && Records[Index].ends_with(":")) {
+    EventName = Records[Index].ltrim().rtrim(':');
+    Index++;
+
+    if (PerfEventFilter.empty()) {
+      WithColor::warning() << "No --perf-event filter was specified, but an "
+                              "\"event\" field was found in line "
+                           << TraceIt.getLineNumber() << ": "
+                           << TraceIt.getCurrentLine() << "\n";
+    } else if (std::find(PerfEventFilter.begin(), PerfEventFilter.end(),
+                         EventName) == PerfEventFilter.end()) {
+      TraceIt.advance();
+      return false;
+    }
+
+  } else if (!PerfEventFilter.empty()) {
+    WithColor::warning() << "A --perf-event filter was specified, but no "
+                            "\"event\" field found in line "
+                         << TraceIt.getLineNumber() << ": "
+                         << TraceIt.getCurrentLine() << "\n";
+  }
+
   uint64_t LeadingAddr;
-  if (!Records.empty() && !Records[0].contains('/')) {
-    if (Records[0].getAsInteger(16, LeadingAddr)) {
+  if (Records.size() > Index && !Records[Index].contains('/')) {
+    if (Records[Index].getAsInteger(16, LeadingAddr)) {
       WarnInvalidLBR(TraceIt);
       TraceIt.advance();
       return false;
     }
-    Index = 1;
+    Index++;
+  }
+
+  // We assume that if we saw an event name we also saw a leading addr.
+  // In other words, LeadingAddr is set if Index is 1 or 2.
+  if (LeadingIPOnly && Index > 0) {
+    // Form a profile only from the sample IP. Do not assume an LBR stack
+    // follows, and ignore it if it does.
+    uint64_t SampleIP = Binary->canonicalizeVirtualAddress(LeadingAddr);
+    bool SampleIPIsInternal = Binary->addressIsCode(SampleIP);
+    if (SampleIPIsInternal) {
+      // Form a half LBR entry where the sample IP is the destination.
+      LBRStack.emplace_back(LBREntry(SampleIP, SampleIP));
+    }
+    TraceIt.advance();
+    return !LBRStack.empty();
   }
 
   // Now extract LBR samples - note that we do not reverse the
@@ -902,6 +958,20 @@ void PerfScriptReader::computeCounterFromLBR(const PerfSample *Sample,
                                              uint64_t Repeat) {
   SampleCounter &Counter = SampleCounters.begin()->second;
   uint64_t EndAddress = 0;
+
+  if (LeadingIPOnly) {
+    assert(Sample->LBRStack.size() == 1 &&
+           "Expected only half LBR entries for ip-only mode");
+    const LBREntry &LBR = *(Sample->LBRStack.begin());
+    uint64_t SourceAddress = LBR.Source;
+    uint64_t TargetAddress = LBR.Target;
+    if (SourceAddress == TargetAddress &&
+        Binary->addressIsCode(TargetAddress)) {
+      Counter.recordRangeCount(SourceAddress, TargetAddress, Repeat);
+    }
+    return;
+  }
+
   for (const LBREntry &LBR : Sample->LBRStack) {
     uint64_t SourceAddress = LBR.Source;
     uint64_t TargetAddress = LBR.Target;
@@ -1062,6 +1132,18 @@ bool PerfScriptReader::isLBRSample(StringRef Line) {
   Line.trim().split(Records, " ", 2, false);
   if (Records.size() < 2)
     return false;
+  // Check if there is an event name before the leading IP.
+  // If there is, it will be in Records[0]. To skip it, we'll re-split on
+  // Records[1], which should contain the rest of the line.
+  if (Records[0].contains(":")) {
+    // If so, consume the event name and continue processing the rest of the
+    // line.
+    StringRef IPAndLBR = Records[1].ltrim();
+    Records.clear();
+    IPAndLBR.split(Records, " ", 2, false);
+    if (Records.size() < 2)
+      return false;
+  }
   if (Records[1].starts_with("0x") && Records[1].contains('/'))
     return true;
   return false;
@@ -1152,6 +1234,18 @@ void PerfScriptReader::warnInvalidRange() {
     const PerfSample *Sample = Item.first.getPtr();
     uint64_t Count = Item.second;
     uint64_t EndAddress = 0;
+
+    if (LeadingIPOnly) {
+      assert(Sample->LBRStack.size() == 1 &&
+             "Expected only half LBR entries for ip-only mode");
+      const LBREntry &LBR = *(Sample->LBRStack.begin());
+      if (LBR.Source == LBR.Target && LBR.Source != ExternalAddr) {
+        // This is an leading-addr-only profile.
+        Ranges[{LBR.Source, LBR.Source}] += Count;
+      }
+      continue;
+    }
+
     for (const LBREntry &LBR : Sample->LBRStack) {
       uint64_t SourceAddress = LBR.Source;
       uint64_t StartAddress = LBR.Target;
@@ -1199,11 +1293,15 @@ void PerfScriptReader::warnInvalidRange() {
         !Binary->addressIsCode(EndAddress))
       continue;
 
-    if (!Binary->addressIsCode(StartAddress) ||
-        !Binary->addressIsTransfer(EndAddress)) {
-      InstNotBoundary += I.second;
-      WarnInvalidRange(StartAddress, EndAddress, EndNotBoundaryMsg);
-    }
+    // IP samples can indicate activity on individual instructions rather than
+    // basic blocks/edges. In this mode, don't warn if sampled IPs aren't
+    // branches.
+    if (!LeadingIPOnly)
+      if (!Binary->addressIsCode(StartAddress) ||
+          !Binary->addressIsTransfer(EndAddress)) {
+        InstNotBoundary += I.second;
+        WarnInvalidRange(StartAddress, EndAddress, EndNotBoundaryMsg);
+      }
 
     auto *FRange = Binary->findFuncRange(StartAddress);
     if (!FRange) {
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index 53a25b279b432..175556c2220e6 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -104,6 +104,8 @@ cl::opt<bool> InferMissingFrames(
         "Infer missing call frames due to compiler tail call elimination."),
     llvm::cl::Optional);
 
+extern cl::opt<bool> LeadingIPOnly;
+
 using namespace llvm;
 using namespace sampleprof;
 
@@ -388,18 +390,25 @@ void ProfileGeneratorBase::updateBodySamplesforFunctionProfile(
   // Use the maximum count of samples with same line location
   uint32_t Discriminator = getBaseDiscriminator(LeafLoc.Location.Discriminator);
 
-  // Use duplication factor to compensated for loop unroll/vectorization.
-  // Note that this is only needed when we're taking MAX of the counts at
-  // the location instead of SUM.
-  Count *= getDuplicationFactor(LeafLoc.Location.Discriminator);
-
-  ErrorOr<uint64_t> R =
-      FunctionProfile.findSamplesAt(LeafLoc.Location.LineOffset, Discriminator);
-
-  uint64_t PreviousCount = R ? R.get() : 0;
-  if (PreviousCount <= Count) {
+  if (LeadingIPOnly) {
+    // When computing an IP-based profile we take the SUM of counts at the
+    // location instead of applying duplication factors and taking the MAX.
     FunctionProfile.addBodySamples(LeafLoc.Location.LineOffset, Discriminator,
-                                   Count - PreviousCount);
+                                   Count);
+  } else {
+    // Otherwise, use duplication factor to compensate for loop
+    // unroll/vectorization. Note that this is only needed when we're taking
+    // MAX of the counts at the location instead of SUM.
+    Count *= getDuplicationFactor(LeafLoc.Location.Discriminator);
+
+    ErrorOr<uint64_t> R = FunctionProfile.findSamplesAt(
+        LeafLoc.Location.LineOffset, Discriminator);
+
+    uint64_t PreviousCount = R ? R.get() : 0;
+    if (PreviousCount <= Count) {
+      FunctionProfile.addBodySamples(LeafLoc.Location.LineOffset, Discriminator,
+                                     Count - PreviousCount);
+    }
   }
 }
 

From 05f986e143d3d258e52e1c61b827ca4243cef842 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 21 Jul 2024 10:11:37 +0100
Subject: [PATCH 330/486] [LV] Add tests for loops with switches.

---
 .../LoopVectorize/X86/predicate-switch.ll     | 557 ++++++++++++++++++
 .../LoopVectorize/predicate-switch.ll         |  93 +++
 2 files changed, 650 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/predicate-switch.ll

diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
new file mode 100644
index 0000000000000..b8ce3c40920a3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
@@ -0,0 +1,557 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -mtriple=x86_64-apple-macosx -mcpu=skylake-avx512 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=IC1 %s
+; RUN: opt -p loop-vectorize -mtriple=x86_64-apple-macosx -mcpu=skylake-avx512 -force-vector-interleave=2 -S %s | FileCheck --check-prefixes=IC2 %s
+
+define void @switch_default_to_latch_common_dest(ptr %start, ptr %end) {
+; IC1-LABEL: define void @switch_default_to_latch_common_dest(
+; IC1-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0:[0-9]+]] {
+; IC1-NEXT:  [[ENTRY:.*]]:
+; IC1-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC1:       [[LOOP_HEADER]]:
+; IC1-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC1-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    switch i64 [[L]], label %[[LOOP_LATCH]] [
+; IC1-NEXT:      i64 -12, label %[[IF_THEN:.*]]
+; IC1-NEXT:      i64 13, label %[[IF_THEN]]
+; IC1-NEXT:    ]
+; IC1:       [[IF_THEN]]:
+; IC1-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[LOOP_LATCH]]:
+; IC1-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC1-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC1-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC1:       [[EXIT]]:
+; IC1-NEXT:    ret void
+;
+; IC2-LABEL: define void @switch_default_to_latch_common_dest(
+; IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0:[0-9]+]] {
+; IC2-NEXT:  [[ENTRY:.*]]:
+; IC2-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC2:       [[LOOP_HEADER]]:
+; IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC2-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    switch i64 [[L]], label %[[LOOP_LATCH]] [
+; IC2-NEXT:      i64 -12, label %[[IF_THEN:.*]]
+; IC2-NEXT:      i64 13, label %[[IF_THEN]]
+; IC2-NEXT:    ]
+; IC2:       [[IF_THEN]]:
+; IC2-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[LOOP_LATCH]]:
+; IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC2-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC2-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC2:       [[EXIT]]:
+; IC2-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
+  %l = load i64, ptr %ptr.iv, align 1
+  switch i64 %l, label %loop.latch [
+  i64 -12, label %if.then
+  i64 13, label %if.then
+  ]
+
+if.then:
+  store i64 42, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
+  %ec = icmp eq ptr %ptr.iv.next, %end
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @switch_all_dests_distinct(ptr %start, ptr %end) {
+; IC1-LABEL: define void @switch_all_dests_distinct(
+; IC1-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
+; IC1-NEXT:  [[ENTRY:.*]]:
+; IC1-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC1:       [[LOOP_HEADER]]:
+; IC1-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC1-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
+; IC1-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
+; IC1-NEXT:      i64 13, label %[[IF_THEN_2:.*]]
+; IC1-NEXT:      i64 0, label %[[IF_THEN_3:.*]]
+; IC1-NEXT:    ]
+; IC1:       [[IF_THEN_1]]:
+; IC1-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[IF_THEN_2]]:
+; IC1-NEXT:    store i64 0, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[IF_THEN_3]]:
+; IC1-NEXT:    store i64 1, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[DEFAULT]]:
+; IC1-NEXT:    store i64 2, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[LOOP_LATCH]]:
+; IC1-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC1-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC1-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC1:       [[EXIT]]:
+; IC1-NEXT:    ret void
+;
+; IC2-LABEL: define void @switch_all_dests_distinct(
+; IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
+; IC2-NEXT:  [[ENTRY:.*]]:
+; IC2-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC2:       [[LOOP_HEADER]]:
+; IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC2-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
+; IC2-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
+; IC2-NEXT:      i64 13, label %[[IF_THEN_2:.*]]
+; IC2-NEXT:      i64 0, label %[[IF_THEN_3:.*]]
+; IC2-NEXT:    ]
+; IC2:       [[IF_THEN_1]]:
+; IC2-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[IF_THEN_2]]:
+; IC2-NEXT:    store i64 0, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[IF_THEN_3]]:
+; IC2-NEXT:    store i64 1, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[DEFAULT]]:
+; IC2-NEXT:    store i64 2, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[LOOP_LATCH]]:
+; IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC2-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC2-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC2:       [[EXIT]]:
+; IC2-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
+  %l = load i64, ptr %ptr.iv, align 1
+  switch i64 %l, label %default [
+  i64 -12, label %if.then.1
+  i64 13, label %if.then.2
+  i64 0, label %if.then.3
+  ]
+
+if.then.1:
+  store i64 42, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+if.then.2:
+  store i64 0, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+if.then.3:
+  store i64 1, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+default:
+  store i64 2, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
+  %ec = icmp eq ptr %ptr.iv.next, %end
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+
+define void @switch_multiple_common_dests(ptr %start, ptr %end) {
+; IC1-LABEL: define void @switch_multiple_common_dests(
+; IC1-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
+; IC1-NEXT:  [[ENTRY:.*]]:
+; IC1-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC1:       [[LOOP_HEADER]]:
+; IC1-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC1-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
+; IC1-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
+; IC1-NEXT:      i64 0, label %[[IF_THEN_1]]
+; IC1-NEXT:      i64 13, label %[[IF_THEN_2:.*]]
+; IC1-NEXT:      i64 14, label %[[IF_THEN_2]]
+; IC1-NEXT:      i64 15, label %[[IF_THEN_2]]
+; IC1-NEXT:    ]
+; IC1:       [[IF_THEN_1]]:
+; IC1-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[IF_THEN_2]]:
+; IC1-NEXT:    store i64 0, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[DEFAULT]]:
+; IC1-NEXT:    store i64 2, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[LOOP_LATCH]]:
+; IC1-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC1-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC1-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC1:       [[EXIT]]:
+; IC1-NEXT:    ret void
+;
+; IC2-LABEL: define void @switch_multiple_common_dests(
+; IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
+; IC2-NEXT:  [[ENTRY:.*]]:
+; IC2-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC2:       [[LOOP_HEADER]]:
+; IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC2-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
+; IC2-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
+; IC2-NEXT:      i64 0, label %[[IF_THEN_1]]
+; IC2-NEXT:      i64 13, label %[[IF_THEN_2:.*]]
+; IC2-NEXT:      i64 14, label %[[IF_THEN_2]]
+; IC2-NEXT:      i64 15, label %[[IF_THEN_2]]
+; IC2-NEXT:    ]
+; IC2:       [[IF_THEN_1]]:
+; IC2-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[IF_THEN_2]]:
+; IC2-NEXT:    store i64 0, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[DEFAULT]]:
+; IC2-NEXT:    store i64 2, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[LOOP_LATCH]]:
+; IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC2-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC2-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC2:       [[EXIT]]:
+; IC2-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
+  %l = load i64, ptr %ptr.iv, align 1
+  switch i64 %l, label %default [
+  i64 -12, label %if.then.1
+  i64 0, label %if.then.1
+  i64 13, label %if.then.2
+  i64 14, label %if.then.2
+  i64 15, label %if.then.2
+  ]
+
+if.then.1:
+  store i64 42, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+if.then.2:
+  store i64 0, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+default:
+  store i64 2, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
+  %ec = icmp eq ptr %ptr.iv.next, %end
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
+; IC1-LABEL: define void @switch4_default_common_dest_with_case(
+; IC1-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
+; IC1-NEXT:  [[ENTRY:.*]]:
+; IC1-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC1:       [[LOOP_HEADER]]:
+; IC1-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC1-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
+; IC1-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
+; IC1-NEXT:      i64 13, label %[[IF_THEN_2:.*]]
+; IC1-NEXT:      i64 0, label %[[DEFAULT]]
+; IC1-NEXT:    ]
+; IC1:       [[IF_THEN_1]]:
+; IC1-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[IF_THEN_2]]:
+; IC1-NEXT:    store i64 0, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[DEFAULT]]:
+; IC1-NEXT:    store i64 2, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[LOOP_LATCH]]:
+; IC1-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC1-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC1-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC1:       [[EXIT]]:
+; IC1-NEXT:    ret void
+;
+; IC2-LABEL: define void @switch4_default_common_dest_with_case(
+; IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
+; IC2-NEXT:  [[ENTRY:.*]]:
+; IC2-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC2:       [[LOOP_HEADER]]:
+; IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC2-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
+; IC2-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
+; IC2-NEXT:      i64 13, label %[[IF_THEN_2:.*]]
+; IC2-NEXT:      i64 0, label %[[DEFAULT]]
+; IC2-NEXT:    ]
+; IC2:       [[IF_THEN_1]]:
+; IC2-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[IF_THEN_2]]:
+; IC2-NEXT:    store i64 0, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[DEFAULT]]:
+; IC2-NEXT:    store i64 2, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[LOOP_LATCH]]:
+; IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC2-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC2-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC2:       [[EXIT]]:
+; IC2-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
+  %l = load i64, ptr %ptr.iv, align 1
+  switch i64 %l, label %default [
+  i64 -12, label %if.then.1
+  i64 13, label %if.then.2
+  i64 0, label %default
+  ]
+
+if.then.1:
+  store i64 42, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+if.then.2:
+  store i64 0, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+default:
+  store i64 2, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
+  %ec = icmp eq ptr %ptr.iv.next, %end
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @switch_under_br_default_common_dest_with_case(ptr %start, ptr %end, i64 %x) {
+; IC1-LABEL: define void @switch_under_br_default_common_dest_with_case(
+; IC1-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
+; IC1-NEXT:  [[ENTRY:.*]]:
+; IC1-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC1:       [[LOOP_HEADER]]:
+; IC1-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC1-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    [[C:%.*]] = icmp ule i64 [[L]], [[X]]
+; IC1-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; IC1:       [[THEN]]:
+; IC1-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
+; IC1-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
+; IC1-NEXT:      i64 13, label %[[IF_THEN_2:.*]]
+; IC1-NEXT:      i64 0, label %[[DEFAULT]]
+; IC1-NEXT:    ]
+; IC1:       [[IF_THEN_1]]:
+; IC1-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[IF_THEN_2]]:
+; IC1-NEXT:    store i64 0, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[DEFAULT]]:
+; IC1-NEXT:    store i64 2, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[LOOP_LATCH]]:
+; IC1-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC1-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC1-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC1:       [[EXIT]]:
+; IC1-NEXT:    ret void
+;
+; IC2-LABEL: define void @switch_under_br_default_common_dest_with_case(
+; IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
+; IC2-NEXT:  [[ENTRY:.*]]:
+; IC2-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC2:       [[LOOP_HEADER]]:
+; IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC2-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    [[C:%.*]] = icmp ule i64 [[L]], [[X]]
+; IC2-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; IC2:       [[THEN]]:
+; IC2-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
+; IC2-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
+; IC2-NEXT:      i64 13, label %[[IF_THEN_2:.*]]
+; IC2-NEXT:      i64 0, label %[[DEFAULT]]
+; IC2-NEXT:    ]
+; IC2:       [[IF_THEN_1]]:
+; IC2-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[IF_THEN_2]]:
+; IC2-NEXT:    store i64 0, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[DEFAULT]]:
+; IC2-NEXT:    store i64 2, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[LOOP_LATCH]]:
+; IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC2-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC2-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC2:       [[EXIT]]:
+; IC2-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
+  %l = load i64, ptr %ptr.iv, align 1
+  %c = icmp ule i64 %l, %x
+  br i1 %c, label %then, label %loop.latch
+
+then:
+  switch i64 %l, label %default [
+  i64 -12, label %if.then.1
+  i64 13, label %if.then.2
+  i64 0, label %default
+  ]
+
+if.then.1:
+  store i64 42, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+if.then.2:
+  store i64 0, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+default:
+  store i64 2, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
+  %ec = icmp eq ptr %ptr.iv.next, %end
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end, i64 %x) {
+; IC1-LABEL: define void @br_under_switch_default_common_dest_with_case(
+; IC1-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
+; IC1-NEXT:  [[ENTRY:.*]]:
+; IC1-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC1:       [[LOOP_HEADER]]:
+; IC1-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC1-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
+; IC1-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
+; IC1-NEXT:      i64 13, label %[[IF_THEN_2:.*]]
+; IC1-NEXT:      i64 0, label %[[DEFAULT]]
+; IC1-NEXT:    ]
+; IC1:       [[IF_THEN_1]]:
+; IC1-NEXT:    [[C:%.*]] = icmp ule i64 [[L]], [[X]]
+; IC1-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[IF_THEN_2]]
+; IC1:       [[THEN]]:
+; IC1-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[DEFAULT]]
+; IC1:       [[IF_THEN_2]]:
+; IC1-NEXT:    store i64 0, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[DEFAULT]]:
+; IC1-NEXT:    store i64 2, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[LOOP_LATCH]]:
+; IC1-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC1-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC1-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC1:       [[EXIT]]:
+; IC1-NEXT:    ret void
+;
+; IC2-LABEL: define void @br_under_switch_default_common_dest_with_case(
+; IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
+; IC2-NEXT:  [[ENTRY:.*]]:
+; IC2-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC2:       [[LOOP_HEADER]]:
+; IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC2-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
+; IC2-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
+; IC2-NEXT:      i64 13, label %[[IF_THEN_2:.*]]
+; IC2-NEXT:      i64 0, label %[[DEFAULT]]
+; IC2-NEXT:    ]
+; IC2:       [[IF_THEN_1]]:
+; IC2-NEXT:    [[C:%.*]] = icmp ule i64 [[L]], [[X]]
+; IC2-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[IF_THEN_2]]
+; IC2:       [[THEN]]:
+; IC2-NEXT:    store i64 42, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[DEFAULT]]
+; IC2:       [[IF_THEN_2]]:
+; IC2-NEXT:    store i64 0, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[DEFAULT]]:
+; IC2-NEXT:    store i64 2, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[LOOP_LATCH]]:
+; IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
+; IC2-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC2-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC2:       [[EXIT]]:
+; IC2-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
+  %l = load i64, ptr %ptr.iv, align 1
+  switch i64 %l, label %default [
+  i64 -12, label %if.then.1
+  i64 13, label %if.then.2
+  i64 0, label %default
+  ]
+
+if.then.1:
+  %c = icmp ule i64 %l, %x
+  br i1 %c, label %then, label %if.then.2
+
+then:
+  store i64 42, ptr %ptr.iv, align 1
+  br label %default
+
+if.then.2:
+  store i64 0, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+default:
+  store i64 2, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
+  %ec = icmp eq ptr %ptr.iv.next, %end
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/predicate-switch.ll
new file mode 100644
index 0000000000000..dba53c8717f59
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/predicate-switch.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=IC1 %s
+; RUN: opt -p loop-vectorize -force-vector-width=2 -force-vector-interleave=2 -S %s | FileCheck --check-prefixes=IC2 %s
+
+define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
+; IC1-LABEL: define void @switch4_default_common_dest_with_case(
+; IC1-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
+; IC1-NEXT:  [[ENTRY:.*]]:
+; IC1-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC1:       [[LOOP_HEADER]]:
+; IC1-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC1-NEXT:    [[L:%.*]] = load i8, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    switch i8 [[L]], label %[[DEFAULT:.*]] [
+; IC1-NEXT:      i8 -12, label %[[IF_THEN_1:.*]]
+; IC1-NEXT:      i8 13, label %[[IF_THEN_2:.*]]
+; IC1-NEXT:      i8 0, label %[[DEFAULT]]
+; IC1-NEXT:    ]
+; IC1:       [[IF_THEN_1]]:
+; IC1-NEXT:    store i8 42, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[IF_THEN_2]]:
+; IC1-NEXT:    store i8 0, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[DEFAULT]]:
+; IC1-NEXT:    store i8 2, ptr [[PTR_IV]], align 1
+; IC1-NEXT:    br label %[[LOOP_LATCH]]
+; IC1:       [[LOOP_LATCH]]:
+; IC1-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 1
+; IC1-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC1-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC1:       [[EXIT]]:
+; IC1-NEXT:    ret void
+;
+; IC2-LABEL: define void @switch4_default_common_dest_with_case(
+; IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
+; IC2-NEXT:  [[ENTRY:.*]]:
+; IC2-NEXT:    br label %[[LOOP_HEADER:.*]]
+; IC2:       [[LOOP_HEADER]]:
+; IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC2-NEXT:    [[L:%.*]] = load i8, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    switch i8 [[L]], label %[[DEFAULT:.*]] [
+; IC2-NEXT:      i8 -12, label %[[IF_THEN_1:.*]]
+; IC2-NEXT:      i8 13, label %[[IF_THEN_2:.*]]
+; IC2-NEXT:      i8 0, label %[[DEFAULT]]
+; IC2-NEXT:    ]
+; IC2:       [[IF_THEN_1]]:
+; IC2-NEXT:    store i8 42, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[IF_THEN_2]]:
+; IC2-NEXT:    store i8 0, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[DEFAULT]]:
+; IC2-NEXT:    store i8 2, ptr [[PTR_IV]], align 1
+; IC2-NEXT:    br label %[[LOOP_LATCH]]
+; IC2:       [[LOOP_LATCH]]:
+; IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 1
+; IC2-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; IC2-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; IC2:       [[EXIT]]:
+; IC2-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
+  %l = load i8, ptr %ptr.iv, align 1
+  switch i8 %l, label %default [
+  i8 -12, label %if.then.1
+  i8 13, label %if.then.2
+  i8 0, label %default
+  ]
+
+if.then.1:
+  store i8 42, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+if.then.2:
+  store i8 0, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+default:
+  store i8 2, ptr %ptr.iv, align 1
+  br label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 1
+  %ec = icmp eq ptr %ptr.iv.next, %end
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}

From a41a4ac78294c728fb70a51623c602ea7f3e308a Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen@sifive.com>
Date: Sun, 21 Jul 2024 19:05:29 +0800
Subject: [PATCH 331/486] [compiler-rt][RISCV] Implement
 __init_riscv_feature_bits (#85790)

Base on https://github.com/riscv-non-isa/riscv-c-api-doc/pull/74, this
patch defines the `__riscv_feature_bits` and
`__riscv_vendor_feature_bits` structures to store the enabled feature
bits at runtime.

It also introduces the `__init_riscv_feature_bits` function to update
these structures based on the platform query mechanism.

Additionally, the groupid/bitmask definitions from
https://github.com/riscv-non-isa/riscv-c-api-doc/pull/74 are declared
and used to update the `__riscv_feature_bits` and
`__riscv_vendor_feature_bits` structures.

---------

Co-authored-by: Kito Cheng <kito.cheng@gmail.com>
---
 compiler-rt/lib/builtins/CMakeLists.txt       |   1 +
 compiler-rt/lib/builtins/riscv/feature_bits.c | 298 ++++++++++++++++++
 2 files changed, 299 insertions(+)
 create mode 100644 compiler-rt/lib/builtins/riscv/feature_bits.c

diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index f780b917cbdb1..744bbfacf32f1 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -739,6 +739,7 @@ endif()
 set(powerpc64le_SOURCES ${powerpc64_SOURCES})
 
 set(riscv_SOURCES
+  riscv/feature_bits.c
   riscv/fp_mode.c
   riscv/save.S
   riscv/restore.S
diff --git a/compiler-rt/lib/builtins/riscv/feature_bits.c b/compiler-rt/lib/builtins/riscv/feature_bits.c
new file mode 100644
index 0000000000000..77422935bd2d3
--- /dev/null
+++ b/compiler-rt/lib/builtins/riscv/feature_bits.c
@@ -0,0 +1,298 @@
+//=== feature_bits.c - Update RISC-V Feature Bits Structure -*- C -*-=========//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#define RISCV_FEATURE_BITS_LENGTH 1
+struct {
+  unsigned length;
+  unsigned long long features[RISCV_FEATURE_BITS_LENGTH];
+} __riscv_feature_bits __attribute__((visibility("hidden"), nocommon));
+
+#define RISCV_VENDOR_FEATURE_BITS_LENGTH 1
+struct {
+  unsigned vendorID;
+  unsigned length;
+  unsigned long long features[RISCV_VENDOR_FEATURE_BITS_LENGTH];
+} __riscv_vendor_feature_bits __attribute__((visibility("hidden"), nocommon));
+
+// NOTE: Should sync-up with RISCVFeatures.td
+// TODO: Maybe generate a header from tablegen then include it.
+#define A_GROUPID 0
+#define A_BITMASK (1ULL << 0)
+#define C_GROUPID 0
+#define C_BITMASK (1ULL << 2)
+#define D_GROUPID 0
+#define D_BITMASK (1ULL << 3)
+#define F_GROUPID 0
+#define F_BITMASK (1ULL << 5)
+#define I_GROUPID 0
+#define I_BITMASK (1ULL << 8)
+#define M_GROUPID 0
+#define M_BITMASK (1ULL << 12)
+#define V_GROUPID 0
+#define V_BITMASK (1ULL << 21)
+#define ZACAS_GROUPID 0
+#define ZACAS_BITMASK (1ULL << 26)
+#define ZBA_GROUPID 0
+#define ZBA_BITMASK (1ULL << 27)
+#define ZBB_GROUPID 0
+#define ZBB_BITMASK (1ULL << 28)
+#define ZBC_GROUPID 0
+#define ZBC_BITMASK (1ULL << 29)
+#define ZBKB_GROUPID 0
+#define ZBKB_BITMASK (1ULL << 30)
+#define ZBKC_GROUPID 0
+#define ZBKC_BITMASK (1ULL << 31)
+#define ZBKX_GROUPID 0
+#define ZBKX_BITMASK (1ULL << 32)
+#define ZBS_GROUPID 0
+#define ZBS_BITMASK (1ULL << 33)
+#define ZFA_GROUPID 0
+#define ZFA_BITMASK (1ULL << 34)
+#define ZFH_GROUPID 0
+#define ZFH_BITMASK (1ULL << 35)
+#define ZFHMIN_GROUPID 0
+#define ZFHMIN_BITMASK (1ULL << 36)
+#define ZICBOZ_GROUPID 0
+#define ZICBOZ_BITMASK (1ULL << 37)
+#define ZICOND_GROUPID 0
+#define ZICOND_BITMASK (1ULL << 38)
+#define ZIHINTNTL_GROUPID 0
+#define ZIHINTNTL_BITMASK (1ULL << 39)
+#define ZIHINTPAUSE_GROUPID 0
+#define ZIHINTPAUSE_BITMASK (1ULL << 40)
+#define ZKND_GROUPID 0
+#define ZKND_BITMASK (1ULL << 41)
+#define ZKNE_GROUPID 0
+#define ZKNE_BITMASK (1ULL << 42)
+#define ZKNH_GROUPID 0
+#define ZKNH_BITMASK (1ULL << 43)
+#define ZKSED_GROUPID 0
+#define ZKSED_BITMASK (1ULL << 44)
+#define ZKSH_GROUPID 0
+#define ZKSH_BITMASK (1ULL << 45)
+#define ZKT_GROUPID 0
+#define ZKT_BITMASK (1ULL << 46)
+#define ZTSO_GROUPID 0
+#define ZTSO_BITMASK (1ULL << 47)
+#define ZVBB_GROUPID 0
+#define ZVBB_BITMASK (1ULL << 48)
+#define ZVBC_GROUPID 0
+#define ZVBC_BITMASK (1ULL << 49)
+#define ZVFH_GROUPID 0
+#define ZVFH_BITMASK (1ULL << 50)
+#define ZVFHMIN_GROUPID 0
+#define ZVFHMIN_BITMASK (1ULL << 51)
+#define ZVKB_GROUPID 0
+#define ZVKB_BITMASK (1ULL << 52)
+#define ZVKG_GROUPID 0
+#define ZVKG_BITMASK (1ULL << 53)
+#define ZVKNED_GROUPID 0
+#define ZVKNED_BITMASK (1ULL << 54)
+#define ZVKNHA_GROUPID 0
+#define ZVKNHA_BITMASK (1ULL << 55)
+#define ZVKNHB_GROUPID 0
+#define ZVKNHB_BITMASK (1ULL << 56)
+#define ZVKSED_GROUPID 0
+#define ZVKSED_BITMASK (1ULL << 57)
+#define ZVKSH_GROUPID 0
+#define ZVKSH_BITMASK (1ULL << 58)
+#define ZVKT_GROUPID 0
+#define ZVKT_BITMASK (1ULL << 59)
+
+#if defined(__linux__)
+
+static long syscall_impl_5_args(long number, long arg1, long arg2, long arg3,
+                                long arg4, long arg5) {
+  register long a7 __asm__("a7") = number;
+  register long a0 __asm__("a0") = arg1;
+  register long a1 __asm__("a1") = arg2;
+  register long a2 __asm__("a2") = arg3;
+  register long a3 __asm__("a3") = arg4;
+  register long a4 __asm__("a4") = arg5;
+  __asm__ __volatile__("ecall\n\t"
+                       : "=r"(a0)
+                       : "r"(a7), "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(a4)
+                       : "memory");
+  return a0;
+}
+
+#define RISCV_HWPROBE_KEY_MVENDORID 0
+#define RISCV_HWPROBE_KEY_MARCHID 1
+#define RISCV_HWPROBE_KEY_MIMPID 2
+#define RISCV_HWPROBE_KEY_BASE_BEHAVIOR 3
+#define RISCV_HWPROBE_BASE_BEHAVIOR_IMA (1ULL << 0)
+#define RISCV_HWPROBE_KEY_IMA_EXT_0 4
+#define RISCV_HWPROBE_IMA_FD (1ULL << 0)
+#define RISCV_HWPROBE_IMA_C (1ULL << 1)
+#define RISCV_HWPROBE_IMA_V (1ULL << 2)
+#define RISCV_HWPROBE_EXT_ZBA (1ULL << 3)
+#define RISCV_HWPROBE_EXT_ZBB (1ULL << 4)
+#define RISCV_HWPROBE_EXT_ZBS (1ULL << 5)
+#define RISCV_HWPROBE_EXT_ZICBOZ (1ULL << 6)
+#define RISCV_HWPROBE_EXT_ZBC (1ULL << 7)
+#define RISCV_HWPROBE_EXT_ZBKB (1ULL << 8)
+#define RISCV_HWPROBE_EXT_ZBKC (1ULL << 9)
+#define RISCV_HWPROBE_EXT_ZBKX (1ULL << 10)
+#define RISCV_HWPROBE_EXT_ZKND (1ULL << 11)
+#define RISCV_HWPROBE_EXT_ZKNE (1ULL << 12)
+#define RISCV_HWPROBE_EXT_ZKNH (1ULL << 13)
+#define RISCV_HWPROBE_EXT_ZKSED (1ULL << 14)
+#define RISCV_HWPROBE_EXT_ZKSH (1ULL << 15)
+#define RISCV_HWPROBE_EXT_ZKT (1ULL << 16)
+#define RISCV_HWPROBE_EXT_ZVBB (1ULL << 17)
+#define RISCV_HWPROBE_EXT_ZVBC (1ULL << 18)
+#define RISCV_HWPROBE_EXT_ZVKB (1ULL << 19)
+#define RISCV_HWPROBE_EXT_ZVKG (1ULL << 20)
+#define RISCV_HWPROBE_EXT_ZVKNED (1ULL << 21)
+#define RISCV_HWPROBE_EXT_ZVKNHA (1ULL << 22)
+#define RISCV_HWPROBE_EXT_ZVKNHB (1ULL << 23)
+#define RISCV_HWPROBE_EXT_ZVKSED (1ULL << 24)
+#define RISCV_HWPROBE_EXT_ZVKSH (1ULL << 25)
+#define RISCV_HWPROBE_EXT_ZVKT (1ULL << 26)
+#define RISCV_HWPROBE_EXT_ZFH (1ULL << 27)
+#define RISCV_HWPROBE_EXT_ZFHMIN (1ULL << 28)
+#define RISCV_HWPROBE_EXT_ZIHINTNTL (1ULL << 29)
+#define RISCV_HWPROBE_EXT_ZVFH (1ULL << 30)
+#define RISCV_HWPROBE_EXT_ZVFHMIN (1ULL << 31)
+#define RISCV_HWPROBE_EXT_ZFA (1ULL << 32)
+#define RISCV_HWPROBE_EXT_ZTSO (1ULL << 33)
+#define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34)
+#define RISCV_HWPROBE_EXT_ZICOND (1ULL << 35)
+#define RISCV_HWPROBE_EXT_ZIHINTPAUSE (1ULL << 36)
+#define RISCV_HWPROBE_KEY_CPUPERF_0 5
+#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)
+#define RISCV_HWPROBE_MISALIGNED_EMULATED (1ULL << 0)
+#define RISCV_HWPROBE_MISALIGNED_SLOW (2 << 0)
+#define RISCV_HWPROBE_MISALIGNED_FAST (3 << 0)
+#define RISCV_HWPROBE_MISALIGNED_UNSUPPORTED (4 << 0)
+#define RISCV_HWPROBE_MISALIGNED_MASK (7 << 0)
+#define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6
+/* Increase RISCV_HWPROBE_MAX_KEY when adding items. */
+
+struct riscv_hwprobe {
+  long long key;
+  unsigned long long value;
+};
+
+#define __NR_riscv_hwprobe 258
+static long initHwProbe(struct riscv_hwprobe *Hwprobes, int len) {
+  return syscall_impl_5_args(__NR_riscv_hwprobe, (long)Hwprobes, len, 0, 0, 0);
+}
+
+#define SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(EXTNAME)                    \
+  SET_SINGLE_IMAEXT_RISCV_FEATURE(RISCV_HWPROBE_EXT_##EXTNAME, EXTNAME)
+
+#define SET_SINGLE_IMAEXT_RISCV_FEATURE(HWPROBE_BITMASK, EXT)                  \
+  SET_SINGLE_RISCV_FEATURE(IMAEXT0Value &HWPROBE_BITMASK, EXT)
+
+#define SET_SINGLE_RISCV_FEATURE(COND, EXT)                                    \
+  if (COND) {                                                                  \
+    SET_RISCV_FEATURE(EXT);                                                    \
+  }
+
+#define SET_RISCV_FEATURE(EXT) features[EXT##_GROUPID] |= EXT##_BITMASK
+
+static void initRISCVFeature(struct riscv_hwprobe Hwprobes[]) {
+
+  // Note: If a hwprobe key is unknown to the kernel, its key field
+  // will be cleared to -1, and its value set to 0.
+  // This unsets all extension bitmask bits.
+
+  // Init vendor extension
+  __riscv_vendor_feature_bits.length = 0;
+  __riscv_vendor_feature_bits.vendorID = Hwprobes[2].value;
+
+  // Init standard extension
+  // TODO: Maybe Extension implied generate from tablegen?
+  __riscv_feature_bits.length = RISCV_FEATURE_BITS_LENGTH;
+
+  unsigned long long features[RISCV_FEATURE_BITS_LENGTH];
+  int i;
+
+  for (i = 0; i < RISCV_FEATURE_BITS_LENGTH; i++)
+    features[i] = 0;
+
+  // Check RISCV_HWPROBE_KEY_BASE_BEHAVIOR
+  unsigned long long BaseValue = Hwprobes[0].value;
+  if (BaseValue & RISCV_HWPROBE_BASE_BEHAVIOR_IMA) {
+    SET_RISCV_FEATURE(I);
+    SET_RISCV_FEATURE(M);
+    SET_RISCV_FEATURE(A);
+  }
+
+  // Check RISCV_HWPROBE_KEY_IMA_EXT_0
+  unsigned long long IMAEXT0Value = Hwprobes[1].value;
+  if (IMAEXT0Value & RISCV_HWPROBE_IMA_FD) {
+    SET_RISCV_FEATURE(F);
+    SET_RISCV_FEATURE(D);
+  }
+
+  SET_SINGLE_IMAEXT_RISCV_FEATURE(RISCV_HWPROBE_IMA_C, C);
+  SET_SINGLE_IMAEXT_RISCV_FEATURE(RISCV_HWPROBE_IMA_V, V);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZBA);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZBB);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZBS);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZICBOZ);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZBC);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZBKB);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZBKC);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZBKX);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZKND);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZKNE);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZKNH);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZKSED);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZKSH);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZKT);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVBB);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVBC);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVKB);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVKG);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVKNED);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVKNHA);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVKNHB);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVKSED);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVKSH);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVKT);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZFH);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZFHMIN);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZIHINTNTL);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZIHINTPAUSE);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVFH);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZVFHMIN);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZFA);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZTSO);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZACAS);
+  SET_RISCV_HWPROBE_EXT_SINGLE_RISCV_FEATURE(ZICOND);
+
+  for (i = 0; i < RISCV_FEATURE_BITS_LENGTH; i++)
+    __riscv_feature_bits.features[i] = features[i];
+}
+
+#endif // defined(__linux__)
+
+static int FeaturesBitCached = 0;
+
+void __init_riscv_feature_bits() {
+
+  if (FeaturesBitCached)
+    return;
+
+#if defined(__linux__)
+  struct riscv_hwprobe Hwprobes[] = {
+      {RISCV_HWPROBE_KEY_BASE_BEHAVIOR, 0},
+      {RISCV_HWPROBE_KEY_IMA_EXT_0, 0},
+      {RISCV_HWPROBE_KEY_MVENDORID, 0},
+  };
+  if (initHwProbe(Hwprobes, sizeof(Hwprobes) / sizeof(Hwprobes[0])))
+    return;
+
+  initRISCVFeature(Hwprobes);
+#endif // defined(__linux__)
+
+  FeaturesBitCached = 1;
+}

From 14ec4746cc342b3f62fcdd378e409aa1b0e39c3c Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sun, 21 Jul 2024 13:06:02 +0200
Subject: [PATCH 332/486] [libc++] Makes `unique_ptr operator*() noexcept.
 (#98047)

This implements
 - LWG2762  unique_ptr operator*() should be noexcept.

Differential Revision: https://reviews.llvm.org/D128214
---
 libcxx/docs/Status/Cxx23Issues.csv            |  2 +-
 libcxx/include/__memory/unique_ptr.h          | 16 ++++++-
 libcxx/include/memory                         |  3 +-
 libcxx/include/optional                       | 16 +++----
 .../noexcept_operator_star.compile.pass.cpp   | 25 +++++++++++
 .../dereference.pass.cpp                      | 10 +----
 .../dereference_const.pass.cpp                | 10 +----
 .../dereference_const_rvalue.pass.cpp         | 10 +----
 .../dereference_rvalue.pass.cpp               | 10 +----
 .../optional.object.observe/op_arrow.pass.cpp |  9 +---
 .../op_arrow_const.pass.cpp                   |  9 +---
 .../dereference.single.pass.cpp               | 42 ++++++++++++++++++-
 12 files changed, 97 insertions(+), 65 deletions(-)
 create mode 100644 libcxx/test/std/utilities/memory/unique.ptr/noexcept_operator_star.compile.pass.cpp

diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index 547d38c25c1da..50104052effad 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -99,7 +99,7 @@
 "","","","","",""
 `2191 <https://wg21.link/LWG2191>`__,"Incorrect specification of ``match_results(match_results&&)``","October 2021","|Nothing To Do|",""
 `2381 <https://wg21.link/LWG2381>`__,"Inconsistency in parsing floating point numbers","October 2021","|Complete|","19.0"
-`2762 <https://wg21.link/LWG2762>`__,"``unique_ptr operator*()`` should be ``noexcept``","October 2021","",""
+`2762 <https://wg21.link/LWG2762>`__,"``unique_ptr operator*()`` should be ``noexcept``","October 2021","|Complete|","19.0"
 `3121 <https://wg21.link/LWG3121>`__,"``tuple`` constructor constraints for ``UTypes&&...`` overloads","October 2021","",""
 `3123 <https://wg21.link/LWG3123>`__,"``duration`` constructor from representation shouldn't be effectively non-throwing","October 2021","","","|chrono|"
 `3146 <https://wg21.link/LWG3146>`__,"Excessive unwrapping in ``std::ref/cref``","October 2021","|Complete|","14.0"
diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h
index 9519e4283868b..f75259473efb1 100644
--- a/libcxx/include/__memory/unique_ptr.h
+++ b/libcxx/include/__memory/unique_ptr.h
@@ -36,7 +36,9 @@
 #include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/is_void.h>
 #include <__type_traits/remove_extent.h>
+#include <__type_traits/remove_pointer.h>
 #include <__type_traits/type_identity.h>
+#include <__utility/declval.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
 #include <cstddef>
@@ -50,6 +52,17 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Ptr>
+struct __is_noexcept_deref_or_void {
+  static constexpr bool value = noexcept(*std::declval<_Ptr>());
+};
+
+template <>
+struct __is_noexcept_deref_or_void<void*> : true_type {};
+#endif
+
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS default_delete {
   static_assert(!is_function<_Tp>::value, "default_delete cannot be instantiated for function types");
@@ -252,7 +265,8 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr {
     return *this;
   }
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 __add_lvalue_reference_t<_Tp> operator*() const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 __add_lvalue_reference_t<_Tp> operator*() const
+      _NOEXCEPT_(__is_noexcept_deref_or_void<pointer>::value) {
     return *__ptr_.first();
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer operator->() const _NOEXCEPT { return __ptr_.first(); }
diff --git a/libcxx/include/memory b/libcxx/include/memory
index e1b8462191f84..b940a32c3ebe6 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -451,7 +451,8 @@ public:
     constexpr unique_ptr& operator=(nullptr_t) noexcept;                              // constexpr since C++23
 
     // observers
-    typename constexpr add_lvalue_reference<T>::type operator*() const;               // constexpr since C++23
+    constexpr
+    add_lvalue_reference<T>::type operator*() const noexcept(see below);              // constexpr since C++23
     constexpr pointer operator->() const noexcept;                                    // constexpr since C++23
     constexpr pointer get() const noexcept;                                           // constexpr since C++23
     constexpr deleter_type& get_deleter() noexcept;                                   // constexpr since C++23
diff --git a/libcxx/include/optional b/libcxx/include/optional
index e550745c17c00..f9cbcbfa595d1 100644
--- a/libcxx/include/optional
+++ b/libcxx/include/optional
@@ -136,12 +136,12 @@ namespace std {
     void swap(optional &) noexcept(see below ); // constexpr in C++20
 
     // [optional.observe], observers
-    constexpr T const *operator->() const;
-    constexpr T *operator->();
-    constexpr T const &operator*() const &;
-    constexpr T &operator*() &;
-    constexpr T &&operator*() &&;
-    constexpr const T &&operator*() const &&;
+    constexpr T const *operator->() const noexcept;
+    constexpr T *operator->() noexcept;
+    constexpr T const &operator*() const & noexcept;
+    constexpr T &operator*() & noexcept;
+    constexpr T &&operator*() && noexcept;
+    constexpr const T &&operator*() const && noexcept;
     constexpr explicit operator bool() const noexcept;
     constexpr bool has_value() const noexcept;
     constexpr T const &value() const &;
@@ -783,12 +783,12 @@ public:
     }
   }
 
-  _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<value_type const> operator->() const {
+  _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<value_type const> operator->() const noexcept {
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator-> called on a disengaged value");
     return std::addressof(this->__get());
   }
 
-  _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<value_type> operator->() {
+  _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<value_type> operator->() noexcept {
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator-> called on a disengaged value");
     return std::addressof(this->__get());
   }
diff --git a/libcxx/test/std/utilities/memory/unique.ptr/noexcept_operator_star.compile.pass.cpp b/libcxx/test/std/utilities/memory/unique.ptr/noexcept_operator_star.compile.pass.cpp
new file mode 100644
index 0000000000000..a2d788005f0cd
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/unique.ptr/noexcept_operator_star.compile.pass.cpp
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03
+
+// unique_ptr
+
+// add_lvalue_reference_t<T> operator*() const noexcept(noexcept(*declval<pointer>()));
+
+// Dereferencing pointer directly in noexcept fails for a void pointer.  This
+// is not SFINAE-ed away leading to a hard error. The issue was originally
+// triggered by
+// test/std/utilities/memory/unique.ptr/iterator_concept_conformance.compile.pass.cpp
+//
+// This test validates whether the code compiles.
+
+#include <memory>
+
+extern const std::unique_ptr<void> p;
+void f() { [[maybe_unused]] bool b = noexcept(p.operator*()); }
diff --git a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference.pass.cpp b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference.pass.cpp
index 5b04e5a35aafa..49b4d21a28066 100644
--- a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference.pass.cpp
+++ b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference.pass.cpp
@@ -44,15 +44,7 @@ int main(int, char**)
     {
         optional<X> opt; ((void)opt);
         ASSERT_SAME_TYPE(decltype(*opt), X&);
-        LIBCPP_STATIC_ASSERT(noexcept(*opt));
-        // ASSERT_NOT_NOEXCEPT(*opt);
-        // FIXME: This assertion fails with GCC because it can see that
-        // (A) operator*() is constexpr, and
-        // (B) there is no path through the function that throws.
-        // It's arguable if this is the correct behavior for the noexcept
-        // operator.
-        // Regardless this function should still be noexcept(false) because
-        // it has a narrow contract.
+        ASSERT_NOEXCEPT(*opt);
     }
     {
         optional<X> opt(X{});
diff --git a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_const.pass.cpp b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_const.pass.cpp
index f323cd1a5e405..ff86d9534faf6 100644
--- a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_const.pass.cpp
+++ b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_const.pass.cpp
@@ -37,15 +37,7 @@ int main(int, char**)
     {
         const optional<X> opt; ((void)opt);
         ASSERT_SAME_TYPE(decltype(*opt), X const&);
-        LIBCPP_STATIC_ASSERT(noexcept(*opt));
-        // ASSERT_NOT_NOEXCEPT(*opt);
-        // FIXME: This assertion fails with GCC because it can see that
-        // (A) operator*() is constexpr, and
-        // (B) there is no path through the function that throws.
-        // It's arguable if this is the correct behavior for the noexcept
-        // operator.
-        // Regardless this function should still be noexcept(false) because
-        // it has a narrow contract.
+        ASSERT_NOEXCEPT(*opt);
     }
     {
         constexpr optional<X> opt(X{});
diff --git a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_const_rvalue.pass.cpp b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_const_rvalue.pass.cpp
index 68591c5e2dbcb..646857fdc0465 100644
--- a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_const_rvalue.pass.cpp
+++ b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_const_rvalue.pass.cpp
@@ -37,15 +37,7 @@ int main(int, char**)
     {
         const optional<X> opt; ((void)opt);
         ASSERT_SAME_TYPE(decltype(*std::move(opt)), X const &&);
-        LIBCPP_STATIC_ASSERT(noexcept(*opt));
-        // ASSERT_NOT_NOEXCEPT(*std::move(opt));
-        // FIXME: This assertion fails with GCC because it can see that
-        // (A) operator*() is constexpr, and
-        // (B) there is no path through the function that throws.
-        // It's arguable if this is the correct behavior for the noexcept
-        // operator.
-        // Regardless this function should still be noexcept(false) because
-        // it has a narrow contract.
+        ASSERT_NOEXCEPT(*std::move(opt));
     }
     {
         constexpr optional<X> opt(X{});
diff --git a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_rvalue.pass.cpp b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_rvalue.pass.cpp
index 67edbb903353e..16bf2e4336c69 100644
--- a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_rvalue.pass.cpp
+++ b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/dereference_rvalue.pass.cpp
@@ -44,15 +44,7 @@ int main(int, char**)
     {
         optional<X> opt; ((void)opt);
         ASSERT_SAME_TYPE(decltype(*std::move(opt)), X&&);
-        LIBCPP_STATIC_ASSERT(noexcept(*opt));
-        // ASSERT_NOT_NOEXCEPT(*std::move(opt));
-        // FIXME: This assertion fails with GCC because it can see that
-        // (A) operator*() is constexpr, and
-        // (B) there is no path through the function that throws.
-        // It's arguable if this is the correct behavior for the noexcept
-        // operator.
-        // Regardless this function should still be noexcept(false) because
-        // it has a narrow contract.
+        ASSERT_NOEXCEPT(*std::move(opt));
     }
     {
         optional<X> opt(X{});
diff --git a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/op_arrow.pass.cpp b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/op_arrow.pass.cpp
index 7cf043f9375c1..2b5fba546ef42 100644
--- a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/op_arrow.pass.cpp
+++ b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/op_arrow.pass.cpp
@@ -41,14 +41,7 @@ int main(int, char**)
     {
         std::optional<X> opt; ((void)opt);
         ASSERT_SAME_TYPE(decltype(opt.operator->()), X*);
-        // ASSERT_NOT_NOEXCEPT(opt.operator->());
-        // FIXME: This assertion fails with GCC because it can see that
-        // (A) operator->() is constexpr, and
-        // (B) there is no path through the function that throws.
-        // It's arguable if this is the correct behavior for the noexcept
-        // operator.
-        // Regardless this function should still be noexcept(false) because
-        // it has a narrow contract.
+        ASSERT_NOEXCEPT(opt.operator->());
     }
     {
         optional<X> opt(X{});
diff --git a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/op_arrow_const.pass.cpp b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/op_arrow_const.pass.cpp
index b9c1fe7794b66..d8ce932bd7810 100644
--- a/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/op_arrow_const.pass.cpp
+++ b/libcxx/test/std/utilities/optional/optional.object/optional.object.observe/op_arrow_const.pass.cpp
@@ -40,14 +40,7 @@ int main(int, char**)
     {
         const std::optional<X> opt; ((void)opt);
         ASSERT_SAME_TYPE(decltype(opt.operator->()), X const*);
-        // ASSERT_NOT_NOEXCEPT(opt.operator->());
-        // FIXME: This assertion fails with GCC because it can see that
-        // (A) operator->() is constexpr, and
-        // (B) there is no path through the function that throws.
-        // It's arguable if this is the correct behavior for the noexcept
-        // operator.
-        // Regardless this function should still be noexcept(false) because
-        // it has a narrow contract.
+        ASSERT_NOEXCEPT(opt.operator->());
     }
     {
         constexpr optional<X> opt(X{});
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/dereference.single.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/dereference.single.pass.cpp
index 4a1e9704afc9c..17902a4ce27a4 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/dereference.single.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/dereference.single.pass.cpp
@@ -14,12 +14,50 @@
 
 #include <memory>
 #include <cassert>
+#include <vector>
 
 #include "test_macros.h"
 
+#if TEST_STD_VER >= 11
+struct ThrowDereference {
+  TEST_CONSTEXPR_CXX23 ThrowDereference& operator*() noexcept(false);
+  TEST_CONSTEXPR_CXX23 operator bool() const { return false; }
+};
+
+struct Deleter {
+  using pointer = ThrowDereference;
+  TEST_CONSTEXPR_CXX23 void operator()(ThrowDereference&) const {}
+};
+#endif
+
 TEST_CONSTEXPR_CXX23 bool test() {
-  std::unique_ptr<int> p(new int(3));
-  assert(*p == 3);
+  ASSERT_NOEXCEPT(*(std::unique_ptr<void>{}));
+#if TEST_STD_VER >= 11
+  static_assert(noexcept(*std::declval<std::unique_ptr<void>>()), "");
+#endif
+  {
+    std::unique_ptr<int> p(new int(3));
+    assert(*p == 3);
+    ASSERT_NOEXCEPT(*p);
+  }
+#if TEST_STD_VER >= 11
+  {
+    std::unique_ptr<std::vector<int>> p(new std::vector<int>{3, 4, 5});
+    assert((*p)[0] == 3);
+    assert((*p)[1] == 4);
+    assert((*p)[2] == 5);
+    ASSERT_NOEXCEPT(*p);
+  }
+  {
+    std::unique_ptr<ThrowDereference> p;
+    ASSERT_NOEXCEPT(*p);
+  }
+  {
+    // The noexcept status of *unique_ptr<>::pointer should be propagated.
+    std::unique_ptr<ThrowDereference, Deleter> p;
+    ASSERT_NOT_NOEXCEPT(*p);
+  }
+#endif
 
   return true;
 }

From cbebacef5e12c985ca32c894bcfddec9adb2fc85 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Sun, 21 Jul 2024 12:57:31 +0100
Subject: [PATCH 333/486] [AArch64] Add UQXTN2 patterns

Similar to the existing UQXTN and SQXTN2 patterns, we can generate a UQXTN2
from concat(Vd, trunc(min(X, 255)))
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   | 11 ++++
 llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll | 56 ++++++++-----------
 .../test/CodeGen/AArch64/fptoui-sat-vector.ll | 31 ++++------
 llvm/test/CodeGen/AArch64/qmovn.ll            |  8 +--
 4 files changed, 49 insertions(+), 57 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index fe4589f4fbdae..643bcc33f9201 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -5363,6 +5363,17 @@ def : Pat<(v4i16 (trunc (smax (smin (v4i32 V128:$Vn), (v4i32 VImm7FFF)),
                               (v4i32 VImm8000)))),
           (SQXTNv4i16 V128:$Vn)>;
 
+// concat_vectors(Vd, trunc(umin(X, 255))) -> UQXTRN(Vd, Vn)
+def : Pat<(v16i8 (concat_vectors
+                 (v8i8 V64:$Vd),
+                 (v8i8 (trunc (umin (v8i16 V128:$Vn), (v8i16 VImmFF)))))),
+          (UQXTNv16i8 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>;
+// concat_vectors(Vd, trunc(umin(X, 65535))) -> UQXTRN(Vd, Vn)
+def : Pat<(v8i16 (concat_vectors
+                 (v4i16 V64:$Vd),
+                 (v4i16 (trunc (umin (v4i32 V128:$Vn), (v4i32 VImmFFFF)))))),
+          (UQXTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>;
+
 // concat_vectors(Vd, trunc(smin(smax Vm, -128), 127) ~> SQXTN2(Vd, Vn)
 // with reversed min/max
 def : Pat<(v16i8 (concat_vectors
diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
index 4e8bfcd9d7516..0138bef9c3845 100644
--- a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
@@ -283,14 +283,12 @@ entry:
 define <8 x i16> @utesth_f16i16(<8 x half> %x) {
 ; CHECK-CVT-LABEL: utesth_f16i16:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v0.8h
-; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-CVT-NEXT:    movi v1.2d, #0x00ffff0000ffff
-; CHECK-CVT-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-CVT-NEXT:    umin v2.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtzu v2.4s, v0.4s
+; CHECK-CVT-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-CVT-NEXT:    uqxtn2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: utesth_f16i16:
@@ -308,14 +306,12 @@ entry:
 define <8 x i16> @ustest_f16i16(<8 x half> %x) {
 ; CHECK-CVT-LABEL: ustest_f16i16:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v0.8h
-; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-CVT-NEXT:    movi v1.2d, #0x00ffff0000ffff
-; CHECK-CVT-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-CVT-NEXT:    umin v2.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtzu v2.4s, v0.4s
+; CHECK-CVT-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-CVT-NEXT:    uqxtn2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: ustest_f16i16:
@@ -909,14 +905,12 @@ entry:
 define <8 x i16> @utesth_f16i16_mm(<8 x half> %x) {
 ; CHECK-CVT-LABEL: utesth_f16i16_mm:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v0.8h
-; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-CVT-NEXT:    movi v1.2d, #0x00ffff0000ffff
-; CHECK-CVT-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-CVT-NEXT:    umin v2.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtzu v2.4s, v0.4s
+; CHECK-CVT-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-CVT-NEXT:    uqxtn2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: utesth_f16i16_mm:
@@ -933,14 +927,12 @@ entry:
 define <8 x i16> @ustest_f16i16_mm(<8 x half> %x) {
 ; CHECK-CVT-LABEL: ustest_f16i16_mm:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v0.8h
-; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-CVT-NEXT:    movi v1.2d, #0x00ffff0000ffff
-; CHECK-CVT-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-CVT-NEXT:    umin v2.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtzu v2.4s, v0.4s
+; CHECK-CVT-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-CVT-NEXT:    uqxtn2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: ustest_f16i16_mm:
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
index 16e04070b6543..b03d145d1408d 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
@@ -2509,12 +2509,10 @@ define <16 x i8> @test_unsigned_v16f32_v16i8(<16 x float> %f) {
 define <8 x i16> @test_unsigned_v8f32_v8i16(<8 x float> %f) {
 ; CHECK-LABEL: test_unsigned_v8f32_v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v2.2d, #0x00ffff0000ffff
-; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
 ; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    umin v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    umin v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    uqxtn v0.4h, v0.4s
+; CHECK-NEXT:    uqxtn2 v0.8h, v1.4s
 ; CHECK-NEXT:    ret
     %x = call <8 x i16> @llvm.fptoui.sat.v8f32.v8i16(<8 x float> %f)
     ret <8 x i16> %x
@@ -2523,17 +2521,14 @@ define <8 x i16> @test_unsigned_v8f32_v8i16(<8 x float> %f) {
 define <16 x i16> @test_unsigned_v16f32_v16i16(<16 x float> %f) {
 ; CHECK-LABEL: test_unsigned_v16f32_v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v4.2d, #0x00ffff0000ffff
-; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
 ; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    fcvtzu v3.4s, v3.4s
 ; CHECK-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-NEXT:    umin v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    umin v0.4s, v0.4s, v4.4s
-; CHECK-NEXT:    umin v3.4s, v3.4s, v4.4s
-; CHECK-NEXT:    umin v2.4s, v2.4s, v4.4s
-; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NEXT:    fcvtzu v4.4s, v1.4s
+; CHECK-NEXT:    uqxtn v0.4h, v0.4s
+; CHECK-NEXT:    uqxtn v1.4h, v2.4s
+; CHECK-NEXT:    fcvtzu v2.4s, v3.4s
+; CHECK-NEXT:    uqxtn2 v0.8h, v4.4s
+; CHECK-NEXT:    uqxtn2 v1.8h, v2.4s
 ; CHECK-NEXT:    ret
     %x = call <16 x i16> @llvm.fptoui.sat.v16f32.v16i16(<16 x float> %f)
     ret <16 x i16> %x
@@ -2632,12 +2627,10 @@ define <16 x i8> @test_unsigned_v16f16_v16i8(<16 x half> %f) {
 ;
 ; CHECK-FP16-LABEL: test_unsigned_v16f16_v16i8:
 ; CHECK-FP16:       // %bb.0:
-; CHECK-FP16-NEXT:    movi v2.2d, #0xff00ff00ff00ff
-; CHECK-FP16-NEXT:    fcvtzu v1.8h, v1.8h
 ; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-FP16-NEXT:    umin v1.8h, v1.8h, v2.8h
-; CHECK-FP16-NEXT:    umin v0.8h, v0.8h, v2.8h
-; CHECK-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-FP16-NEXT:    uqxtn v0.8b, v0.8h
+; CHECK-FP16-NEXT:    uqxtn2 v0.16b, v1.8h
 ; CHECK-FP16-NEXT:    ret
     %x = call <16 x i8> @llvm.fptoui.sat.v16f16.v16i8(<16 x half> %f)
     ret <16 x i8> %x
diff --git a/llvm/test/CodeGen/AArch64/qmovn.ll b/llvm/test/CodeGen/AArch64/qmovn.ll
index dbdf9c58f8aba..35c172adbad3d 100644
--- a/llvm/test/CodeGen/AArch64/qmovn.ll
+++ b/llvm/test/CodeGen/AArch64/qmovn.ll
@@ -250,10 +250,8 @@ entry:
 define <16 x i8> @unsigned_v8i16_to_v16i8(<8 x i8> %x, <8 x i16> %y) {
 ; CHECK-LABEL: unsigned_v8i16_to_v16i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v2.2d, #0xff00ff00ff00ff
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umin v1.8h, v1.8h, v2.8h
-; CHECK-NEXT:    xtn2 v0.16b, v1.8h
+; CHECK-NEXT:    uqxtn2 v0.16b, v1.8h
 ; CHECK-NEXT:    ret
 entry:
   %min = call <8 x i16> @llvm.umin.v8i16(<8 x i16> %y, <8 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
@@ -265,10 +263,8 @@ entry:
 define <8 x i16> @unsigned_v4i32_to_v8i16(<4 x i16> %x, <4 x i32> %y) {
 ; CHECK-LABEL: unsigned_v4i32_to_v8i16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v2.2d, #0x00ffff0000ffff
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umin v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    xtn2 v0.8h, v1.4s
+; CHECK-NEXT:    uqxtn2 v0.8h, v1.4s
 ; CHECK-NEXT:    ret
 entry:
   %min = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %y, <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>)

From f6eb89cdd02d73a3c9a0da858c3100986282aceb Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Sun, 21 Jul 2024 15:59:41 +0400
Subject: [PATCH 334/486] [lldb][Windows] Fixed Host::Kill() (#99721)

HostProcessWindows::Terminate() correctly uses m_process which type is
process_t (HANDLE) to call ::TerminateProcess(). But Host::Kill() uses a
cast from pid, which is wrong.

This patch fixes #51793
---
 lldb/source/Host/windows/Host.cpp                             | 4 +++-
 .../API/functionalities/gdb_remote_client/TestPlatformKill.py | 1 -
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lldb/source/Host/windows/Host.cpp b/lldb/source/Host/windows/Host.cpp
index 6908f0003eaf7..642092f61d924 100644
--- a/lldb/source/Host/windows/Host.cpp
+++ b/lldb/source/Host/windows/Host.cpp
@@ -103,7 +103,9 @@ lldb::thread_t Host::GetCurrentThread() {
 }
 
 void Host::Kill(lldb::pid_t pid, int signo) {
-  TerminateProcess((HANDLE)pid, 1);
+  AutoHandle handle(::OpenProcess(PROCESS_TERMINATE, FALSE, pid), nullptr);
+  if (handle.IsValid())
+    ::TerminateProcess(handle.get(), 1);
 }
 
 const char *Host::GetSignalAsCString(int signo) { return NULL; }
diff --git a/lldb/test/API/functionalities/gdb_remote_client/TestPlatformKill.py b/lldb/test/API/functionalities/gdb_remote_client/TestPlatformKill.py
index 46cda4d66b488..430f1871d3b30 100644
--- a/lldb/test/API/functionalities/gdb_remote_client/TestPlatformKill.py
+++ b/lldb/test/API/functionalities/gdb_remote_client/TestPlatformKill.py
@@ -8,7 +8,6 @@
 
 class TestPlatformKill(GDBRemoteTestBase):
     @skipIfRemote
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr52451")
     def test_kill_different_platform(self):
         """Test connecting to a remote linux platform"""
 

From c8c0b18b5d58415b79ea13b62eee70c130c26a0a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 21 Jul 2024 14:03:40 +0100
Subject: [PATCH 335/486] [LV] Update tests to not have dead interleave groups.

Update existing tests with dead interleave groups by adding users. This
ensures the tests keep testing what they were intended to test with a
planned change to skip unused instructions in cost computations.
---
 .../LoopVectorize/AArch64/interleaved_cost.ll |  76 +--
 .../LoopVectorize/ARM/interleaved_cost.ll     |  60 +-
 .../LoopVectorize/ARM/mve-interleaved-cost.ll | 630 +++++++++---------
 .../SystemZ/load-scalarization-cost-0.ll      |   6 +-
 .../first-order-recurrence-scalable-vf1.ll    |   4 +-
 5 files changed, 389 insertions(+), 387 deletions(-)

diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
index 78798725b5b9c..21af9ae801e16 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
@@ -15,21 +15,21 @@ entry:
 ; VF_8-LABEL:  Checking a loop in 'i8_factor_2'
 ; VF_8:          Found an estimated cost of 2 for VF 8 For instruction: %tmp2 = load i8, ptr %tmp0, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i8, ptr %tmp1, align 1
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i8 0, ptr %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 %tmp2, ptr %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i8 %tmp3, ptr %tmp1, align 1
 ; VF_16-LABEL: Checking a loop in 'i8_factor_2'
 ; VF_16:         Found an estimated cost of 2 for VF 16 For instruction: %tmp2 = load i8, ptr %tmp0, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i8, ptr %tmp1, align 1
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_16-NEXT:    Found an estimated cost of 2 for VF 16 For instruction: store i8 0, ptr %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 %tmp2, ptr %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 2 for VF 16 For instruction: store i8 %tmp3, ptr %tmp1, align 1
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i8, ptr %tmp0, align 1
   %tmp3 = load i8, ptr %tmp1, align 1
-  store i8 0, ptr %tmp0, align 1
-  store i8 0, ptr %tmp1, align 1
+  store i8 %tmp2, ptr %tmp0, align 1
+  store i8 %tmp3, ptr %tmp1, align 1
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -46,26 +46,26 @@ entry:
 ; VF_4-LABEL: Checking a loop in 'i16_factor_2'
 ; VF_4:          Found an estimated cost of 2 for VF 4 For instruction: %tmp2 = load i16, ptr %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i16, ptr %tmp1, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i16 0, ptr %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 %tmp2, ptr %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i16 %tmp3, ptr %tmp1, align 2
 ; VF_8-LABEL:  Checking a loop in 'i16_factor_2'
 ; VF_8:          Found an estimated cost of 2 for VF 8 For instruction: %tmp2 = load i16, ptr %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i16, ptr %tmp1, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i16 0, ptr %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 %tmp2, ptr %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i16 %tmp3, ptr %tmp1, align 2
 ; VF_16-LABEL: Checking a loop in 'i16_factor_2'
 ; VF_16:         Found an estimated cost of 4 for VF 16 For instruction: %tmp2 = load i16, ptr %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i16, ptr %tmp1, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_16-NEXT:    Found an estimated cost of 4 for VF 16 For instruction: store i16 0, ptr %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 %tmp2, ptr %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 4 for VF 16 For instruction: store i16 %tmp3, ptr %tmp1, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i16, ptr %tmp0, align 2
   %tmp3 = load i16, ptr %tmp1, align 2
-  store i16 0, ptr %tmp0, align 2
-  store i16 0, ptr %tmp1, align 2
+  store i16 %tmp2, ptr %tmp0, align 2
+  store i16 %tmp3, ptr %tmp1, align 2
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -82,31 +82,31 @@ entry:
 ; VF_2-LABEL:  Checking a loop in 'i32_factor_2'
 ; VF_2:          Found an estimated cost of 2 for VF 2 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_2-NEXT:     Found an estimated cost of 2 for VF 2 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 2 for VF 2 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 ; VF_4-LABEL:  Checking a loop in 'i32_factor_2'
 ; VF_4:          Found an estimated cost of 2 for VF 4 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 ; VF_8-LABEL:  Checking a loop in 'i32_factor_2'
 ; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 ; VF_16-LABEL: Checking a loop in 'i32_factor_2'
 ; VF_16:         Found an estimated cost of 8 for VF 16 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i32, ptr %tmp0, align 4
   %tmp3 = load i32, ptr %tmp1, align 4
-  store i32 0, ptr %tmp0, align 4
-  store i32 0, ptr %tmp1, align 4
+  store i32 %tmp2, ptr %tmp0, align 4
+  store i32 %tmp3, ptr %tmp1, align 4
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -123,31 +123,31 @@ entry:
 ; VF_2-LABEL:  Checking a loop in 'i64_factor_2'
 ; VF_2:          Found an estimated cost of 2 for VF 2 For instruction: %tmp2 = load i64, ptr %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i64, ptr %tmp1, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_2-NEXT:     Found an estimated cost of 2 for VF 2 For instruction: store i64 0, ptr %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 %tmp2, ptr %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 2 for VF 2 For instruction: store i64 %tmp3, ptr %tmp1, align 8
 ; VF_4-LABEL:  Checking a loop in 'i64_factor_2'
 ; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i64, ptr %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i64, ptr %tmp1, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i64 0, ptr %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 %tmp2, ptr %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i64 %tmp3, ptr %tmp1, align 8
 ; VF_8-LABEL:  Checking a loop in 'i64_factor_2'
 ; VF_8:          Found an estimated cost of 8 for VF 8 For instruction: %tmp2 = load i64, ptr %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i64, ptr %tmp1, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_8-NEXT:     Found an estimated cost of 8 for VF 8 For instruction: store i64 0, ptr %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 %tmp2, ptr %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 8 for VF 8 For instruction: store i64 %tmp3, ptr %tmp1, align 8
 ; VF_16-LABEL: Checking a loop in 'i64_factor_2'
 ; VF_16:         Found an estimated cost of 16 for VF 16 For instruction: %tmp2 = load i64, ptr %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i64, ptr %tmp1, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_16-NEXT:    Found an estimated cost of 16 for VF 16 For instruction: store i64 0, ptr %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 %tmp2, ptr %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 16 for VF 16 For instruction: store i64 %tmp3, ptr %tmp1, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i64.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i64.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i64, ptr %tmp0, align 8
   %tmp3 = load i64, ptr %tmp1, align 8
-  store i64 0, ptr %tmp0, align 8
-  store i64 0, ptr %tmp1, align 8
+  store i64 %tmp2, ptr %tmp0, align 8
+  store i64 %tmp3, ptr %tmp1, align 8
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -170,16 +170,16 @@ entry:
 ; VF_2-LABEL: Checking a loop in 'i64_factor_8'
 ; VF_2:         Found an estimated cost of 16 for VF 2 For instruction: %tmp2 = load i64, ptr %tmp0, align 8
 ; VF_2-NEXT:    Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i64, ptr %tmp1, align 8
-; VF_2-NEXT:    Found an estimated cost of 8 for VF 2 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_2-NEXT:    Found an estimated cost of 8 for VF 2 For instruction: store i64 0, ptr %tmp1, align 8
+; VF_2-NEXT:    Found an estimated cost of 12 for VF 2 For instruction: store i64 %tmp2, ptr %tmp0, align 8
+; VF_2-NEXT:    Found an estimated cost of 12 for VF 2 For instruction: store i64 %tmp3, ptr %tmp1, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i64.8, ptr %data, i64 %i, i32 2
   %tmp1 = getelementptr inbounds %i64.8, ptr %data, i64 %i, i32 6
   %tmp2 = load i64, ptr %tmp0, align 8
   %tmp3 = load i64, ptr %tmp1, align 8
-  store i64 0, ptr %tmp0, align 8
-  store i64 0, ptr %tmp1, align 8
+  store i64 %tmp2, ptr %tmp0, align 8
+  store i64 %tmp3, ptr %tmp1, align 8
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
index d08c16dcb3ed1..214d9abd712cd 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
@@ -15,21 +15,21 @@ entry:
 ; VF_8-LABEL:  Checking a loop in 'i8_factor_2'
 ; VF_8:          Found an estimated cost of 2 for VF 8 For instruction: %tmp2 = load i8, ptr %tmp0, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i8, ptr %tmp1, align 1
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i8 0, ptr %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 %tmp2, ptr %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i8 %tmp3, ptr %tmp1, align 1
 ; VF_16-LABEL: Checking a loop in 'i8_factor_2'
 ; VF_16:         Found an estimated cost of 2 for VF 16 For instruction: %tmp2 = load i8, ptr %tmp0, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i8, ptr %tmp1, align 1
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_16-NEXT:    Found an estimated cost of 2 for VF 16 For instruction: store i8 0, ptr %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 %tmp2, ptr %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 2 for VF 16 For instruction: store i8 %tmp3, ptr %tmp1, align 1
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i8, ptr %tmp0, align 1
   %tmp3 = load i8, ptr %tmp1, align 1
-  store i8 0, ptr %tmp0, align 1
-  store i8 0, ptr %tmp1, align 1
+  store i8 %tmp2, ptr %tmp0, align 1
+  store i8 %tmp3, ptr %tmp1, align 1
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -46,26 +46,26 @@ entry:
 ; VF_4-LABEL:  Checking a loop in 'i16_factor_2'
 ; VF_4:          Found an estimated cost of 2 for VF 4 For instruction: %tmp2 = load i16, ptr %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i16, ptr %tmp1, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i16 0, ptr %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 %tmp2, ptr %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i16 %tmp3, ptr %tmp1, align 2
 ; VF_8-LABEL:  Checking a loop in 'i16_factor_2'
 ; VF_8:          Found an estimated cost of 2 for VF 8 For instruction: %tmp2 = load i16, ptr %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i16, ptr %tmp1, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i16 0, ptr %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 %tmp2, ptr %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i16 %tmp3, ptr %tmp1, align 2
 ; VF_16-LABEL: Checking a loop in 'i16_factor_2'
 ; VF_16:         Found an estimated cost of 4 for VF 16 For instruction: %tmp2 = load i16, ptr %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i16, ptr %tmp1, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_16-NEXT:    Found an estimated cost of 4 for VF 16 For instruction: store i16 0, ptr %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 %tmp2, ptr %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 4 for VF 16 For instruction: store i16 %tmp3, ptr %tmp1, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i16, ptr %tmp0, align 2
   %tmp3 = load i16, ptr %tmp1, align 2
-  store i16 0, ptr %tmp0, align 2
-  store i16 0, ptr %tmp1, align 2
+  store i16 %tmp2, ptr %tmp0, align 2
+  store i16 %tmp3, ptr %tmp1, align 2
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -82,31 +82,31 @@ entry:
 ; VF_2-LABEL:  Checking a loop in 'i32_factor_2'
 ; VF_2:          Found an estimated cost of 2 for VF 2 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_2-NEXT:     Found an estimated cost of 2 for VF 2 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 2 for VF 2 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 ; VF_4-LABEL:  Checking a loop in 'i32_factor_2'
 ; VF_4:          Found an estimated cost of 2 for VF 4 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 ; VF_8-LABEL:  Checking a loop in 'i32_factor_2'
 ; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 ; VF_16-LABEL: Checking a loop in 'i32_factor_2'
 ; VF_16:         Found an estimated cost of 8 for VF 16 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i32, ptr %tmp0, align 4
   %tmp3 = load i32, ptr %tmp1, align 4
-  store i32 0, ptr %tmp0, align 4
-  store i32 0, ptr %tmp1, align 4
+  store i32 %tmp2, ptr %tmp0, align 4
+  store i32 %tmp3, ptr %tmp1, align 4
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -123,21 +123,21 @@ entry:
 ; VF_4-LABEL: Checking a loop in 'half_factor_2'
 ; VF_4:         Found an estimated cost of 40 for VF 4 For instruction: %tmp2 = load half, ptr %tmp0, align 2
 ; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load half, ptr %tmp1, align 2
-; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_4-NEXT:    Found an estimated cost of 32 for VF 4 For instruction: store half 0xH0000, ptr %tmp1, align 2
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store half %tmp2, ptr %tmp0, align 2
+; VF_4-NEXT:    Found an estimated cost of 40 for VF 4 For instruction: store half %tmp3, ptr %tmp1, align 2
 ; VF_8-LABEL: Checking a loop in 'half_factor_2'
 ; VF_8:         Found an estimated cost of 80 for VF 8 For instruction: %tmp2 = load half, ptr %tmp0, align 2
 ; VF_8-NEXT:    Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load half, ptr %tmp1, align 2
-; VF_8-NEXT:    Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_8-NEXT:    Found an estimated cost of 64 for VF 8 For instruction: store half 0xH0000, ptr %tmp1, align 2
+; VF_8-NEXT:    Found an estimated cost of 0 for VF 8 For instruction: store half %tmp2, ptr %tmp0, align 2
+; VF_8-NEXT:    Found an estimated cost of 80 for VF 8 For instruction: store half %tmp3, ptr %tmp1, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %half.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %half.2, ptr %data, i64 %i, i32 1
   %tmp2 = load half, ptr %tmp0, align 2
   %tmp3 = load half, ptr %tmp1, align 2
-  store half 0., ptr %tmp0, align 2
-  store half 0., ptr %tmp1, align 2
+  store half %tmp2, ptr %tmp0, align 2
+  store half %tmp3, ptr %tmp1, align 2
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
index 97aa2035ded55..c7a04e3669ed6 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
@@ -17,31 +17,31 @@ entry:
 ; VF_2-LABEL:  Checking a loop in 'i8_factor_2'
 ; VF_2:          Found an estimated cost of 24 for VF 2 For instruction: %tmp2 = load i8, ptr %tmp0, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i8, ptr %tmp1, align 1
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store i8 0, ptr %tmp1, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 %tmp2, ptr %tmp0, align 1
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i8 %tmp3, ptr %tmp1, align 1
 ; VF_4-LABEL:  Checking a loop in 'i8_factor_2'
 ; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i8, ptr %tmp0, align 1
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i8, ptr %tmp1, align 1
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i8 0, ptr %tmp1, align 1
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 %tmp2, ptr %tmp0, align 1
+; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i8 %tmp3, ptr %tmp1, align 1
 ; VF_8-LABEL:  Checking a loop in 'i8_factor_2'
 ; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load i8, ptr %tmp0, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i8, ptr %tmp1, align 1
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i8 0, ptr %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 %tmp2, ptr %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i8 %tmp3, ptr %tmp1, align 1
 ; VF_16-LABEL: Checking a loop in 'i8_factor_2'
 ; VF_16:         Found an estimated cost of 4 for VF 16 For instruction: %tmp2 = load i8, ptr %tmp0, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i8, ptr %tmp1, align 1
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_16-NEXT:    Found an estimated cost of 4 for VF 16 For instruction: store i8 0, ptr %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 %tmp2, ptr %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 4 for VF 16 For instruction: store i8 %tmp3, ptr %tmp1, align 1
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i8, ptr %tmp0, align 1
   %tmp3 = load i8, ptr %tmp1, align 1
-  store i8 0, ptr %tmp0, align 1
-  store i8 0, ptr %tmp1, align 1
+  store i8 %tmp2, ptr %tmp0, align 1
+  store i8 %tmp3, ptr %tmp1, align 1
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -58,31 +58,31 @@ entry:
 ; VF_2-LABEL:  Checking a loop in 'i16_factor_2'
 ; VF_2:          Found an estimated cost of 24 for VF 2 For instruction: %tmp2 = load i16, ptr %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i16, ptr %tmp1, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store i16 0, ptr %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 %tmp2, ptr %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i16 %tmp3, ptr %tmp1, align 2
 ; VF_4-LABEL:  Checking a loop in 'i16_factor_2'
 ; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i16, ptr %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i16, ptr %tmp1, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i16 0, ptr %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 %tmp2, ptr %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i16 %tmp3, ptr %tmp1, align 2
 ; VF_8-LABEL:  Checking a loop in 'i16_factor_2'
 ; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load i16, ptr %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i16, ptr %tmp1, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i16 0, ptr %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 %tmp2, ptr %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i16 %tmp3, ptr %tmp1, align 2
 ; VF_16-LABEL: Checking a loop in 'i16_factor_2'
 ; VF_16:         Found an estimated cost of 8 for VF 16 For instruction: %tmp2 = load i16, ptr %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i16, ptr %tmp1, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store i16 0, ptr %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 %tmp2, ptr %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store i16 %tmp3, ptr %tmp1, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i16, ptr %tmp0, align 2
   %tmp3 = load i16, ptr %tmp1, align 2
-  store i16 0, ptr %tmp0, align 2
-  store i16 0, ptr %tmp1, align 2
+  store i16 %tmp2, ptr %tmp0, align 2
+  store i16 %tmp3, ptr %tmp1, align 2
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -99,31 +99,31 @@ entry:
 ; VF_2-LABEL:  Checking a loop in 'i32_factor_2'
 ; VF_2:          Found an estimated cost of 24 for VF 2 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 ; VF_4-LABEL:  Checking a loop in 'i32_factor_2'
 ; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 ; VF_8-LABEL:  Checking a loop in 'i32_factor_2'
 ; VF_8:          Found an estimated cost of 8 for VF 8 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_8-NEXT:     Found an estimated cost of 8 for VF 8 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 8 for VF 8 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 ; VF_16-LABEL: Checking a loop in 'i32_factor_2'
 ; VF_16:         Found an estimated cost of 16 for VF 16 For instruction: %tmp2 = load i32, ptr %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i32, ptr %tmp1, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_16-NEXT:    Found an estimated cost of 16 for VF 16 For instruction: store i32 0, ptr %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 %tmp2, ptr %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 16 for VF 16 For instruction: store i32 %tmp3, ptr %tmp1, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i32, ptr %tmp0, align 4
   %tmp3 = load i32, ptr %tmp1, align 4
-  store i32 0, ptr %tmp0, align 4
-  store i32 0, ptr %tmp1, align 4
+  store i32 %tmp2, ptr %tmp0, align 4
+  store i32 %tmp3, ptr %tmp1, align 4
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -140,31 +140,31 @@ entry:
 ; VF_2-LABEL:  Checking a loop in 'i64_factor_2'
 ; VF_2:          Found an estimated cost of 44 for VF 2 For instruction: %tmp2 = load i64, ptr %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i64, ptr %tmp1, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i64 0, ptr %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 %tmp2, ptr %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 44 for VF 2 For instruction: store i64 %tmp3, ptr %tmp1, align 8
 ; VF_4-LABEL:  Checking a loop in 'i64_factor_2'
 ; VF_4:          Found an estimated cost of 88 for VF 4 For instruction: %tmp2 = load i64, ptr %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i64, ptr %tmp1, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store i64 0, ptr %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 %tmp2, ptr %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 88 for VF 4 For instruction: store i64 %tmp3, ptr %tmp1, align 8
 ; VF_8-LABEL:  Checking a loop in 'i64_factor_2'
 ; VF_8:          Found an estimated cost of 176 for VF 8 For instruction: %tmp2 = load i64, ptr %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i64, ptr %tmp1, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store i64 0, ptr %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 %tmp2, ptr %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 176 for VF 8 For instruction: store i64 %tmp3, ptr %tmp1, align 8
 ; VF_16-LABEL: Checking a loop in 'i64_factor_2'
 ; VF_16:         Found an estimated cost of 352 for VF 16 For instruction: %tmp2 = load i64, ptr %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i64, ptr %tmp1, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store i64 0, ptr %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 %tmp2, ptr %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 352 for VF 16 For instruction: store i64 %tmp3, ptr %tmp1, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i64.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %i64.2, ptr %data, i64 %i, i32 1
   %tmp2 = load i64, ptr %tmp0, align 8
   %tmp3 = load i64, ptr %tmp1, align 8
-  store i64 0, ptr %tmp0, align 8
-  store i64 0, ptr %tmp1, align 8
+  store i64 %tmp2, ptr %tmp0, align 8
+  store i64 %tmp3, ptr %tmp1, align 8
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -181,31 +181,31 @@ entry:
 ; VF_2-LABEL:  Checking a loop in 'f16_factor_2'
 ; VF_2:          Found an estimated cost of 12 for VF 2 For instruction: %tmp2 = load half, ptr %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load half, ptr %tmp1, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store half 0xH0000, ptr %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half %tmp2, ptr %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store half %tmp3, ptr %tmp1, align 2
 ; VF_4-LABEL:  Checking a loop in 'f16_factor_2'
 ; VF_4:          Found an estimated cost of 18 for VF 4 For instruction: %tmp2 = load half, ptr %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load half, ptr %tmp1, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_4-NEXT:     Found an estimated cost of 16 for VF 4 For instruction: store half 0xH0000, ptr %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half %tmp2, ptr %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 18 for VF 4 For instruction: store half %tmp3, ptr %tmp1, align 2
 ; VF_8-LABEL:  Checking a loop in 'f16_factor_2'
 ; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load half, ptr %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load half, ptr %tmp1, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store half 0xH0000, ptr %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half %tmp2, ptr %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store half %tmp3, ptr %tmp1, align 2
 ; VF_16-LABEL: Checking a loop in 'f16_factor_2'
 ; VF_16:         Found an estimated cost of 8 for VF 16 For instruction: %tmp2 = load half, ptr %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load half, ptr %tmp1, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store half 0xH0000, ptr %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half %tmp2, ptr %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store half %tmp3, ptr %tmp1, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f16.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %f16.2, ptr %data, i64 %i, i32 1
   %tmp2 = load half, ptr %tmp0, align 2
   %tmp3 = load half, ptr %tmp1, align 2
-  store half 0.0, ptr %tmp0, align 2
-  store half 0.0, ptr %tmp1, align 2
+  store half %tmp2, ptr %tmp0, align 2
+  store half %tmp3, ptr %tmp1, align 2
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -222,31 +222,31 @@ entry:
 ; VF_2-LABEL:  Checking a loop in 'f32_factor_2'
 ; VF_2:          Found an estimated cost of 10 for VF 2 For instruction: %tmp2 = load float, ptr %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load float, ptr %tmp1, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float %tmp2, ptr %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 10 for VF 2 For instruction: store float %tmp3, ptr %tmp1, align 4
 ; VF_4-LABEL:  Checking a loop in 'f32_factor_2'
 ; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load float, ptr %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load float, ptr %tmp1, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float %tmp2, ptr %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store float %tmp3, ptr %tmp1, align 4
 ; VF_8-LABEL:  Checking a loop in 'f32_factor_2'
 ; VF_8:          Found an estimated cost of 8 for VF 8 For instruction: %tmp2 = load float, ptr %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load float, ptr %tmp1, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_8-NEXT:     Found an estimated cost of 8 for VF 8 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float %tmp2, ptr %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 8 for VF 8 For instruction: store float %tmp3, ptr %tmp1, align 4
 ; VF_16-LABEL: Checking a loop in 'f32_factor_2'
 ; VF_16:         Found an estimated cost of 16 for VF 16 For instruction: %tmp2 = load float, ptr %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load float, ptr %tmp1, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_16-NEXT:    Found an estimated cost of 16 for VF 16 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float %tmp2, ptr %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 16 for VF 16 For instruction: store float %tmp3, ptr %tmp1, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f32.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %f32.2, ptr %data, i64 %i, i32 1
   %tmp2 = load float, ptr %tmp0, align 4
   %tmp3 = load float, ptr %tmp1, align 4
-  store float 0.0, ptr %tmp0, align 4
-  store float 0.0, ptr %tmp1, align 4
+  store float %tmp2, ptr %tmp0, align 4
+  store float %tmp3, ptr %tmp1, align 4
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -263,31 +263,31 @@ entry:
 ; VF_2-LABEL:  Checking a loop in 'f64_factor_2'
 ; VF_2:          Found an estimated cost of 12 for VF 2 For instruction: %tmp2 = load double, ptr %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load double, ptr %tmp1, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double %tmp2, ptr %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store double %tmp3, ptr %tmp1, align 8
 ; VF_4-LABEL:  Checking a loop in 'f64_factor_2'
 ; VF_4:          Found an estimated cost of 24 for VF 4 For instruction: %tmp2 = load double, ptr %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load double, ptr %tmp1, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_4-NEXT:     Found an estimated cost of 16 for VF 4 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double %tmp2, ptr %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store double %tmp3, ptr %tmp1, align 8
 ; VF_8-LABEL:  Checking a loop in 'f64_factor_2'
 ; VF_8:          Found an estimated cost of 48 for VF 8 For instruction: %tmp2 = load double, ptr %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load double, ptr %tmp1, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_8-NEXT:     Found an estimated cost of 32 for VF 8 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double %tmp2, ptr %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store double %tmp3, ptr %tmp1, align 8
 ; VF_16-LABEL: Checking a loop in 'f64_factor_2'
 ; VF_16:         Found an estimated cost of 96 for VF 16 For instruction: %tmp2 = load double, ptr %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load double, ptr %tmp1, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_16-NEXT:    Found an estimated cost of 64 for VF 16 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double %tmp2, ptr %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store double %tmp3, ptr %tmp1, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f64.2, ptr %data, i64 %i, i32 0
   %tmp1 = getelementptr inbounds %f64.2, ptr %data, i64 %i, i32 1
   %tmp2 = load double, ptr %tmp0, align 8
   %tmp3 = load double, ptr %tmp1, align 8
-  store double 0.0, ptr %tmp0, align 8
-  store double 0.0, ptr %tmp1, align 8
+  store double %tmp2, ptr %tmp0, align 8
+  store double %tmp3, ptr %tmp1, align 8
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -309,30 +309,30 @@ entry:
 ; VF_2:          Found an estimated cost of 36 for VF 2 For instruction: %tmp3 = load i8, ptr %tmp0, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i8, ptr %tmp1, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i8, ptr %tmp2, align 1
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, ptr %tmp1, align 1
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i8 0, ptr %tmp2, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 %tmp3, ptr %tmp0, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 %tmp4, ptr %tmp1, align 1
+; VF_2-NEXT:     Found an estimated cost of 36 for VF 2 For instruction: store i8 %tmp5, ptr %tmp2, align 1
 ; VF_4-LABEL:  Checking a loop in 'i8_factor_3'
 ; VF_4:          Found an estimated cost of 72 for VF 4 For instruction: %tmp3 = load i8, ptr %tmp0, align 1
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i8, ptr %tmp1, align 1
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i8, ptr %tmp2, align 1
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 0, ptr %tmp1, align 1
-; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store i8 0, ptr %tmp2, align 1
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 %tmp3,  ptr %tmp0, align 1
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 %tmp4, ptr %tmp1, align 1
+; VF_4-NEXT:     Found an estimated cost of 72 for VF 4 For instruction: store i8 %tmp5, ptr %tmp2, align 1
 ; VF_8-LABEL:  Checking a loop in 'i8_factor_3'
 ; VF_8:          Found an estimated cost of 144 for VF 8 For instruction: %tmp3 = load i8, ptr %tmp0, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i8, ptr %tmp1, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i8, ptr %tmp2, align 1
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, ptr %tmp1, align 1
-; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store i8 0, ptr %tmp2, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 %tmp3, ptr %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 %tmp4, ptr %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 144 for VF 8 For instruction: store i8 %tmp5, ptr %tmp2, align 1
 ; VF_16-LABEL: Checking a loop in 'i8_factor_3'
 ; VF_16:         Found an estimated cost of 288 for VF 16 For instruction: %tmp3 = load i8, ptr %tmp0, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i8, ptr %tmp1, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i8, ptr %tmp2, align 1
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, ptr %tmp1, align 1
-; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store i8 0, ptr %tmp2, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 %tmp3, ptr %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 %tmp4, ptr %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 288 for VF 16 For instruction: store i8 %tmp5, ptr %tmp2, align 1
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 0
@@ -341,9 +341,9 @@ for.body:
   %tmp3 = load i8, ptr %tmp0, align 1
   %tmp4 = load i8, ptr %tmp1, align 1
   %tmp5 = load i8, ptr %tmp2, align 1
-  store i8 0, ptr %tmp0, align 1
-  store i8 0, ptr %tmp1, align 1
-  store i8 0, ptr %tmp2, align 1
+  store i8 %tmp3, ptr %tmp0, align 1
+  store i8 %tmp4, ptr %tmp1, align 1
+  store i8 %tmp5, ptr %tmp2, align 1
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -361,30 +361,30 @@ entry:
 ; VF_2:          Found an estimated cost of 36 for VF 2 For instruction: %tmp3 = load i16, ptr %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i16, ptr %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i16, ptr %tmp2, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, ptr %tmp1, align 2
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i16 0, ptr %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 %tmp3, ptr %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 %tmp4, ptr %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 36 for VF 2 For instruction: store i16 %tmp5, ptr %tmp2, align 2
 ; VF_4-LABEL:  Checking a loop in 'i16_factor_3'
 ; VF_4:          Found an estimated cost of 72 for VF 4 For instruction: %tmp3 = load i16, ptr %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i16, ptr %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i16, ptr %tmp2, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, ptr %tmp1, align 2
-; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store i16 0, ptr %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 %tmp3, ptr %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 %tmp4, ptr %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 72 for VF 4 For instruction: store i16 %tmp5, ptr %tmp2, align 2
 ; VF_8-LABEL:  Checking a loop in 'i16_factor_3'
 ; VF_8:          Found an estimated cost of 144 for VF 8 For instruction: %tmp3 = load i16, ptr %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i16, ptr %tmp1, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i16, ptr %tmp2, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, ptr %tmp1, align 2
-; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store i16 0, ptr %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 %tmp3, ptr %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 %tmp4, ptr %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 144 for VF 8 For instruction: store i16 %tmp5, ptr %tmp2, align 2
 ; VF_16-LABEL: Checking a loop in 'i16_factor_3'
 ; VF_16:         Found an estimated cost of 288 for VF 16 For instruction: %tmp3 = load i16, ptr %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i16, ptr %tmp1, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i16, ptr %tmp2, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, ptr %tmp1, align 2
-; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store i16 0, ptr %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 %tmp3, ptr %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 %tmp4, ptr %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 288 for VF 16 For instruction: store i16 %tmp5, ptr %tmp2, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 0
@@ -393,9 +393,9 @@ for.body:
   %tmp3 = load i16, ptr %tmp0, align 2
   %tmp4 = load i16, ptr %tmp1, align 2
   %tmp5 = load i16, ptr %tmp2, align 2
-  store i16 0, ptr %tmp0, align 2
-  store i16 0, ptr %tmp1, align 2
-  store i16 0, ptr %tmp2, align 2
+  store i16 %tmp3, ptr %tmp0, align 2
+  store i16 %tmp4, ptr %tmp1, align 2
+  store i16 %tmp5, ptr %tmp2, align 2
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -413,30 +413,30 @@ entry:
 ; VF_2:          Found an estimated cost of 36 for VF 2 For instruction: %tmp3 = load i32, ptr %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i32, ptr %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i32, ptr %tmp2, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, ptr %tmp1, align 4
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i32 0, ptr %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 %tmp3, ptr %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 %tmp4, ptr %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 36 for VF 2 For instruction: store i32 %tmp5, ptr %tmp2, align 4
 ; VF_4-LABEL:  Checking a loop in 'i32_factor_3'
 ; VF_4:          Found an estimated cost of 24 for VF 4 For instruction: %tmp3 = load i32, ptr %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i32, ptr %tmp1, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i32, ptr %tmp2, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, ptr %tmp1, align 4
-; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store i32 0, ptr %tmp2, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 %tmp3, ptr %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 %tmp4, ptr %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store i32 %tmp5, ptr %tmp2, align 4
 ; VF_8-LABEL:  Checking a loop in 'i32_factor_3'
 ; VF_8:          Found an estimated cost of 144 for VF 8 For instruction: %tmp3 = load i32, ptr %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i32, ptr %tmp1, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i32, ptr %tmp2, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, ptr %tmp1, align 4
-; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store i32 0, ptr %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 %tmp3, ptr %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 %tmp4, ptr %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 144 for VF 8 For instruction: store i32 %tmp5, ptr %tmp2, align 4
 ; VF_16-LABEL: Checking a loop in 'i32_factor_3'
 ; VF_16:         Found an estimated cost of 288 for VF 16 For instruction: %tmp3 = load i32, ptr %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i32, ptr %tmp1, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i32, ptr %tmp2, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, ptr %tmp1, align 4
-; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store i32 0, ptr %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 %tmp3, ptr %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 %tmp4, ptr %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 288 for VF 16 For instruction: store i32 %tmp5, ptr %tmp2, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 0
@@ -445,9 +445,9 @@ for.body:
   %tmp3 = load i32, ptr %tmp0, align 4
   %tmp4 = load i32, ptr %tmp1, align 4
   %tmp5 = load i32, ptr %tmp2, align 4
-  store i32 0, ptr %tmp0, align 4
-  store i32 0, ptr %tmp1, align 4
-  store i32 0, ptr %tmp2, align 4
+  store i32 %tmp3, ptr %tmp0, align 4
+  store i32 %tmp4, ptr %tmp1, align 4
+  store i32 %tmp5, ptr %tmp2, align 4
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -465,30 +465,30 @@ entry:
 ; VF_2:          Found an estimated cost of 66 for VF 2 For instruction: %tmp3 = load i64, ptr %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i64, ptr %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i64, ptr %tmp2, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, ptr %tmp1, align 8
-; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store i64 0, ptr %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 %tmp3, ptr %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 %tmp4, ptr %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 66 for VF 2 For instruction: store i64 %tmp5, ptr %tmp2, align 8
 ; VF_4-LABEL:  Checking a loop in 'i64_factor_3'
 ; VF_4:          Found an estimated cost of 132 for VF 4 For instruction: %tmp3 = load i64, ptr %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i64, ptr %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i64, ptr %tmp2, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, ptr %tmp1, align 8
-; VF_4-NEXT:     Found an estimated cost of 36 for VF 4 For instruction: store i64 0, ptr %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 %tmp3, ptr %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 %tmp4, ptr %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 132 for VF 4 For instruction: store i64 %tmp5, ptr %tmp2, align 8
 ; VF_8-LABEL:  Checking a loop in 'i64_factor_3'
 ; VF_8:          Found an estimated cost of 264 for VF 8 For instruction: %tmp3 = load i64, ptr %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i64, ptr %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i64, ptr %tmp2, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, ptr %tmp1, align 8
-; VF_8-NEXT:     Found an estimated cost of 72 for VF 8 For instruction: store i64 0, ptr %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 %tmp3, ptr %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 %tmp4, ptr %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 264 for VF 8 For instruction: store i64 %tmp5, ptr %tmp2, align 8
 ; VF_16-LABEL: Checking a loop in 'i64_factor_3'
 ; VF_16:         Found an estimated cost of 528 for VF 16 For instruction: %tmp3 = load i64, ptr %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i64, ptr %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i64, ptr %tmp2, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, ptr %tmp1, align 8
-; VF_16-NEXT:    Found an estimated cost of 144 for VF 16 For instruction: store i64 0, ptr %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 %tmp3, ptr %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 %tmp4, ptr %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 528 for VF 16 For instruction: store i64 %tmp5, ptr %tmp2, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 0
@@ -497,9 +497,9 @@ for.body:
   %tmp3 = load i64, ptr %tmp0, align 8
   %tmp4 = load i64, ptr %tmp1, align 8
   %tmp5 = load i64, ptr %tmp2, align 8
-  store i64 0, ptr %tmp0, align 8
-  store i64 0, ptr %tmp1, align 8
-  store i64 0, ptr %tmp2, align 8
+  store i64 %tmp3, ptr %tmp0, align 8
+  store i64 %tmp4, ptr %tmp1, align 8
+  store i64 %tmp5, ptr %tmp2, align 8
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -517,30 +517,30 @@ entry:
 ; VF_2:          Found an estimated cost of 18 for VF 2 For instruction: %tmp3 = load half, ptr %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load half, ptr %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load half, ptr %tmp2, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, ptr %tmp1, align 2
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store half 0xH0000, ptr %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half %tmp3, ptr %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half %tmp4, ptr %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store half %tmp5, ptr %tmp2, align 2
 ; VF_4-LABEL:  Checking a loop in 'f16_factor_3'
 ; VF_4:          Found an estimated cost of 28 for VF 4 For instruction: %tmp3 = load half, ptr %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load half, ptr %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load half, ptr %tmp2, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, ptr %tmp1, align 2
-; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store half 0xH0000, ptr %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half %tmp3, ptr %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half %tmp4, ptr %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 28 for VF 4 For instruction: store half %tmp5, ptr %tmp2, align 2
 ; VF_8-LABEL:  Checking a loop in 'f16_factor_3'
 ; VF_8:          Found an estimated cost of 56 for VF 8 For instruction: %tmp3 = load half, ptr %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load half, ptr %tmp1, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load half, ptr %tmp2, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, ptr %tmp1, align 2
-; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store half 0xH0000, ptr %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half %tmp3, ptr %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half %tmp4, ptr %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 56 for VF 8 For instruction: store half %tmp5, ptr %tmp2, align 2
 ; VF_16-LABEL: Checking a loop in 'f16_factor_3'
 ; VF_16:         Found an estimated cost of 112 for VF 16 For instruction: %tmp3 = load half, ptr %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load half, ptr %tmp1, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load half, ptr %tmp2, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, ptr %tmp1, align 2
-; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store half 0xH0000, ptr %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half %tmp3, ptr %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half %tmp4, ptr %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 112 for VF 16 For instruction: store half %tmp5, ptr %tmp2, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f16.3, ptr %data, i64 %i, i32 0
@@ -549,9 +549,9 @@ for.body:
   %tmp3 = load half, ptr %tmp0, align 2
   %tmp4 = load half, ptr %tmp1, align 2
   %tmp5 = load half, ptr %tmp2, align 2
-  store half 0.0, ptr %tmp0, align 2
-  store half 0.0, ptr %tmp1, align 2
-  store half 0.0, ptr %tmp2, align 2
+  store half %tmp3, ptr %tmp0, align 2
+  store half %tmp4, ptr %tmp1, align 2
+  store half %tmp5, ptr %tmp2, align 2
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -569,30 +569,30 @@ entry:
 ; VF_2:          Found an estimated cost of 16 for VF 2 For instruction: %tmp3 = load float, ptr %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load float, ptr %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load float, ptr %tmp2, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store float 0.000000e+00, ptr %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float %tmp3, ptr %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float %tmp4, ptr %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store float %tmp5, ptr %tmp2, align 4
 ; VF_4-LABEL:  Checking a loop in 'f32_factor_3'
 ; VF_4:          Found an estimated cost of 24 for VF 4 For instruction: %tmp3 = load float, ptr %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load float, ptr %tmp1, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load float, ptr %tmp2, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
-; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store float 0.000000e+00, ptr %tmp2, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float %tmp3, ptr %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float %tmp4, ptr %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store float %tmp5, ptr %tmp2, align 4
 ; VF_8-LABEL:  Checking a loop in 'f32_factor_3'
 ; VF_8:          Found an estimated cost of 64 for VF 8 For instruction: %tmp3 = load float, ptr %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load float, ptr %tmp1, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load float, ptr %tmp2, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
-; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store float 0.000000e+00, ptr %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float %tmp3, ptr %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float %tmp4, ptr %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store float %tmp5, ptr %tmp2, align 4
 ; VF_16-LABEL: Checking a loop in 'f32_factor_3'
 ; VF_16:         Found an estimated cost of 128 for VF 16 For instruction: %tmp3 = load float, ptr %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load float, ptr %tmp1, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load float, ptr %tmp2, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
-; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store float 0.000000e+00, ptr %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float %tmp3, ptr %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float %tmp4, ptr %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store float %tmp5, ptr %tmp2, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f32.3, ptr %data, i64 %i, i32 0
@@ -601,9 +601,9 @@ for.body:
   %tmp3 = load float, ptr %tmp0, align 4
   %tmp4 = load float, ptr %tmp1, align 4
   %tmp5 = load float, ptr %tmp2, align 4
-  store float 0.0, ptr %tmp0, align 4
-  store float 0.0, ptr %tmp1, align 4
-  store float 0.0, ptr %tmp2, align 4
+  store float %tmp3, ptr %tmp0, align 4
+  store float %tmp4, ptr %tmp1, align 4
+  store float %tmp5, ptr %tmp2, align 4
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -621,30 +621,30 @@ entry:
 ; VF_2:          Found an estimated cost of 18 for VF 2 For instruction: %tmp3 = load double, ptr %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load double, ptr %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load double, ptr %tmp2, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store double 0.000000e+00, ptr %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double %tmp3, ptr %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double %tmp4, ptr %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store double %tmp5, ptr %tmp2, align 8
 ; VF_4-LABEL:  Checking a loop in 'f64_factor_3'
 ; VF_4:          Found an estimated cost of 36 for VF 4 For instruction: %tmp3 = load double, ptr %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load double, ptr %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load double, ptr %tmp2, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
-; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store double 0.000000e+00, ptr %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double %tmp3, ptr %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double %tmp4, ptr %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 36 for VF 4 For instruction: store double %tmp5, ptr %tmp2, align 8
 ; VF_8-LABEL:  Checking a loop in 'f64_factor_3'
 ; VF_8:          Found an estimated cost of 72 for VF 8 For instruction: %tmp3 = load double, ptr %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load double, ptr %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load double, ptr %tmp2, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
-; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store double 0.000000e+00, ptr %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double %tmp3, ptr %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double %tmp4, ptr %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 72 for VF 8 For instruction: store double %tmp5, ptr %tmp2, align 8
 ; VF_16-LABEL: Checking a loop in 'f64_factor_3'
 ; VF_16:         Found an estimated cost of 144 for VF 16 For instruction: %tmp3 = load double, ptr %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load double, ptr %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load double, ptr %tmp2, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
-; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store double 0.000000e+00, ptr %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double %tmp3, ptr %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double %tmp4, ptr %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 144 for VF 16 For instruction: store double %tmp5, ptr %tmp2, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f64.3, ptr %data, i64 %i, i32 0
@@ -653,9 +653,9 @@ for.body:
   %tmp3 = load double, ptr %tmp0, align 8
   %tmp4 = load double, ptr %tmp1, align 8
   %tmp5 = load double, ptr %tmp2, align 8
-  store double 0.0, ptr %tmp0, align 8
-  store double 0.0, ptr %tmp1, align 8
-  store double 0.0, ptr %tmp2, align 8
+  store double %tmp3, ptr %tmp0, align 8
+  store double %tmp4, ptr %tmp1, align 8
+  store double %tmp5, ptr %tmp2, align 8
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -677,37 +677,37 @@ entry:
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i8, ptr %tmp1, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i8, ptr %tmp2, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i8, ptr %tmp3, align 1
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, ptr %tmp1, align 1
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, ptr %tmp2, align 1
-; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store i8 0, ptr %tmp3, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 %tmp4, ptr %tmp0, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 %tmp5, ptr %tmp1, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 %tmp6, ptr %tmp2, align 1
+; VF_2-NEXT:     Found an estimated cost of 48 for VF 2 For instruction: store i8 %tmp7, ptr %tmp3, align 1
 ; VF_4-LABEL: Checking a loop in 'i8_factor_4'
 ; VF_4:         Found an estimated cost of 96 for VF 4 For instruction: %tmp4 = load i8, ptr %tmp0, align 1
 ; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i8, ptr %tmp1, align 1
 ; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i8, ptr %tmp2, align 1
 ; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i8, ptr %tmp3, align 1
-; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 0, ptr %tmp1, align 1
-; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 0, ptr %tmp2, align 1
-; VF_4-NEXT:    Found an estimated cost of 32 for VF 4 For instruction: store i8 0, ptr %tmp3, align 1
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 %tmp4, ptr %tmp0, align 1
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 %tmp5, ptr %tmp1, align 1
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 %tmp6, ptr %tmp2, align 1
+; VF_4-NEXT:    Found an estimated cost of 96 for VF 4 For instruction: store i8 %tmp7, ptr %tmp3, align 1
 ; VF_8-LABEL:  Checking a loop in 'i8_factor_4'
 ; VF_8:          Found an estimated cost of 192 for VF 8 For instruction: %tmp4 = load i8, ptr %tmp0, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i8, ptr %tmp1, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i8, ptr %tmp2, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i8, ptr %tmp3, align 1
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, ptr %tmp1, align 1
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, ptr %tmp2, align 1
-; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store i8 0, ptr %tmp3, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 %tmp4, ptr %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 %tmp5, ptr %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 %tmp6, ptr %tmp2, align 1
+; VF_8-NEXT:     Found an estimated cost of 192 for VF 8 For instruction: store i8 %tmp7, ptr %tmp3, align 1
 ; VF_16-LABEL: Checking a loop in 'i8_factor_4'
 ; VF_16:         Found an estimated cost of 384 for VF 16 For instruction: %tmp4 = load i8, ptr %tmp0, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i8, ptr %tmp1, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i8, ptr %tmp2, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i8, ptr %tmp3, align 1
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, ptr %tmp0, align 1
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, ptr %tmp1, align 1
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, ptr %tmp2, align 1
-; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store i8 0, ptr %tmp3, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 %tmp4, ptr %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 %tmp5, ptr %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 %tmp6, ptr %tmp2, align 1
+; VF_16-NEXT:    Found an estimated cost of 384 for VF 16 For instruction: store i8 %tmp7, ptr %tmp3, align 1
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i8.4, ptr %data, i64 %i, i32 0
@@ -718,10 +718,10 @@ for.body:
   %tmp5 = load i8, ptr %tmp1, align 1
   %tmp6 = load i8, ptr %tmp2, align 1
   %tmp7 = load i8, ptr %tmp3, align 1
-  store i8 0, ptr %tmp0, align 1
-  store i8 0, ptr %tmp1, align 1
-  store i8 0, ptr %tmp2, align 1
-  store i8 0, ptr %tmp3, align 1
+  store i8 %tmp4, ptr %tmp0, align 1
+  store i8 %tmp5, ptr %tmp1, align 1
+  store i8 %tmp6, ptr %tmp2, align 1
+  store i8 %tmp7, ptr %tmp3, align 1
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -740,37 +740,37 @@ entry:
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i16, ptr %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i16, ptr %tmp2, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i16, ptr %tmp3, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, ptr %tmp1, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, ptr %tmp2, align 2
-; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store i16 0, ptr %tmp3, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 %tmp4, ptr %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 %tmp5, ptr %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 %tmp6, ptr %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 48 for VF 2 For instruction: store i16 %tmp7, ptr %tmp3, align 2
 ; VF_4-LABEL:  Checking a loop in 'i16_factor_4'
 ; VF_4:          Found an estimated cost of 96 for VF 4 For instruction: %tmp4 = load i16, ptr %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i16, ptr %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i16, ptr %tmp2, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i16, ptr %tmp3, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, ptr %tmp1, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, ptr %tmp2, align 2
-; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store i16 0, ptr %tmp3, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 %tmp4, ptr %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 %tmp5, ptr %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 %tmp6, ptr %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 96 for VF 4 For instruction: store i16 %tmp7, ptr %tmp3, align 2
 ; VF_8-LABEL:  Checking a loop in 'i16_factor_4'
 ; VF_8:          Found an estimated cost of 192 for VF 8 For instruction: %tmp4 = load i16, ptr %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i16, ptr %tmp1, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i16, ptr %tmp2, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i16, ptr %tmp3, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, ptr %tmp1, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, ptr %tmp2, align 2
-; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store i16 0, ptr %tmp3, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 %tmp4, ptr %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 %tmp5, ptr %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 %tmp6, ptr %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 192 for VF 8 For instruction: store i16 %tmp7, ptr %tmp3, align 2
 ; VF_16-LABEL: Checking a loop in 'i16_factor_4'
 ; VF_16:         Found an estimated cost of 384 for VF 16 For instruction: %tmp4 = load i16, ptr %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i16, ptr %tmp1, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i16, ptr %tmp2, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i16, ptr %tmp3, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, ptr %tmp0, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, ptr %tmp1, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, ptr %tmp2, align 2
-; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store i16 0, ptr %tmp3, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 %tmp4, ptr %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 %tmp5, ptr %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 %tmp6, ptr %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 384 for VF 16 For instruction: store i16 %tmp7, ptr %tmp3, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i16.4, ptr %data, i64 %i, i32 0
@@ -781,10 +781,10 @@ for.body:
   %tmp5 = load i16, ptr %tmp1, align 2
   %tmp6 = load i16, ptr %tmp2, align 2
   %tmp7 = load i16, ptr %tmp3, align 2
-  store i16 0, ptr %tmp0, align 2
-  store i16 0, ptr %tmp1, align 2
-  store i16 0, ptr %tmp2, align 2
-  store i16 0, ptr %tmp3, align 2
+  store i16 %tmp4, ptr %tmp0, align 2
+  store i16 %tmp5, ptr %tmp1, align 2
+  store i16 %tmp6, ptr %tmp2, align 2
+  store i16 %tmp7, ptr %tmp3, align 2
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -803,37 +803,37 @@ entry:
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i32, ptr %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i32, ptr %tmp2, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i32, ptr %tmp3, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, ptr %tmp1, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, ptr %tmp2, align 4
-; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store i32 0, ptr %tmp3, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 %tmp4, ptr %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 %tmp5, ptr %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 %tmp6, ptr %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 48 for VF 2 For instruction: store i32 %tmp7, ptr %tmp3, align 4
 ; VF_4-LABEL:  Checking a loop in 'i32_factor_4'
 ; VF_4:          Found an estimated cost of 32 for VF 4 For instruction: %tmp4 = load i32, ptr %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i32, ptr %tmp1, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i32, ptr %tmp2, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i32, ptr %tmp3, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, ptr %tmp1, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, ptr %tmp2, align 4
-; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store i32 0, ptr %tmp3, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 %tmp4, ptr %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 %tmp5, ptr %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 %tmp6, ptr %tmp2, align 4
+; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store i32 %tmp7, ptr %tmp3, align 4
 ; VF_8-LABEL:  Checking a loop in 'i32_factor_4'
 ; VF_8:          Found an estimated cost of 192 for VF 8 For instruction: %tmp4 = load i32, ptr %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i32, ptr %tmp1, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i32, ptr %tmp2, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i32, ptr %tmp3, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, ptr %tmp1, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, ptr %tmp2, align 4
-; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store i32 0, ptr %tmp3, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 %tmp4, ptr %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 %tmp5, ptr %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 %tmp6, ptr %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 192 for VF 8 For instruction: store i32 %tmp7, ptr %tmp3, align 4
 ; VF_16-LABEL: Checking a loop in 'i32_factor_4'
 ; VF_16:         Found an estimated cost of 384 for VF 16 For instruction: %tmp4 = load i32, ptr %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i32, ptr %tmp1, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i32, ptr %tmp2, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i32, ptr %tmp3, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, ptr %tmp0, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, ptr %tmp1, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, ptr %tmp2, align 4
-; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store i32 0, ptr %tmp3, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 %tmp4, ptr %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 %tmp5, ptr %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 %tmp6, ptr %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 384 for VF 16 For instruction: store i32 %tmp7, ptr %tmp3, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i32.4, ptr %data, i64 %i, i32 0
@@ -844,10 +844,10 @@ for.body:
   %tmp5 = load i32, ptr %tmp1, align 4
   %tmp6 = load i32, ptr %tmp2, align 4
   %tmp7 = load i32, ptr %tmp3, align 4
-  store i32 0, ptr %tmp0, align 4
-  store i32 0, ptr %tmp1, align 4
-  store i32 0, ptr %tmp2, align 4
-  store i32 0, ptr %tmp3, align 4
+  store i32 %tmp4, ptr %tmp0, align 4
+  store i32 %tmp5, ptr %tmp1, align 4
+  store i32 %tmp6, ptr %tmp2, align 4
+  store i32 %tmp7, ptr %tmp3, align 4
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -866,37 +866,37 @@ entry:
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i64, ptr %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i64, ptr %tmp2, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i64, ptr %tmp3, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, ptr %tmp1, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, ptr %tmp2, align 8
-; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i64 0, ptr %tmp3, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 %tmp4, ptr %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 %tmp5, ptr %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 %tmp6, ptr %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 88 for VF 2 For instruction: store i64 %tmp7, ptr %tmp3, align 8
 ; VF_4-LABEL:  Checking a loop in 'i64_factor_4'
 ; VF_4:          Found an estimated cost of 176 for VF 4 For instruction: %tmp4 = load i64, ptr %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i64, ptr %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i64, ptr %tmp2, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i64, ptr %tmp3, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, ptr %tmp1, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, ptr %tmp2, align 8
-; VF_4-NEXT:     Found an estimated cost of 48 for VF 4 For instruction: store i64 0, ptr %tmp3, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 %tmp4, ptr %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 %tmp5, ptr %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 %tmp6, ptr %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 176 for VF 4 For instruction: store i64 %tmp7, ptr %tmp3, align 8
 ; VF_8-LABEL:  Checking a loop in 'i64_factor_4'
 ; VF_8:          Found an estimated cost of 352 for VF 8 For instruction: %tmp4 = load i64, ptr %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i64, ptr %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i64, ptr %tmp2, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i64, ptr %tmp3, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, ptr %tmp1, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, ptr %tmp2, align 8
-; VF_8-NEXT:     Found an estimated cost of 96 for VF 8 For instruction: store i64 0, ptr %tmp3, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 %tmp4, ptr %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 %tmp5, ptr %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 %tmp6, ptr %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 352 for VF 8 For instruction: store i64 %tmp7, ptr %tmp3, align 8
 ; VF_16-LABEL: Checking a loop in 'i64_factor_4'
 ; VF_16:         Found an estimated cost of 704 for VF 16 For instruction: %tmp4 = load i64, ptr %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i64, ptr %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i64, ptr %tmp2, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i64, ptr %tmp3, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, ptr %tmp0, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, ptr %tmp1, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, ptr %tmp2, align 8
-; VF_16-NEXT:    Found an estimated cost of 192 for VF 16 For instruction: store i64 0, ptr %tmp3, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 %tmp4, ptr %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 %tmp5, ptr %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 %tmp6, ptr %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 704 for VF 16 For instruction: store i64 %tmp7, ptr %tmp3, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i64.4, ptr %data, i64 %i, i32 0
@@ -907,10 +907,10 @@ for.body:
   %tmp5 = load i64, ptr %tmp1, align 8
   %tmp6 = load i64, ptr %tmp2, align 8
   %tmp7 = load i64, ptr %tmp3, align 8
-  store i64 0, ptr %tmp0, align 8
-  store i64 0, ptr %tmp1, align 8
-  store i64 0, ptr %tmp2, align 8
-  store i64 0, ptr %tmp3, align 8
+  store i64 %tmp4, ptr %tmp0, align 8
+  store i64 %tmp5, ptr %tmp1, align 8
+  store i64 %tmp6, ptr %tmp2, align 8
+  store i64 %tmp7, ptr %tmp3, align 8
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -929,37 +929,37 @@ entry:
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load half, ptr %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load half, ptr %tmp2, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load half, ptr %tmp3, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, ptr %tmp1, align 2
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, ptr %tmp2, align 2
-; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store half 0xH0000, ptr %tmp3, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half %tmp4, ptr %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half %tmp5, ptr %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half %tmp6, ptr %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store half %tmp7, ptr %tmp3, align 2
 ; VF_4-LABEL:  Checking a loop in 'f16_factor_4'
 ; VF_4:          Found an estimated cost of 36 for VF 4 For instruction: %tmp4 = load half, ptr %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load half, ptr %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load half, ptr %tmp2, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load half, ptr %tmp3, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, ptr %tmp1, align 2
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, ptr %tmp2, align 2
-; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store half 0xH0000, ptr %tmp3, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half %tmp4, ptr %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half %tmp5, ptr %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half %tmp6, ptr %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 36 for VF 4 For instruction: store half %tmp7, ptr %tmp3, align 2
 ; VF_8-LABEL:  Checking a loop in 'f16_factor_4'
 ; VF_8:          Found an estimated cost of 72 for VF 8 For instruction: %tmp4 = load half, ptr %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load half, ptr %tmp1, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load half, ptr %tmp2, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load half, ptr %tmp3, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, ptr %tmp1, align 2
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, ptr %tmp2, align 2
-; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store half 0xH0000, ptr %tmp3, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half %tmp4, ptr %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half %tmp5, ptr %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half %tmp6, ptr %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 72 for VF 8 For instruction: store half %tmp7, ptr %tmp3, align 2
 ; VF_16-LABEL: Checking a loop in 'f16_factor_4'
 ; VF_16:         Found an estimated cost of 144 for VF 16 For instruction: %tmp4 = load half, ptr %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load half, ptr %tmp1, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load half, ptr %tmp2, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load half, ptr %tmp3, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, ptr %tmp0, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, ptr %tmp1, align 2
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, ptr %tmp2, align 2
-; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store half 0xH0000, ptr %tmp3, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half %tmp4, ptr %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half %tmp5, ptr %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half %tmp6, ptr %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 144 for VF 16 For instruction: store half %tmp7, ptr %tmp3, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f16.4, ptr %data, i64 %i, i32 0
@@ -970,10 +970,10 @@ for.body:
   %tmp5 = load half, ptr %tmp1, align 2
   %tmp6 = load half, ptr %tmp2, align 2
   %tmp7 = load half, ptr %tmp3, align 2
-  store half 0.0, ptr %tmp0, align 2
-  store half 0.0, ptr %tmp1, align 2
-  store half 0.0, ptr %tmp2, align 2
-  store half 0.0, ptr %tmp3, align 2
+  store half %tmp4, ptr %tmp0, align 2
+  store half %tmp5, ptr %tmp1, align 2
+  store half %tmp6, ptr %tmp2, align 2
+  store half %tmp7, ptr %tmp3, align 2
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -992,37 +992,37 @@ entry:
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load float, ptr %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load float, ptr %tmp2, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load float, ptr %tmp3, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, ptr %tmp2, align 4
-; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store float 0.000000e+00, ptr %tmp3, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float %tmp4, ptr %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float %tmp5, ptr %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float %tmp6, ptr %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 20 for VF 2 For instruction: store float %tmp7, ptr %tmp3, align 4
 ; VF_4-LABEL:  Checking a loop in 'f32_factor_4'
 ; VF_4:          Found an estimated cost of 32 for VF 4 For instruction: %tmp4 = load float, ptr %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load float, ptr %tmp1, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load float, ptr %tmp2, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load float, ptr %tmp3, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, ptr %tmp2, align 4
-; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store float 0.000000e+00, ptr %tmp3, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float %tmp4, ptr %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float %tmp5, ptr %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float %tmp6, ptr %tmp2, align 4
+; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store float %tmp7, ptr %tmp3, align 4
 ; VF_8-LABEL:  Checking a loop in 'f32_factor_4'
 ; VF_8:          Found an estimated cost of 80 for VF 8 For instruction: %tmp4 = load float, ptr %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load float, ptr %tmp1, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load float, ptr %tmp2, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load float, ptr %tmp3, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, ptr %tmp2, align 4
-; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store float 0.000000e+00, ptr %tmp3, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float %tmp4, ptr %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float %tmp5, ptr %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float %tmp6, ptr %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 80 for VF 8 For instruction: store float %tmp7, ptr %tmp3, align 4
 ; VF_16-LABEL: Checking a loop in 'f32_factor_4'
 ; VF_16:         Found an estimated cost of 160 for VF 16 For instruction: %tmp4 = load float, ptr %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load float, ptr %tmp1, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load float, ptr %tmp2, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load float, ptr %tmp3, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, ptr %tmp0, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, ptr %tmp1, align 4
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, ptr %tmp2, align 4
-; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store float 0.000000e+00, ptr %tmp3, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float %tmp4, ptr %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float %tmp5, ptr %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float %tmp6, ptr %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 160 for VF 16 For instruction: store float %tmp7, ptr %tmp3, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f32.4, ptr %data, i64 %i, i32 0
@@ -1033,10 +1033,10 @@ for.body:
   %tmp5 = load float, ptr %tmp1, align 4
   %tmp6 = load float, ptr %tmp2, align 4
   %tmp7 = load float, ptr %tmp3, align 4
-  store float 0.0, ptr %tmp0, align 4
-  store float 0.0, ptr %tmp1, align 4
-  store float 0.0, ptr %tmp2, align 4
-  store float 0.0, ptr %tmp3, align 4
+  store float %tmp4, ptr %tmp0, align 4
+  store float %tmp5, ptr %tmp1, align 4
+  store float %tmp6, ptr %tmp2, align 4
+  store float %tmp7, ptr %tmp3, align 4
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
@@ -1055,37 +1055,37 @@ entry:
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load double, ptr %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load double, ptr %tmp2, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load double, ptr %tmp3, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
-; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, ptr %tmp2, align 8
-; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store double 0.000000e+00, ptr %tmp3, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double %tmp4, ptr %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double %tmp5, ptr %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double %tmp6, ptr %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store double %tmp7, ptr %tmp3, align 8
 ; VF_4-LABEL:  Checking a loop in 'f64_factor_4'
 ; VF_4:          Found an estimated cost of 48 for VF 4 For instruction: %tmp4 = load double, ptr %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load double, ptr %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load double, ptr %tmp2, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load double, ptr %tmp3, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
-; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, ptr %tmp2, align 8
-; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store double 0.000000e+00, ptr %tmp3, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double %tmp4, ptr %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double %tmp5, ptr %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double %tmp6, ptr %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 48 for VF 4 For instruction: store double %tmp7, ptr %tmp3, align 8
 ; VF_8-LABEL:  Checking a loop in 'f64_factor_4'
 ; VF_8:          Found an estimated cost of 96 for VF 8 For instruction: %tmp4 = load double, ptr %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load double, ptr %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load double, ptr %tmp2, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load double, ptr %tmp3, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
-; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, ptr %tmp2, align 8
-; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store double 0.000000e+00, ptr %tmp3, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double %tmp4, ptr %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double %tmp5, ptr %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double %tmp6, ptr %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 96 for VF 8 For instruction: store double %tmp7, ptr %tmp3, align 8
 ; VF_16-LABEL: Checking a loop in 'f64_factor_4'
 ; VF_16:         Found an estimated cost of 192 for VF 16 For instruction: %tmp4 = load double, ptr %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load double, ptr %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load double, ptr %tmp2, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load double, ptr %tmp3, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, ptr %tmp0, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, ptr %tmp1, align 8
-; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, ptr %tmp2, align 8
-; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store double 0.000000e+00, ptr %tmp3, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double %tmp4, ptr %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double %tmp5, ptr %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double %tmp6, ptr %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 192 for VF 16 For instruction: store double %tmp7, ptr %tmp3, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f64.4, ptr %data, i64 %i, i32 0
@@ -1096,10 +1096,10 @@ for.body:
   %tmp5 = load double, ptr %tmp1, align 8
   %tmp6 = load double, ptr %tmp2, align 8
   %tmp7 = load double, ptr %tmp3, align 8
-  store double 0.0, ptr %tmp0, align 8
-  store double 0.0, ptr %tmp1, align 8
-  store double 0.0, ptr %tmp2, align 8
-  store double 0.0, ptr %tmp3, align 8
+  store double %tmp4, ptr %tmp0, align 8
+  store double %tmp5, ptr %tmp1, align 8
+  store double %tmp6, ptr %tmp2, align 8
+  store double %tmp7, ptr %tmp3, align 8
   %i.next = add nuw nsw i64 %i, 1
   %cond = icmp slt i64 %i.next, %n
   br i1 %cond, label %for.body, label %for.end
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll
index 83dc618fc83ff..02f8bc61cbb62 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll
@@ -12,9 +12,11 @@ entry:
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %mul = mul nsw i64 %iv, %s
-  %gep = getelementptr inbounds double, ptr %Src, i64 %mul
-  %bct = bitcast ptr %gep to ptr
+  %gep.src = getelementptr inbounds double, ptr %Src, i64 %mul
+  %bct = bitcast ptr %gep.src to ptr
   %ld = load i64, ptr %bct
+  %gep.data = getelementptr inbounds i64, ptr %data, i64 %iv
+  store i64 %ld, ptr %gep.data
   %iv.next = add nuw nsw i64 %iv, 1
   %cmp110.us = icmp slt i64 %iv.next, %n
   br i1 %cmp110.us, label %for.body, label %for.end
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll
index 314a133debbd7..98a942a501077 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll
@@ -51,7 +51,7 @@ define void @pr97452_scalable_vf1_for_no_live_out(ptr %src, ptr noalias %dst) {
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[L]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    store i64 [[FOR]], ptr [[GEP_DST]], align 8
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 22
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
 ; CHECK:       [[EXIT]]:
@@ -67,7 +67,7 @@ loop:
   %gep = getelementptr inbounds i64, ptr %src, i64 %iv
   %l = load i64, ptr %gep, align 8
   %gep.dst = getelementptr inbounds i64, ptr %dst, i64 %iv
-  store i64 %l, ptr %gep.dst
+  store i64 %for, ptr %gep.dst
   %ec = icmp eq i64 %iv, 22
   br i1 %ec, label %exit, label %loop
 

From af0f58cf14329f5e0fe56fed7c9eb4cd3107a1ce Mon Sep 17 00:00:00 2001
From: Job Henandez Lara <hj93@protonmail.com>
Date: Sun, 21 Jul 2024 08:17:41 -0700
Subject: [PATCH 336/486] [libc][math][c23] Add entrypoints and tests for
 fsqrt{,l,f128} (#99669)

---
 libc/config/linux/x86_64/entrypoints.txt    |  3 ++
 libc/docs/math/index.rst                    |  2 +-
 libc/spec/llvm_libc_ext.td                  |  2 ++
 libc/spec/stdc.td                           |  3 ++
 libc/src/math/CMakeLists.txt                |  4 +++
 libc/src/math/fsqrt.h                       | 20 +++++++++++
 libc/src/math/fsqrtf128.h                   | 21 ++++++++++++
 libc/src/math/fsqrtl.h                      | 20 +++++++++++
 libc/src/math/generic/CMakeLists.txt        | 37 +++++++++++++++++++++
 libc/src/math/generic/fsqrt.cpp             | 18 ++++++++++
 libc/src/math/generic/fsqrtf128.cpp         | 20 +++++++++++
 libc/src/math/generic/fsqrtl.cpp            | 20 +++++++++++
 libc/test/src/math/CMakeLists.txt           | 26 +++++++++++++++
 libc/test/src/math/fsqrt_test.cpp           | 13 ++++++++
 libc/test/src/math/fsqrtl_test.cpp          | 13 ++++++++
 libc/test/src/math/smoke/CMakeLists.txt     | 37 +++++++++++++++++++++
 libc/test/src/math/smoke/fsqrt_test.cpp     | 13 ++++++++
 libc/test/src/math/smoke/fsqrtf128_test.cpp | 13 ++++++++
 libc/test/src/math/smoke/fsqrtl_test.cpp    | 13 ++++++++
 libc/utils/MPFRWrapper/MPFRUtils.cpp        | 12 +++++++
 20 files changed, 309 insertions(+), 1 deletion(-)
 create mode 100644 libc/src/math/fsqrt.h
 create mode 100644 libc/src/math/fsqrtf128.h
 create mode 100644 libc/src/math/fsqrtl.h
 create mode 100644 libc/src/math/generic/fsqrt.cpp
 create mode 100644 libc/src/math/generic/fsqrtf128.cpp
 create mode 100644 libc/src/math/generic/fsqrtl.cpp
 create mode 100644 libc/test/src/math/fsqrt_test.cpp
 create mode 100644 libc/test/src/math/fsqrtl_test.cpp
 create mode 100644 libc/test/src/math/smoke/fsqrt_test.cpp
 create mode 100644 libc/test/src/math/smoke/fsqrtf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fsqrtl_test.cpp

diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index d4e246efb3f8a..736908809d96c 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -448,6 +448,8 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fromfpx
     libc.src.math.fromfpxf
     libc.src.math.fromfpxl
+    libc.src.math.fsqrt
+    libc.src.math.fsqrtl
     libc.src.math.hypot
     libc.src.math.hypotf
     libc.src.math.ilogb
@@ -659,6 +661,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.frexpf128
     libc.src.math.fromfpf128
     libc.src.math.fromfpxf128
+    libc.src.math.fsqrtf128
     libc.src.math.ilogbf128
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index cff7f69deb7f8..5cf9d8a42929e 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -300,7 +300,7 @@ Higher Math Functions
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | f16sqrt   | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | 7.12.14.6              | F.10.11                    |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fsqrt     | N/A              |                 |                        | N/A                  |                        | 7.12.14.6              | F.10.11                    |
+| fsqrt     | N/A              | |check|         |  |check|               | N/A                  | |check|\*              | 7.12.14.6              | F.10.11                    |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | hypot     | |check|          | |check|         |                        |                      |                        | 7.12.7.4               | F.10.4.4                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/spec/llvm_libc_ext.td b/libc/spec/llvm_libc_ext.td
index 55b354cae74e0..6291a9ceec60d 100644
--- a/libc/spec/llvm_libc_ext.td
+++ b/libc/spec/llvm_libc_ext.td
@@ -85,6 +85,8 @@ def LLVMLibcExt : StandardSpec<"llvm_libc_ext"> {
           GuardedFunctionSpec<"f16sqrtf", RetValSpec<Float16Type>, [ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"f16sqrtl", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
 
+          GuardedFunctionSpec<"fsqrtf128", RetValSpec<FloatType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+
           FunctionSpec<"powi", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>]>,
           FunctionSpec<"powif", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>]>,
       ]
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 285166ef0be78..437d6d04faf96 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -739,6 +739,9 @@ def StdC : StandardSpec<"stdc"> {
 
           GuardedFunctionSpec<"f16mulf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
 
+          FunctionSpec<"fsqrt", RetValSpec<FloatType>, [ArgSpec<DoubleType>]>,
+          FunctionSpec<"fsqrtl", RetValSpec<FloatType>, [ArgSpec<LongDoubleType>]>,
+	  
           GuardedFunctionSpec<"f16divf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
 
           GuardedFunctionSpec<"f16sqrtf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 050bf0f1ebf22..82dd23186187b 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -131,6 +131,10 @@ add_math_entrypoint_object(f16sqrtf)
 add_math_entrypoint_object(f16sqrtl)
 add_math_entrypoint_object(f16sqrtf128)
 
+add_math_entrypoint_object(fsqrt)
+add_math_entrypoint_object(fsqrtl)
+add_math_entrypoint_object(fsqrtf128)
+
 add_math_entrypoint_object(f16sub)
 add_math_entrypoint_object(f16subf)
 add_math_entrypoint_object(f16subl)
diff --git a/libc/src/math/fsqrt.h b/libc/src/math/fsqrt.h
new file mode 100644
index 0000000000000..e6a8c3179f811
--- /dev/null
+++ b/libc/src/math/fsqrt.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fsqrt -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FSQRT_H
+#define LLVM_LIBC_SRC_MATH_FSQRT_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float fsqrt(double x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FSQRT_H
diff --git a/libc/src/math/fsqrtf128.h b/libc/src/math/fsqrtf128.h
new file mode 100644
index 0000000000000..5286cbc33116d
--- /dev/null
+++ b/libc/src/math/fsqrtf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fsqrtf128 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FSQRTF128_H
+#define LLVM_LIBC_SRC_MATH_FSQRTF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float fsqrtf128(float128 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FSQRTF128_H
diff --git a/libc/src/math/fsqrtl.h b/libc/src/math/fsqrtl.h
new file mode 100644
index 0000000000000..a50eea3335458
--- /dev/null
+++ b/libc/src/math/fsqrtl.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fsqrtl ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FSQRTL_H
+#define LLVM_LIBC_SRC_MATH_FSQRTL_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float fsqrtl(long double x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FSQRTL_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 8e8ffb01adf60..7712a6d1ae527 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -4201,6 +4201,43 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  fsqrt
+  SRCS
+    fsqrt.cpp
+  HDRS
+    ../fsqrt.h
+  DEPENDS
+    libc.src.__support.FPUtil.generic.sqrt
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fsqrtl
+  SRCS
+    fsqrtl.cpp
+  HDRS
+    ../fsqrtl.h
+  DEPENDS
+    libc.src.__support.FPUtil.generic.sqrt
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fsqrtf128
+  SRCS
+    fsqrtf128.cpp
+  HDRS
+    ../fsqrtf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.generic.sqrt
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   cbrtf
   SRCS
diff --git a/libc/src/math/generic/fsqrt.cpp b/libc/src/math/generic/fsqrt.cpp
new file mode 100644
index 0000000000000..d54471fd067bf
--- /dev/null
+++ b/libc/src/math/generic/fsqrt.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of fsqrt function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fsqrt.h"
+#include "src/__support/FPUtil/generic/sqrt.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float, fsqrt, (double x)) { return fputil::sqrt<float>(x); }
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fsqrtf128.cpp b/libc/src/math/generic/fsqrtf128.cpp
new file mode 100644
index 0000000000000..f2c049529d6cd
--- /dev/null
+++ b/libc/src/math/generic/fsqrtf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fsqrt128 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fsqrtf128.h"
+#include "src/__support/FPUtil/generic/sqrt.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float, fsqrtf128, (float128 x)) {
+  return fputil::sqrt<float>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fsqrtl.cpp b/libc/src/math/generic/fsqrtl.cpp
new file mode 100644
index 0000000000000..b896a84aeded8
--- /dev/null
+++ b/libc/src/math/generic/fsqrtl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fsqrtl function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fsqrtl.h"
+#include "src/__support/FPUtil/generic/sqrt.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float, fsqrtl, (long double x)) {
+  return fputil::sqrt<float>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index f69b0d9b1b666..586338b2213de 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -2245,6 +2245,32 @@ add_fp_unittest(
     libc.src.math.f16sqrtl
 )
 
+add_fp_unittest(
+  fsqrt_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    fsqrt_test.cpp
+  HDRS
+    SqrtTest.h
+  DEPENDS
+    libc.src.math.fsqrt
+)
+
+add_fp_unittest(
+  fsqrtl_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    fsqrtl_test.cpp
+  HDRS
+    SqrtTest.h
+  DEPENDS
+    libc.src.math.fsqrtl
+)
+
 add_fp_unittest(
   cbrtf_test
   NEED_MPFR
diff --git a/libc/test/src/math/fsqrt_test.cpp b/libc/test/src/math/fsqrt_test.cpp
new file mode 100644
index 0000000000000..8471e3c9365e9
--- /dev/null
+++ b/libc/test/src/math/fsqrt_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fsqrt -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SqrtTest.h"
+
+#include "src/math/fsqrt.h"
+
+LIST_NARROWING_SQRT_TESTS(float, double, LIBC_NAMESPACE::fsqrt)
diff --git a/libc/test/src/math/fsqrtl_test.cpp b/libc/test/src/math/fsqrtl_test.cpp
new file mode 100644
index 0000000000000..1082a33466857
--- /dev/null
+++ b/libc/test/src/math/fsqrtl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fsqrtl ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SqrtTest.h"
+
+#include "src/math/fsqrtl.h"
+
+LIST_NARROWING_SQRT_TESTS(float, long double, LIBC_NAMESPACE::fsqrtl)
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index 69f53c5d27f15..f7caa03d11646 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -3959,6 +3959,43 @@ add_fp_unittest(
     libc.src.math.f16sqrtf128
 )
 
+add_fp_unittest(
+  fsqrt_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fsqrt_test.cpp
+  HDRS
+    SqrtTest.h
+  DEPENDS
+    libc.src.math.fsqrt
+)
+
+
+add_fp_unittest(
+  fsqrtl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fsqrtl_test.cpp
+  HDRS
+    SqrtTest.h
+  DEPENDS
+    libc.src.math.fsqrtl
+)
+
+add_fp_unittest(
+  fsqrtf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fsqrtf128_test.cpp
+  HDRS
+    SqrtTest.h
+  DEPENDS
+    libc.src.math.fsqrtf128
+)
+
 add_fp_unittest(
   sin_test
   SUITE
diff --git a/libc/test/src/math/smoke/fsqrt_test.cpp b/libc/test/src/math/smoke/fsqrt_test.cpp
new file mode 100644
index 0000000000000..8471e3c9365e9
--- /dev/null
+++ b/libc/test/src/math/smoke/fsqrt_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fsqrt -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SqrtTest.h"
+
+#include "src/math/fsqrt.h"
+
+LIST_NARROWING_SQRT_TESTS(float, double, LIBC_NAMESPACE::fsqrt)
diff --git a/libc/test/src/math/smoke/fsqrtf128_test.cpp b/libc/test/src/math/smoke/fsqrtf128_test.cpp
new file mode 100644
index 0000000000000..b125a35b09b9e
--- /dev/null
+++ b/libc/test/src/math/smoke/fsqrtf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fsqrtf128 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SqrtTest.h"
+
+#include "src/math/fsqrtf128.h"
+
+LIST_NARROWING_SQRT_TESTS(float, float128, LIBC_NAMESPACE::fsqrtf128)
diff --git a/libc/test/src/math/smoke/fsqrtl_test.cpp b/libc/test/src/math/smoke/fsqrtl_test.cpp
new file mode 100644
index 0000000000000..1082a33466857
--- /dev/null
+++ b/libc/test/src/math/smoke/fsqrtl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fsqrtl ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SqrtTest.h"
+
+#include "src/math/fsqrtl.h"
+
+LIST_NARROWING_SQRT_TESTS(float, long double, LIBC_NAMESPACE::fsqrtl)
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp
index bb63e0b9f4de0..d2c28c19ab176 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.cpp
+++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp
@@ -887,6 +887,13 @@ template void explain_unary_operation_single_output_error(Operation op,
                                                           long double,
                                                           long double, double,
                                                           RoundingMode);
+template void explain_unary_operation_single_output_error(Operation op, double,
+                                                          float, double,
+                                                          RoundingMode);
+template void explain_unary_operation_single_output_error(Operation op,
+                                                          long double, float,
+                                                          double, RoundingMode);
+
 #ifdef LIBC_TYPES_HAS_FLOAT16
 template void explain_unary_operation_single_output_error(Operation op, float16,
                                                           float16, double,
@@ -1106,6 +1113,11 @@ template bool compare_unary_operation_single_output(Operation, double, double,
 template bool compare_unary_operation_single_output(Operation, long double,
                                                     long double, double,
                                                     RoundingMode);
+template bool compare_unary_operation_single_output(Operation, double, float,
+                                                    double, RoundingMode);
+template bool compare_unary_operation_single_output(Operation, long double,
+                                                    float, double,
+                                                    RoundingMode);
 #ifdef LIBC_TYPES_HAS_FLOAT16
 template bool compare_unary_operation_single_output(Operation, float16, float16,
                                                     double, RoundingMode);

From 7f17b6b740bd49b84430a46b366381bfc8b74fb0 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 09:23:45 -0700
Subject: [PATCH 337/486] [MC] Migrate to new createAsmStreamer that avoids
 unused bool parameters

---
 clang/tools/driver/cc1as_main.cpp                        | 6 ++----
 llvm/lib/CodeGen/LLVMTargetMachine.cpp                   | 4 +---
 llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp           | 6 +++---
 llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp       | 6 +++---
 .../llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp  | 9 ++-------
 llvm/tools/llvm-mc/llvm-mc.cpp                           | 6 ++----
 llvm/tools/llvm-ml/llvm-ml.cpp                           | 7 +++----
 7 files changed, 16 insertions(+), 28 deletions(-)

diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index 15d1e0c2f2f2d..b661a43c88b08 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -548,10 +548,8 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
         TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
 
     auto FOut = std::make_unique<formatted_raw_ostream>(*Out);
-    Str.reset(TheTarget->createAsmStreamer(
-        Ctx, std::move(FOut), /*asmverbose*/ true,
-        /*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB),
-        Opts.ShowInst));
+    Str.reset(TheTarget->createAsmStreamer(Ctx, std::move(FOut), IP,
+                                           std::move(CE), std::move(MAB)));
   } else if (Opts.OutputType == AssemblerInvocation::FT_Null) {
     Str.reset(createNullStreamer(Ctx));
   } else {
diff --git a/llvm/lib/CodeGen/LLVMTargetMachine.cpp b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
index 819187c129c3a..d0dfafeaef561 100644
--- a/llvm/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
@@ -171,9 +171,7 @@ Expected<std::unique_ptr<MCStreamer>> LLVMTargetMachine::createMCStreamer(
         getTarget().createMCAsmBackend(STI, MRI, Options.MCOptions));
     auto FOut = std::make_unique<formatted_raw_ostream>(Out);
     MCStreamer *S = getTarget().createAsmStreamer(
-        Context, std::move(FOut), Options.MCOptions.AsmVerbose, true,
-        InstPrinter, std::move(MCE), std::move(MAB),
-        Options.MCOptions.ShowMCInst);
+        Context, std::move(FOut), InstPrinter, std::move(MCE), std::move(MAB));
     AsmStreamer.reset(S);
     break;
   }
diff --git a/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp b/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
index 45a62daffb033..bca3125368353 100644
--- a/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
+++ b/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
@@ -103,9 +103,9 @@ Error DwarfStreamer::init(Triple TheTriple,
     MIP = TheTarget->createMCInstPrinter(TheTriple, MAI->getAssemblerDialect(),
                                          *MAI, *MII, *MRI);
     MS = TheTarget->createAsmStreamer(
-        *MC, std::make_unique<formatted_raw_ostream>(OutFile), true, true, MIP,
-        std::unique_ptr<MCCodeEmitter>(MCE), std::unique_ptr<MCAsmBackend>(MAB),
-        true);
+        *MC, std::make_unique<formatted_raw_ostream>(OutFile), MIP,
+        std::unique_ptr<MCCodeEmitter>(MCE),
+        std::unique_ptr<MCAsmBackend>(MAB));
     break;
   }
   case DWARFLinker::OutputFileType::Object: {
diff --git a/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp b/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp
index f790766348b53..fbab6b25ca0f1 100644
--- a/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp
+++ b/llvm/lib/DWARFLinker/Parallel/DWARFEmitterImpl.cpp
@@ -82,9 +82,9 @@ Error DwarfEmitterImpl::init(Triple TheTriple,
     MIP = TheTarget->createMCInstPrinter(TheTriple, MAI->getAssemblerDialect(),
                                          *MAI, *MII, *MRI);
     MS = TheTarget->createAsmStreamer(
-        *MC, std::make_unique<formatted_raw_ostream>(OutFile), true, true, MIP,
-        std::unique_ptr<MCCodeEmitter>(MCE), std::unique_ptr<MCAsmBackend>(MAB),
-        true);
+        *MC, std::make_unique<formatted_raw_ostream>(OutFile), MIP,
+        std::unique_ptr<MCCodeEmitter>(MCE),
+        std::unique_ptr<MCAsmBackend>(MAB));
     break;
   }
   case DWARFLinker::OutputFileType::Object: {
diff --git a/llvm/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp b/llvm/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp
index ef9d0f37198ac..615a39ee9bb4d 100644
--- a/llvm/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp
+++ b/llvm/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp
@@ -131,10 +131,6 @@ static int AssembleInput(const char *ProgName, const Target *TheTarget,
 
 
 int AssembleOneInput(const uint8_t *Data, size_t Size) {
-  const bool ShowInst = false;
-  const bool AsmVerbose = false;
-  const bool UseDwarfDirectory = true;
-
   Triple TheTriple(Triple::normalize(TripleName));
 
   SourceMgr SrcMgr;
@@ -204,9 +200,8 @@ int AssembleOneInput(const uint8_t *Data, size_t Size) {
   std::unique_ptr<MCStreamer> Str;
 
   if (FileType == OFT_AssemblyFile) {
-    Str.reset(TheTarget->createAsmStreamer(Ctx, std::move(FOut), AsmVerbose,
-                                           UseDwarfDirectory, IP, std::move(CE),
-                                           std::move(MAB), ShowInst));
+    Str.reset(TheTarget->createAsmStreamer(Ctx, std::move(FOut), IP,
+                                           std::move(CE), std::move(MAB)));
   } else {
     assert(FileType == OFT_ObjectFile && "Invalid file type!");
 
diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp
index 0f1e330541441..3a44ddf17974a 100644
--- a/llvm/tools/llvm-mc/llvm-mc.cpp
+++ b/llvm/tools/llvm-mc/llvm-mc.cpp
@@ -537,10 +537,8 @@ int main(int argc, char **argv) {
     std::unique_ptr<MCAsmBackend> MAB(
         TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
     auto FOut = std::make_unique<formatted_raw_ostream>(*OS);
-    Str.reset(
-        TheTarget->createAsmStreamer(Ctx, std::move(FOut), /*asmverbose*/ true,
-                                     /*useDwarfDirectory*/ true, IP,
-                                     std::move(CE), std::move(MAB), ShowInst));
+    Str.reset(TheTarget->createAsmStreamer(Ctx, std::move(FOut), IP,
+                                           std::move(CE), std::move(MAB)));
 
   } else if (FileType == OFT_Null) {
     Str.reset(TheTarget->createNullStreamer(Ctx));
diff --git a/llvm/tools/llvm-ml/llvm-ml.cpp b/llvm/tools/llvm-ml/llvm-ml.cpp
index bcfec97019c1c..db69109e2d1fa 100644
--- a/llvm/tools/llvm-ml/llvm-ml.cpp
+++ b/llvm/tools/llvm-ml/llvm-ml.cpp
@@ -264,6 +264,7 @@ int llvm_ml_main(int Argc, char **Argv, const llvm::ToolContext &) {
   MCOptions.AssemblyLanguage = "masm";
   MCOptions.MCFatalWarnings = InputArgs.hasArg(OPT_fatal_warnings);
   MCOptions.MCSaveTempLabels = InputArgs.hasArg(OPT_save_temp_labels);
+  MCOptions.ShowMCInst = InputArgs.hasArg(OPT_show_inst);
   MCOptions.AsmVerbose = true;
 
   Triple TheTriple = GetTriple(ProgName, InputArgs);
@@ -386,10 +387,8 @@ int llvm_ml_main(int Argc, char **Argv, const llvm::ToolContext &) {
     std::unique_ptr<MCAsmBackend> MAB(
         TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
     auto FOut = std::make_unique<formatted_raw_ostream>(*OS);
-    Str.reset(TheTarget->createAsmStreamer(
-        Ctx, std::move(FOut), /*asmverbose*/ true,
-        /*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB),
-        InputArgs.hasArg(OPT_show_inst)));
+    Str.reset(TheTarget->createAsmStreamer(Ctx, std::move(FOut), IP,
+                                           std::move(CE), std::move(MAB)));
 
   } else if (FileType == "null") {
     Str.reset(TheTarget->createNullStreamer(Ctx));

From 14a543ea2a1ae0d9e772a34a4e5a87dcc33d5e0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Sun, 21 Jul 2024 17:44:12 +0100
Subject: [PATCH 338/486] [mlir][test] Add comments in a test (nfc) (#99810)

Documents which patterns are tested in:
  * vector-transfer-collapse-inner-most-dims.mlir.
---
 .../Vector/vector-transfer-collapse-inner-most-dims.mlir        | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir b/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir
index 3f36756fe95d7..cd56c1bf9695b 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir
@@ -2,6 +2,7 @@
 
 //-----------------------------------------------------------------------------
 // 1. vector.transfer_read
+// [Pattern: DropInnerMostUnitDimsTransferRead]
 //-----------------------------------------------------------------------------
 
 func.func @contiguous_inner_most(%src: memref<1x1x8x1xf32, strided<[3072, 8, 1, 1], offset: ?>>) -> vector<1x8x1xf32>{
@@ -306,6 +307,7 @@ func.func @negative_non_unit_strides(%src: memref<512x16x1xf32, strided<[8192, 1
 
 //-----------------------------------------------------------------------------
 // 2. vector.transfer_write
+// [Pattern: DropInnerMostUnitDimsTransferWrite]
 //-----------------------------------------------------------------------------
 
 func.func @contiguous_inner_most(%dest: memref<1x512x16x1x1xf32>, %v: vector<1x16x16x1x1xf32>, %i: index) {

From 867faeec054abb4c035673189c1169fef45f54c8 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 09:44:15 -0700
Subject: [PATCH 339/486] [MC] Migrate to createAsmStreamer without unused bool
 parameters

In bolt/lib/Passes/AsmDump.cpp, the MCInstPrinter is created with false
AsmVerbose. The AsmVerbose argument to createAsmStreamer is unused.

Deprecate the legacy Target::createAsmStreamer overload, which might be
used by downstream.
---
 bolt/lib/Passes/AsmDump.cpp           | 7 ++-----
 llvm/include/llvm/MC/TargetRegistry.h | 4 ++--
 llvm/lib/MC/MCAsmStreamer.cpp         | 4 +---
 llvm/lib/MC/TargetRegistry.cpp        | 4 ++--
 4 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/bolt/lib/Passes/AsmDump.cpp b/bolt/lib/Passes/AsmDump.cpp
index 0a12eae8b5f7f..97f985d56ce64 100644
--- a/bolt/lib/Passes/AsmDump.cpp
+++ b/bolt/lib/Passes/AsmDump.cpp
@@ -139,11 +139,8 @@ void dumpFunction(const BinaryFunction &BF) {
   auto FOut = std::make_unique<formatted_raw_ostream>(OS);
   FOut->SetUnbuffered();
   std::unique_ptr<MCStreamer> AsmStreamer(
-      createAsmStreamer(*LocalCtx, std::move(FOut),
-                        /*isVerboseAsm=*/true,
-                        /*useDwarfDirectory=*/false, InstructionPrinter,
-                        std::move(MCEInstance.MCE), std::move(MAB),
-                        /*ShowInst=*/false));
+      createAsmStreamer(*LocalCtx, std::move(FOut), InstructionPrinter,
+                        std::move(MCEInstance.MCE), std::move(MAB)));
   AsmStreamer->initSections(true, *BC.STI);
   std::unique_ptr<TargetMachine> TM(BC.TheTarget->createTargetMachine(
       BC.TripleName, "", "", TargetOptions(), std::nullopt));
diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index 14ceb76d312c5..58d9ad95c64d4 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -86,9 +86,8 @@ MCStreamer *createNullStreamer(MCContext &Ctx);
 /// the assembly.
 MCStreamer *
 createAsmStreamer(MCContext &Ctx, std::unique_ptr<formatted_raw_ostream> OS,
-                  bool isVerboseAsm, bool useDwarfDirectory,
                   MCInstPrinter *InstPrint, std::unique_ptr<MCCodeEmitter> &&CE,
-                  std::unique_ptr<MCAsmBackend> &&TAB, bool ShowInst);
+                  std::unique_ptr<MCAsmBackend> &&TAB);
 
 MCStreamer *createELFStreamer(MCContext &Ctx,
                               std::unique_ptr<MCAsmBackend> &&TAB,
@@ -544,6 +543,7 @@ class Target {
                                 MCInstPrinter *IP,
                                 std::unique_ptr<MCCodeEmitter> CE,
                                 std::unique_ptr<MCAsmBackend> TAB) const;
+  LLVM_DEPRECATED("Use the overload without the 3 unused bool", "")
   MCStreamer *
   createAsmStreamer(MCContext &Ctx, std::unique_ptr<formatted_raw_ostream> OS,
                     bool IsVerboseAsm, bool UseDwarfDirectory,
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index db93a33bbe3f7..9309d5987dc94 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -2657,11 +2657,9 @@ void MCAsmStreamer::doFinalizationAtSectionEnd(MCSection *Section) {
 
 MCStreamer *llvm::createAsmStreamer(MCContext &Context,
                                     std::unique_ptr<formatted_raw_ostream> OS,
-                                    bool isVerboseAsm, bool useDwarfDirectory,
                                     MCInstPrinter *IP,
                                     std::unique_ptr<MCCodeEmitter> &&CE,
-                                    std::unique_ptr<MCAsmBackend> &&MAB,
-                                    bool ShowInst) {
+                                    std::unique_ptr<MCAsmBackend> &&MAB) {
   return new MCAsmStreamer(Context, std::move(OS), IP, std::move(CE),
                            std::move(MAB));
 }
diff --git a/llvm/lib/MC/TargetRegistry.cpp b/llvm/lib/MC/TargetRegistry.cpp
index 4190117c1e264..21c09755e0013 100644
--- a/llvm/lib/MC/TargetRegistry.cpp
+++ b/llvm/lib/MC/TargetRegistry.cpp
@@ -94,8 +94,8 @@ MCStreamer *Target::createAsmStreamer(MCContext &Ctx,
                                       std::unique_ptr<MCCodeEmitter> CE,
                                       std::unique_ptr<MCAsmBackend> TAB) const {
   formatted_raw_ostream &OSRef = *OS;
-  MCStreamer *S = llvm::createAsmStreamer(Ctx, std::move(OS), false, false, IP,
-                                          std::move(CE), std::move(TAB), false);
+  MCStreamer *S = llvm::createAsmStreamer(Ctx, std::move(OS), IP,
+                                          std::move(CE), std::move(TAB));
   auto *TO = Ctx.getTargetOptions();
   createAsmTargetStreamer(*S, OSRef, IP, TO && TO->AsmVerbose);
   return S;

From 233cca169237b91d16092c82bd55ee6a283afe98 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 10:02:47 -0700
Subject: [PATCH 340/486] [ARM,Hexagon] Ignore IsVerboseAsm parameter in favor
 of MCStreamer::isVerboseAsm()

... to improve consistency. Most targets don't use VerboseAsm. When they
do (X86, SystemZ), they use MCStreamer::isVerboseAsm().
---
 llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp      | 9 ++++-----
 .../Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp  | 9 ++++-----
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index 31b577b9c301f..948bf8f762d82 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -114,15 +114,14 @@ class ARMTargetAsmStreamer : public ARMTargetStreamer {
 
 public:
   ARMTargetAsmStreamer(MCStreamer &S, formatted_raw_ostream &OS,
-                       MCInstPrinter &InstPrinter, bool VerboseAsm);
+                       MCInstPrinter &InstPrinter);
 };
 
 ARMTargetAsmStreamer::ARMTargetAsmStreamer(MCStreamer &S,
                                            formatted_raw_ostream &OS,
-                                           MCInstPrinter &InstPrinter,
-                                           bool VerboseAsm)
+                                           MCInstPrinter &InstPrinter)
     : ARMTargetStreamer(S), OS(OS), InstPrinter(InstPrinter),
-      IsVerboseAsm(VerboseAsm) {}
+      IsVerboseAsm(S.isVerboseAsm()) {}
 
 void ARMTargetAsmStreamer::emitFnStart() { OS << "\t.fnstart\n"; }
 void ARMTargetAsmStreamer::emitFnEnd() { OS << "\t.fnend\n"; }
@@ -1464,7 +1463,7 @@ MCTargetStreamer *createARMTargetAsmStreamer(MCStreamer &S,
                                              formatted_raw_ostream &OS,
                                              MCInstPrinter *InstPrint,
                                              bool isVerboseAsm) {
-  return new ARMTargetAsmStreamer(S, OS, *InstPrint, isVerboseAsm);
+  return new ARMTargetAsmStreamer(S, OS, *InstPrint);
 }
 
 MCTargetStreamer *createARMNullTargetStreamer(MCStreamer &S) {
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
index eab7647e633bc..d16b70f439696 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
@@ -226,12 +226,11 @@ namespace {
 
 class HexagonTargetAsmStreamer : public HexagonTargetStreamer {
   formatted_raw_ostream &OS;
-  bool IsVerboseAsm;
 
 public:
   HexagonTargetAsmStreamer(MCStreamer &S, formatted_raw_ostream &OS,
-                           bool IsVerboseAsm, MCInstPrinter &IP)
-      : HexagonTargetStreamer(S), OS(OS), IsVerboseAsm(IsVerboseAsm) {}
+                           MCInstPrinter &IP)
+      : HexagonTargetStreamer(S), OS(OS) {}
 
   void prettyPrintAsm(MCInstPrinter &InstPrinter, uint64_t Address,
                       const MCInst &Inst, const MCSubtargetInfo &STI,
@@ -275,7 +274,7 @@ class HexagonTargetAsmStreamer : public HexagonTargetStreamer {
 
   void emitAttribute(unsigned Attribute, unsigned Value) override {
     OS << "\t.attribute\t" << Attribute << ", " << Twine(Value);
-    if (IsVerboseAsm) {
+    if (getStreamer().isVerboseAsm()) {
       StringRef Name = ELFAttrs::attrTypeAsString(
           Attribute, HexagonAttrs::getHexagonAttributeTags());
       if (!Name.empty())
@@ -379,7 +378,7 @@ static MCInstPrinter *createHexagonMCInstPrinter(const Triple &T,
 static MCTargetStreamer *
 createMCAsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS,
                           MCInstPrinter *IP, bool IsVerboseAsm) {
-  return new HexagonTargetAsmStreamer(S, OS, IsVerboseAsm, *IP);
+  return new HexagonTargetAsmStreamer(S, OS, *IP);
 }
 
 static MCStreamer *createMCStreamer(Triple const &T, MCContext &Context,

From 8f14e39e59c9dc0ea93b098cf8d03fc23c23a7ec Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 10:19:17 -0700
Subject: [PATCH 341/486] [MC] Remove unnecessary isVerboseAsm from
 Target::AsmTargetStreamerCtorTy

The parameter is confusing as it duplicates MCStreamer::isVeboseAsm
(initialized from MCTargetOptions::AsmVerbose). After
233cca169237b91d16092c82bd55ee6a283afe98, no in-tree target uses the
parameter.
---
 llvm/include/llvm/MC/TargetRegistry.h                     | 8 ++++----
 llvm/lib/MC/TargetRegistry.cpp                            | 4 +---
 .../Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp    | 3 +--
 .../lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h | 3 +--
 .../lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp | 7 +++----
 llvm/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp      | 3 +--
 llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp       | 3 +--
 llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h        | 3 +--
 llvm/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp      | 3 +--
 llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp    | 7 +++----
 .../Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp   | 6 +++---
 llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp    | 3 +--
 llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp  | 2 +-
 llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp  | 3 +--
 llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp  | 7 +++----
 llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCTargetDesc.cpp  | 2 +-
 llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp  | 3 +--
 .../Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp   | 8 +++-----
 llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.cpp        | 3 +--
 .../WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp  | 7 +++----
 llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h        | 3 +--
 .../Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp  | 3 +--
 llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp  | 3 +--
 .../lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp | 2 +-
 24 files changed, 39 insertions(+), 60 deletions(-)

diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index 58d9ad95c64d4..fdfbfb45bf86d 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -205,9 +205,9 @@ class Target {
                       std::unique_ptr<MCCodeEmitter> &&Emitter);
 
   using NullTargetStreamerCtorTy = MCTargetStreamer *(*)(MCStreamer &S);
-  using AsmTargetStreamerCtorTy = MCTargetStreamer *(*)(
-      MCStreamer &S, formatted_raw_ostream &OS, MCInstPrinter *InstPrint,
-      bool IsVerboseAsm);
+  using AsmTargetStreamerCtorTy =
+      MCTargetStreamer *(*)(MCStreamer &S, formatted_raw_ostream &OS,
+                            MCInstPrinter *InstPrint);
   using ObjectTargetStreamerCtorTy = MCTargetStreamer *(*)(
       MCStreamer &S, const MCSubtargetInfo &STI);
   using MCRelocationInfoCtorTy = MCRelocationInfo *(*)(const Triple &TT,
@@ -555,7 +555,7 @@ class Target {
                                             MCInstPrinter *InstPrint,
                                             bool IsVerboseAsm) const {
     if (AsmTargetStreamerCtorFn)
-      return AsmTargetStreamerCtorFn(S, OS, InstPrint, IsVerboseAsm);
+      return AsmTargetStreamerCtorFn(S, OS, InstPrint);
     return nullptr;
   }
 
diff --git a/llvm/lib/MC/TargetRegistry.cpp b/llvm/lib/MC/TargetRegistry.cpp
index 21c09755e0013..b6b1a9b2eda9e 100644
--- a/llvm/lib/MC/TargetRegistry.cpp
+++ b/llvm/lib/MC/TargetRegistry.cpp
@@ -14,7 +14,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCObjectWriter.h"
-#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <vector>
@@ -96,8 +95,7 @@ MCStreamer *Target::createAsmStreamer(MCContext &Ctx,
   formatted_raw_ostream &OSRef = *OS;
   MCStreamer *S = llvm::createAsmStreamer(Ctx, std::move(OS), IP,
                                           std::move(CE), std::move(TAB));
-  auto *TO = Ctx.getTargetOptions();
-  createAsmTargetStreamer(*S, OSRef, IP, TO && TO->AsmVerbose);
+  createAsmTargetStreamer(*S, OSRef, IP, /*ignore=*/false);
   return S;
 }
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index bfeca4bd5a92d..8754e551727ea 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -332,8 +332,7 @@ void AArch64TargetELFStreamer::finish() {
 
 MCTargetStreamer *
 llvm::createAArch64AsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS,
-                                     MCInstPrinter *InstPrint,
-                                     bool isVerboseAsm) {
+                                     MCInstPrinter *InstPrint) {
   return new AArch64TargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h
index 7b4f102840aa5..91bdc880998b2 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h
@@ -58,8 +58,7 @@ createAArch64WinCOFFObjectWriter(const Triple &TheTriple);
 
 MCTargetStreamer *createAArch64AsmTargetStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &OS,
-                                                 MCInstPrinter *InstPrint,
-                                                 bool isVerboseAsm);
+                                                 MCInstPrinter *InstPrint);
 
 namespace AArch64_MC {
 void initLLVMToCVRegMapping(MCRegisterInfo *MRI);
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp
index d2ac5a7ebb2fb..37eb0b57fb153 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp
@@ -91,10 +91,9 @@ static MCInstPrinter *createAMDGPUMCInstPrinter(const Triple &T,
   return new AMDGPUInstPrinter(MAI, MII, MRI);
 }
 
-static MCTargetStreamer *createAMDGPUAsmTargetStreamer(MCStreamer &S,
-                                                      formatted_raw_ostream &OS,
-                                                      MCInstPrinter *InstPrint,
-                                                      bool isVerboseAsm) {
+static MCTargetStreamer *
+createAMDGPUAsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS,
+                              MCInstPrinter *InstPrint) {
   return new AMDGPUTargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp b/llvm/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp
index 36b00af2c0b48..8cd39668311fd 100644
--- a/llvm/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp
+++ b/llvm/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp
@@ -77,8 +77,7 @@ ARCTargetStreamer::~ARCTargetStreamer() = default;
 
 static MCTargetStreamer *createTargetAsmStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &OS,
-                                                 MCInstPrinter *InstPrint,
-                                                 bool isVerboseAsm) {
+                                                 MCInstPrinter *InstPrint) {
   return new ARCTargetStreamer(S);
 }
 
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index 948bf8f762d82..649e6dc039830 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -1461,8 +1461,7 @@ namespace llvm {
 
 MCTargetStreamer *createARMTargetAsmStreamer(MCStreamer &S,
                                              formatted_raw_ostream &OS,
-                                             MCInstPrinter *InstPrint,
-                                             bool isVerboseAsm) {
+                                             MCInstPrinter *InstPrint) {
   return new ARMTargetAsmStreamer(S, OS, *InstPrint);
 }
 
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
index 66f19237f275f..39b189780f1ff 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
@@ -67,8 +67,7 @@ MCSubtargetInfo *createARMMCSubtargetInfo(const Triple &TT, StringRef CPU,
 MCTargetStreamer *createARMNullTargetStreamer(MCStreamer &S);
 MCTargetStreamer *createARMTargetAsmStreamer(MCStreamer &S,
                                              formatted_raw_ostream &OS,
-                                             MCInstPrinter *InstPrint,
-                                             bool isVerboseAsm);
+                                             MCInstPrinter *InstPrint);
 MCTargetStreamer *createARMObjectTargetStreamer(MCStreamer &S,
                                                 const MCSubtargetInfo &STI);
 MCTargetStreamer *createARMObjectTargetELFStreamer(MCStreamer &S);
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp
index 119baff83dae4..3153dfb3b827f 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp
@@ -84,8 +84,7 @@ createAVRObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) {
 
 static MCTargetStreamer *createMCAsmTargetStreamer(MCStreamer &S,
                                                    formatted_raw_ostream &OS,
-                                                   MCInstPrinter *InstPrint,
-                                                   bool isVerboseAsm) {
+                                                   MCInstPrinter *InstPrint) {
   return new AVRTargetAsmStreamer(S);
 }
 
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp
index c3403ade389c4..bb122f683644b 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp
@@ -95,10 +95,9 @@ static MCStreamer *createELFStreamer(const Triple &T, MCContext &Ctx,
   return S;
 }
 
-static MCTargetStreamer *createCSKYAsmTargetStreamer(MCStreamer &S,
-                                                     formatted_raw_ostream &OS,
-                                                     MCInstPrinter *InstPrinter,
-                                                     bool isVerboseAsm) {
+static MCTargetStreamer *
+createCSKYAsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS,
+                            MCInstPrinter *InstPrinter) {
   return new CSKYTargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
index d16b70f439696..c16edc9857bfc 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
@@ -375,9 +375,9 @@ static MCInstPrinter *createHexagonMCInstPrinter(const Triple &T,
     return nullptr;
 }
 
-static MCTargetStreamer *
-createMCAsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS,
-                          MCInstPrinter *IP, bool IsVerboseAsm) {
+static MCTargetStreamer *createMCAsmTargetStreamer(MCStreamer &S,
+                                                   formatted_raw_ostream &OS,
+                                                   MCInstPrinter *IP) {
   return new HexagonTargetAsmStreamer(S, OS, *IP);
 }
 
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
index 499cbd873e299..ca95f67174da1 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
@@ -117,8 +117,7 @@ static MCStreamer *createMCStreamer(const Triple &T, MCContext &Context,
 
 static MCTargetStreamer *createMipsAsmTargetStreamer(MCStreamer &S,
                                                      formatted_raw_ostream &OS,
-                                                     MCInstPrinter *InstPrint,
-                                                     bool isVerboseAsm) {
+                                                     MCInstPrinter *InstPrint) {
   return new MipsTargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
index 50e7cc6371253..1cafd236a2925 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
@@ -62,7 +62,7 @@ static MCInstPrinter *createNVPTXMCInstPrinter(const Triple &T,
 
 static MCTargetStreamer *createTargetAsmStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &,
-                                                 MCInstPrinter *, bool) {
+                                                 MCInstPrinter *) {
   return new NVPTXAsmTargetStreamer(S);
 }
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index 9106e6ab6397d..8d1f8624ee406 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -439,8 +439,7 @@ class PPCTargetXCOFFStreamer : public PPCTargetStreamer {
 
 static MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &OS,
-                                                 MCInstPrinter *InstPrint,
-                                                 bool isVerboseAsm) {
+                                                 MCInstPrinter *InstPrint) {
   return new PPCTargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
index 691a5892ae827..e051312d61a7b 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
@@ -110,10 +110,9 @@ createRISCVObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) {
   return nullptr;
 }
 
-static MCTargetStreamer *createRISCVAsmTargetStreamer(MCStreamer &S,
-                                                      formatted_raw_ostream &OS,
-                                                      MCInstPrinter *InstPrint,
-                                                      bool isVerboseAsm) {
+static MCTargetStreamer *
+createRISCVAsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS,
+                             MCInstPrinter *InstPrint) {
   return new RISCVTargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCTargetDesc.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCTargetDesc.cpp
index 2f302ed4c9e83..21a952649ff51 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCTargetDesc.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCTargetDesc.cpp
@@ -51,7 +51,7 @@ createSPIRVMCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
 
 static MCTargetStreamer *createTargetAsmStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &,
-                                                 MCInstPrinter *, bool) {
+                                                 MCInstPrinter *) {
   return new SPIRVTargetStreamer(S);
 }
 
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
index ad6ca0911adb9..2f89bd6ad9c96 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
@@ -91,8 +91,7 @@ createObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) {
 
 static MCTargetStreamer *createTargetAsmStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &OS,
-                                                 MCInstPrinter *InstPrint,
-                                                 bool isVerboseAsm) {
+                                                 MCInstPrinter *InstPrint) {
   return new SparcTargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
index 8e86c59bf9194..f58674ee118ee 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
@@ -223,11 +223,9 @@ class SystemZTargetELFStreamer : public SystemZTargetStreamer {
 };
 } // end namespace
 
-static MCTargetStreamer *
-createAsmTargetStreamer(MCStreamer &S,
-                        formatted_raw_ostream &OS,
-                        MCInstPrinter *InstPrint,
-                        bool isVerboseAsm) {
+static MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
+                                                 formatted_raw_ostream &OS,
+                                                 MCInstPrinter *InstPrint) {
   return new SystemZTargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.cpp
index 6611e4c42eb2b..019748413d32e 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.cpp
@@ -68,8 +68,7 @@ createObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) {
 
 static MCTargetStreamer *createTargetAsmStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &OS,
-                                                 MCInstPrinter *InstPrint,
-                                                 bool isVerboseAsm) {
+                                                 MCInstPrinter *InstPrint) {
   return new VETargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
index 71dfe1062956e..cea282f62fe15 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
@@ -111,10 +111,9 @@ createObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) {
   return new WebAssemblyTargetWasmStreamer(S);
 }
 
-static MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
-                                                 formatted_raw_ostream &OS,
-                                                 MCInstPrinter * /*InstPrint*/,
-                                                 bool /*isVerboseAsm*/) {
+static MCTargetStreamer *
+createAsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS,
+                        MCInstPrinter * /*InstPrint*/) {
   return new WebAssemblyTargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
index 4e83e7e437aeb..e166b68668d9b 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
@@ -114,8 +114,7 @@ MCAsmBackend *createX86_64AsmBackend(const Target &T,
 /// Implements X86-only directives for assembly emission.
 MCTargetStreamer *createX86AsmTargetStreamer(MCStreamer &S,
                                              formatted_raw_ostream &OS,
-                                             MCInstPrinter *InstPrinter,
-                                             bool IsVerboseAsm);
+                                             MCInstPrinter *InstPrinter);
 
 /// Implements X86-only directives for object files.
 MCTargetStreamer *createX86ObjectTargetStreamer(MCStreamer &S,
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp
index 19427617a3d3e..7f8da407f8543 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp
@@ -445,8 +445,7 @@ bool X86WinCOFFTargetStreamer::emitFPOData(const MCSymbol *ProcSym, SMLoc L) {
 
 MCTargetStreamer *llvm::createX86AsmTargetStreamer(MCStreamer &S,
                                                    formatted_raw_ostream &OS,
-                                                   MCInstPrinter *InstPrinter,
-                                                   bool IsVerboseAsm) {
+                                                   MCInstPrinter *InstPrinter) {
   // FIXME: This makes it so we textually assemble COFF directives on ELF.
   // That's kind of nonsensical.
   return new X86WinCOFFAsmTargetStreamer(S, OS, *InstPrinter);
diff --git a/llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp b/llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
index bc69e1868b88f..a3c48735a98a3 100644
--- a/llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
+++ b/llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
@@ -117,8 +117,7 @@ void XCoreTargetAsmStreamer::emitCCBottomFunction(StringRef Name) {
 
 static MCTargetStreamer *createTargetAsmStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &OS,
-                                                 MCInstPrinter *InstPrint,
-                                                 bool isVerboseAsm) {
+                                                 MCInstPrinter *InstPrint) {
   return new XCoreTargetAsmStreamer(S, OS);
 }
 
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
index 87ef66ba742b6..2653c293dc0c4 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
@@ -66,7 +66,7 @@ createXtensaMCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
 
 static MCTargetStreamer *
 createXtensaAsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS,
-                              MCInstPrinter *InstPrint, bool isVerboseAsm) {
+                              MCInstPrinter *InstPrint) {
   return new XtensaTargetAsmStreamer(S, OS);
 }
 

From e9c851484bfeb270368bc71e9d7a4bec986f91e7 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 10:33:50 -0700
Subject: [PATCH 342/486] [MC] Remove unnecessary isVerboseAsm from
 createAsmTargetStreamer

---
 llvm/include/llvm/MC/TargetRegistry.h        | 3 +--
 llvm/lib/MC/TargetRegistry.cpp               | 2 +-
 llvm/tools/llvm-exegesis/lib/SnippetFile.cpp | 3 +--
 llvm/tools/llvm-mca/CodeRegionGenerator.cpp  | 3 +--
 4 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index fdfbfb45bf86d..42d510c17bce3 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -552,8 +552,7 @@ class Target {
 
   MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
                                             formatted_raw_ostream &OS,
-                                            MCInstPrinter *InstPrint,
-                                            bool IsVerboseAsm) const {
+                                            MCInstPrinter *InstPrint) const {
     if (AsmTargetStreamerCtorFn)
       return AsmTargetStreamerCtorFn(S, OS, InstPrint);
     return nullptr;
diff --git a/llvm/lib/MC/TargetRegistry.cpp b/llvm/lib/MC/TargetRegistry.cpp
index b6b1a9b2eda9e..3be6f1d463499 100644
--- a/llvm/lib/MC/TargetRegistry.cpp
+++ b/llvm/lib/MC/TargetRegistry.cpp
@@ -95,7 +95,7 @@ MCStreamer *Target::createAsmStreamer(MCContext &Ctx,
   formatted_raw_ostream &OSRef = *OS;
   MCStreamer *S = llvm::createAsmStreamer(Ctx, std::move(OS), IP,
                                           std::move(CE), std::move(TAB));
-  createAsmTargetStreamer(*S, OSRef, IP, /*ignore=*/false);
+  createAsmTargetStreamer(*S, OSRef, IP);
   return S;
 }
 
diff --git a/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp b/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp
index 431d99c72b808..08da04cc62d9c 100644
--- a/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp
+++ b/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp
@@ -260,8 +260,7 @@ Expected<std::vector<BenchmarkCode>> readSnippets(const LLVMState &State,
           *TM.getMCAsmInfo(), *TM.getMCInstrInfo(), *TM.getMCRegisterInfo()));
   // The following call will take care of calling Streamer.setTargetStreamer.
   TM.getTarget().createAsmTargetStreamer(Streamer, InstPrinterOStream,
-                                         InstPrinter.get(),
-                                         TM.Options.MCOptions.AsmVerbose);
+                                         InstPrinter.get());
   if (!Streamer.getTargetStreamer())
     return make_error<Failure>("cannot create target asm streamer");
 
diff --git a/llvm/tools/llvm-mca/CodeRegionGenerator.cpp b/llvm/tools/llvm-mca/CodeRegionGenerator.cpp
index 863766cd777d0..d9b8c6edb0ec9 100644
--- a/llvm/tools/llvm-mca/CodeRegionGenerator.cpp
+++ b/llvm/tools/llvm-mca/CodeRegionGenerator.cpp
@@ -41,8 +41,7 @@ Expected<const CodeRegions &> AsmCodeRegionGenerator::parseCodeRegions(
   // doesn't show up in the llvm-mca output.
   raw_ostream &OSRef = nulls();
   formatted_raw_ostream FOSRef(OSRef);
-  TheTarget.createAsmTargetStreamer(*Str, FOSRef, IP.get(),
-                                    /*IsVerboseAsm=*/true);
+  TheTarget.createAsmTargetStreamer(*Str, FOSRef, IP.get());
 
   // Create a MCAsmParser and setup the lexer to recognize llvm-mca ASM
   // comments.

From 7f017f0ab4fa9c46ca4c07320a61796b1ca64a63 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 10:49:25 -0700
Subject: [PATCH 343/486] [MC] Drop unnecessary MCSymbol::setExternal calls for
 ELF

Similar to e4c360a897fe062914519d331e8f1e28b2b1fbfd (2020).
---
 llvm/include/llvm/MC/MCSymbol.h                               | 1 +
 llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp   | 1 -
 llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp                  | 1 -
 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp  | 4 +---
 llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp | 4 +---
 5 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/llvm/include/llvm/MC/MCSymbol.h b/llvm/include/llvm/MC/MCSymbol.h
index 5f06249b479eb..e0ba687feb080 100644
--- a/llvm/include/llvm/MC/MCSymbol.h
+++ b/llvm/include/llvm/MC/MCSymbol.h
@@ -403,6 +403,7 @@ class MCSymbol {
     return Fragment;
   }
 
+  // For ELF, use MCSymbolELF::setBinding instead.
   bool isExternal() const { return IsExternal; }
   void setExternal(bool Value) const { IsExternal = Value; }
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index 8754e551727ea..27751633017ca 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -276,7 +276,6 @@ class AArch64ELFStreamer : public MCELFStreamer {
     emitLabel(Symbol);
     Symbol->setType(ELF::STT_NOTYPE);
     Symbol->setBinding(ELF::STB_LOCAL);
-    Symbol->setExternal(false);
   }
 
   int64_t MappingSymbolCounter;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index c24d39b9e5fdd..45ec38cfb4dbc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -96,7 +96,6 @@ bool AMDGPUMCInstLower::lowerOperand(const MachineOperand &MO,
   }
   case MachineOperand::MO_ExternalSymbol: {
     MCSymbol *Sym = Ctx.getOrCreateSymbol(StringRef(MO.getSymbolName()));
-    Sym->setExternal(true);
     const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(Sym, Ctx);
     MCOp = MCOperand::createExpr(Expr);
     return true;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index 531031b580347..edc645103135b 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -828,10 +828,8 @@ void AMDGPUTargetELFStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
   MCSymbolELF *SymbolELF = cast<MCSymbolELF>(Symbol);
   SymbolELF->setType(ELF::STT_OBJECT);
 
-  if (!SymbolELF->isBindingSet()) {
+  if (!SymbolELF->isBindingSet())
     SymbolELF->setBinding(ELF::STB_GLOBAL);
-    SymbolELF->setExternal(true);
-  }
 
   if (SymbolELF->declareCommon(Size, Alignment, true)) {
     report_fatal_error("Symbol: " + Symbol->getName() +
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
index 22c82bf5ac1b4..ddc6898d53dbf 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
@@ -97,10 +97,8 @@ void HexagonMCELFStreamer::HexagonMCEmitCommonSymbol(MCSymbol *Symbol,
   StringRef sbss[4] = {".sbss.1", ".sbss.2", ".sbss.4", ".sbss.8"};
 
   auto ELFSymbol = cast<MCSymbolELF>(Symbol);
-  if (!ELFSymbol->isBindingSet()) {
+  if (!ELFSymbol->isBindingSet())
     ELFSymbol->setBinding(ELF::STB_GLOBAL);
-    ELFSymbol->setExternal(true);
-  }
 
   ELFSymbol->setType(ELF::STT_OBJECT);
 

From e299b163c78d34cf6fb90f9d928291419b5dcaaf Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 10:58:20 -0700
Subject: [PATCH 344/486] [MC] Move isPrivateExtern to MCSymbolMachO

---
 llvm/include/llvm/MC/MCSymbol.h      | 6 ++----
 llvm/include/llvm/MC/MCSymbolMachO.h | 3 +++
 llvm/lib/MC/MCMachOStreamer.cpp      | 5 +++--
 llvm/lib/MC/MachObjectWriter.cpp     | 2 +-
 4 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/MC/MCSymbol.h b/llvm/include/llvm/MC/MCSymbol.h
index e0ba687feb080..6fd5c02d04531 100644
--- a/llvm/include/llvm/MC/MCSymbol.h
+++ b/llvm/include/llvm/MC/MCSymbol.h
@@ -99,7 +99,7 @@ class MCSymbol {
   /// uses binding instead of this bit.
   mutable unsigned IsExternal : 1;
 
-  /// This symbol is private extern.
+  /// Mach-O specific: This symbol is private extern.
   mutable unsigned IsPrivateExtern : 1;
 
   /// This symbol is weak external.
@@ -407,9 +407,7 @@ class MCSymbol {
   bool isExternal() const { return IsExternal; }
   void setExternal(bool Value) const { IsExternal = Value; }
 
-  bool isPrivateExtern() const { return IsPrivateExtern; }
-  void setPrivateExtern(bool Value) { IsPrivateExtern = Value; }
-
+  // COFF-specific
   bool isWeakExternal() const { return IsWeakExternal; }
 
   /// print - Print the value to the stream \p OS.
diff --git a/llvm/include/llvm/MC/MCSymbolMachO.h b/llvm/include/llvm/MC/MCSymbolMachO.h
index 730fbea0059a6..97c9abfcf5f2c 100644
--- a/llvm/include/llvm/MC/MCSymbolMachO.h
+++ b/llvm/include/llvm/MC/MCSymbolMachO.h
@@ -46,6 +46,9 @@ class MCSymbolMachO : public MCSymbol {
   MCSymbolMachO(const MCSymbolTableEntry *Name, bool isTemporary)
       : MCSymbol(SymbolKindMachO, Name, isTemporary) {}
 
+  bool isPrivateExtern() const { return IsPrivateExtern; }
+  void setPrivateExtern(bool Value) { IsPrivateExtern = Value; }
+
   // Reference type methods.
 
   void clearReferenceType() const {
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index 5231d10626f85..cb2e4e70ff395 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -151,12 +151,13 @@ void MCMachOStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
 
 void MCMachOStreamer::emitEHSymAttributes(const MCSymbol *Symbol,
                                           MCSymbol *EHSymbol) {
+  auto *Sym = cast<MCSymbolMachO>(Symbol);
   getAssembler().registerSymbol(*Symbol);
   if (Symbol->isExternal())
     emitSymbolAttribute(EHSymbol, MCSA_Global);
-  if (cast<MCSymbolMachO>(Symbol)->isWeakDefinition())
+  if (Sym->isWeakDefinition())
     emitSymbolAttribute(EHSymbol, MCSA_WeakDefinition);
-  if (Symbol->isPrivateExtern())
+  if (Sym->isPrivateExtern())
     emitSymbolAttribute(EHSymbol, MCSA_PrivateExtern);
 }
 
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index e58e095252d05..f5435c6f3dd16 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -376,7 +376,7 @@ const MCSymbol &MachObjectWriter::findAliasedSymbol(const MCSymbol &Sym) const {
 
 void MachObjectWriter::writeNlist(MachSymbolData &MSD, const MCAssembler &Asm) {
   const MCSymbol *Symbol = MSD.Symbol;
-  const MCSymbol &Data = *Symbol;
+  const auto &Data = cast<MCSymbolMachO>(*Symbol);
   const MCSymbol *AliasedSymbol = &findAliasedSymbol(*Symbol);
   uint8_t SectionIndex = MSD.SectionIndex;
   uint8_t Type = 0;

From 6717dc5c47632372e07365e2453b46569bf24e28 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 11:17:19 -0700
Subject: [PATCH 345/486] *AsmBackend.cpp: Include StringSwitch.h

They currently get the header from MCLinkerOptimizationHint.h, which
will be removed from MCAssembler.h.
---
 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp | 1 +
 llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp       | 1 +
 llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp     | 1 +
 3 files changed, 3 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
index 68774c12516ca..c2d2ca0f90f93 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
@@ -10,6 +10,7 @@
 #include "MCTargetDesc/AMDGPUFixupKinds.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
index d520880d73bbd..0d29912bee264 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
@@ -13,6 +13,7 @@
 #include "MCTargetDesc/AVRAsmBackend.h"
 #include "MCTargetDesc/AVRFixupKinds.h"
 #include "MCTargetDesc/AVRMCTargetDesc.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
index fc95b61fd4df5..f8172e576ce4c 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
@@ -17,6 +17,7 @@
 #include "MCTargetDesc/MipsMCExpr.h"
 #include "MCTargetDesc/MipsMCTargetDesc.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"

From a2af375556486d8027d229f3fae956af8371aa86 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 11:19:51 -0700
Subject: [PATCH 346/486] [MC] Move LOHContainer to MachObjectwriter

---
 llvm/include/llvm/MC/MCAssembler.h        | 13 -------------
 llvm/include/llvm/MC/MCMachObjectWriter.h |  5 +++++
 llvm/lib/MC/MCAssembler.cpp               |  2 --
 llvm/lib/MC/MCMachOStreamer.cpp           |  2 +-
 llvm/lib/MC/MachObjectWriter.cpp          |  5 +++--
 5 files changed, 9 insertions(+), 18 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 4b08d50de9e22..93892ce0e832e 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -18,7 +18,6 @@
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCDwarf.h"
-#include "llvm/MC/MCLinkerOptimizationHint.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/SMLoc.h"
 #include "llvm/Support/VersionTuple.h"
@@ -122,10 +121,6 @@ class MCAssembler {
   // which flags to be set.
   unsigned ELFHeaderEFlags = 0;
 
-  /// Used to communicate Linker Optimization Hint information between
-  /// the Streamer and the .o writer
-  MCLOHContainer LOHContainer;
-
   VersionInfoType VersionInfo;
   VersionInfoType DarwinTargetVariantVersionInfo;
 
@@ -341,14 +336,6 @@ class MCAssembler {
     return LinkerOptions;
   }
 
-  // FIXME: This is a total hack, this should not be here. Once things are
-  // factored so that the streamer has direct access to the .o writer, it can
-  // disappear.
-  MCLOHContainer &getLOHContainer() { return LOHContainer; }
-  const MCLOHContainer &getLOHContainer() const {
-    return const_cast<MCAssembler *>(this)->getLOHContainer();
-  }
-
   struct CGProfileEntry {
     const MCSymbolRefExpr *From;
     const MCSymbolRefExpr *To;
diff --git a/llvm/include/llvm/MC/MCMachObjectWriter.h b/llvm/include/llvm/MC/MCMachObjectWriter.h
index c26c75e98844c..60e3e28819724 100644
--- a/llvm/include/llvm/MC/MCMachObjectWriter.h
+++ b/llvm/include/llvm/MC/MCMachObjectWriter.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCLinkerOptimizationHint.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/StringTableBuilder.h"
@@ -140,6 +141,9 @@ class MachObjectWriter : public MCObjectWriter {
 
   /// @}
 
+  // Used to communicate Linker Optimization Hint information.
+  MCLOHContainer LOHContainer;
+
   MachSymbolData *findSymbolData(const MCSymbol &Sym);
 
   void writeWithPadding(StringRef Str, uint64_t Size);
@@ -177,6 +181,7 @@ class MachObjectWriter : public MCObjectWriter {
     return SectionOrder;
   }
   SectionAddrMap &getSectionAddressMap() { return SectionAddress; }
+  MCLOHContainer &getLOHContainer() { return LOHContainer; }
 
   uint64_t getSectionAddress(const MCSection *Sec) const {
     return SectionAddress.lookup(Sec);
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index c8d12eb5dcf64..ed654674c3854 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -102,7 +102,6 @@ void MCAssembler::reset() {
   ThumbFuncs.clear();
   BundleAlignSize = 0;
   ELFHeaderEFlags = 0;
-  LOHContainer.reset();
   VersionInfo.Major = 0;
   VersionInfo.SDKVersion = VersionTuple();
   DarwinTargetVariantVersionInfo.Major = 0;
@@ -115,7 +114,6 @@ void MCAssembler::reset() {
     getEmitterPtr()->reset();
   if (getWriterPtr())
     getWriterPtr()->reset();
-  getLOHContainer().reset();
 }
 
 bool MCAssembler::registerSection(MCSection &Section) {
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index cb2e4e70ff395..f83e3096f3238 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -118,7 +118,7 @@ class MCMachOStreamer : public MCObjectStreamer {
   }
 
   void emitLOHDirective(MCLOHType Kind, const MCLOHArgs &Args) override {
-    getAssembler().getLOHContainer().addDirective(Kind, Args);
+    getWriter().getLOHContainer().addDirective(Kind, Args);
   }
   void emitCGProfileEntry(const MCSymbolRefExpr *From,
                           const MCSymbolRefExpr *To, uint64_t Count) override {
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index f5435c6f3dd16..3c67f16e3d102 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -55,6 +55,7 @@ void MachObjectWriter::reset() {
   LocalSymbolData.clear();
   ExternalSymbolData.clear();
   UndefinedSymbolData.clear();
+  LOHContainer.reset();
   MCObjectWriter::reset();
 }
 
@@ -839,7 +840,7 @@ uint64_t MachObjectWriter::writeObject(MCAssembler &Asm) {
   }
 
   // Add the loh load command size, if used.
-  uint64_t LOHRawSize = Asm.getLOHContainer().getEmitSize(Asm, *this);
+  uint64_t LOHRawSize = LOHContainer.getEmitSize(Asm, *this);
   uint64_t LOHSize = alignTo(LOHRawSize, is64Bit() ? 8 : 4);
   if (LOHSize) {
     ++NumLoadCommands;
@@ -1063,7 +1064,7 @@ uint64_t MachObjectWriter::writeObject(MCAssembler &Asm) {
 #ifndef NDEBUG
     unsigned Start = W.OS.tell();
 #endif
-    Asm.getLOHContainer().emit(Asm, *this);
+    LOHContainer.emit(Asm, *this);
     // Pad to a multiple of the pointer size.
     W.OS.write_zeros(
         offsetToAlignment(LOHRawSize, is64Bit() ? Align(8) : Align(4)));

From ecaacd14c35996ad6ee53da6af6c9f4cf36d9110 Mon Sep 17 00:00:00 2001
From: Utkarsh Saxena <usx@google.com>
Date: Sun, 21 Jul 2024 20:55:34 +0200
Subject: [PATCH 347/486] Reapply "Add source file name for template
 instantiations in -ftime-trace" (#99757)

Reverts https://github.com/llvm/llvm-project/pull/99731

Remove accidentally added temporary file.
Also, fix the uninitialized read of line number.
---
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/Driver/Options.td         |   4 +
 .../include/clang/Frontend/FrontendOptions.h  |   8 +-
 clang/lib/Driver/ToolChains/Clang.cpp         |   1 +
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  11 +-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |  11 +-
 clang/test/Driver/ftime-trace-sections.cpp    |   2 +-
 clang/test/Driver/ftime-trace.cpp             |  39 +++---
 clang/tools/driver/cc1_main.cpp               |   3 +-
 clang/unittests/Support/TimeProfilerTest.cpp  | 123 ++++++++++++++----
 llvm/include/llvm/Support/TimeProfiler.h      |  23 +++-
 llvm/lib/Support/TimeProfiler.cpp             |  61 +++++++--
 12 files changed, 225 insertions(+), 64 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 4638b91b48f95..4ab6bd9de8ea9 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -752,6 +752,9 @@ Improvements to Clang's time-trace
 - Clang now specifies that using ``auto`` in a lambda parameter is a C++14 extension when
   appropriate. (`#46059: <https://github.com/llvm/llvm-project/issues/46059>`_).
 
+- Clang now adds source file infomation for template instantiations as ``event["args"]["filename"]``. This
+  added behind an option ``-ftime-trace-verbose``. This is expected to increase the size of trace by 2-3 times.
+
 Improvements to Coverage Mapping
 --------------------------------
 
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8707e71f2a319..9c6cebd77ff0a 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3998,6 +3998,10 @@ def ftime_trace_granularity_EQ : Joined<["-"], "ftime-trace-granularity=">, Grou
   HelpText<"Minimum time granularity (in microseconds) traced by time profiler">,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
   MarshallingInfoInt<FrontendOpts<"TimeTraceGranularity">, "500u">;
+def ftime_trace_verbose : Joined<["-"], "ftime-trace-verbose">, Group<f_Group>,
+  HelpText<"Make time trace capture verbose event details (e.g. source filenames). This can increase the size of the output by 2-3 times">,
+  Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
+  MarshallingInfoFlag<FrontendOpts<"TimeTraceVerbose">>;
 def ftime_trace_EQ : Joined<["-"], "ftime-trace=">, Group<f_Group>,
   HelpText<"Similar to -ftime-trace. Specify the JSON file or a directory which will contain the JSON file">,
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 5e5034fe01eb5..8241925c98476 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -580,6 +580,11 @@ class FrontendOptions {
   /// Minimum time granularity (in microseconds) traced by time profiler.
   unsigned TimeTraceGranularity;
 
+  /// Make time trace capture verbose event details (e.g. source filenames).
+  /// This can increase the size of the output by 2-3 times.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned TimeTraceVerbose : 1;
+
   /// Path which stores the output files for -ftime-trace
   std::string TimeTracePath;
 
@@ -601,7 +606,8 @@ class FrontendOptions {
         EmitSymbolGraph(false), EmitExtensionSymbolGraphs(false),
         EmitSymbolGraphSymbolLabelsForTesting(false),
         EmitPrettySymbolGraphs(false), GenReducedBMI(false),
-        UseClangIRPipeline(false), TimeTraceGranularity(500) {}
+        UseClangIRPipeline(false), TimeTraceGranularity(500),
+        TimeTraceVerbose(false) {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file
   /// extension. For example, "c" would return Language::C.
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 71cdaa10416f4..364e55a628c34 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6759,6 +6759,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (const char *Name = C.getTimeTraceFile(&JA)) {
     CmdArgs.push_back(Args.MakeArgString("-ftime-trace=" + Twine(Name)));
     Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_granularity_EQ);
+    Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_verbose);
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_ftrapv_handler_EQ)) {
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index a7bc6749c5852..725b62db5e80a 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -3426,11 +3426,16 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
     return true;
 
   llvm::TimeTraceScope TimeScope("InstantiateClass", [&]() {
-    std::string Name;
-    llvm::raw_string_ostream OS(Name);
+    llvm::TimeTraceMetadata M;
+    llvm::raw_string_ostream OS(M.Detail);
     Instantiation->getNameForDiagnostic(OS, getPrintingPolicy(),
                                         /*Qualified=*/true);
-    return Name;
+    if (llvm::isTimeTraceVerbose()) {
+      auto Loc = SourceMgr.getExpansionLoc(Instantiation->getLocation());
+      M.File = SourceMgr.getFilename(Loc);
+      M.Line = SourceMgr.getExpansionLineNumber(Loc);
+    }
+    return M;
   });
 
   Pattern = PatternDef;
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 97161febc15f7..a12d2eff1d2c8 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -4966,11 +4966,16 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   }
 
   llvm::TimeTraceScope TimeScope("InstantiateFunction", [&]() {
-    std::string Name;
-    llvm::raw_string_ostream OS(Name);
+    llvm::TimeTraceMetadata M;
+    llvm::raw_string_ostream OS(M.Detail);
     Function->getNameForDiagnostic(OS, getPrintingPolicy(),
                                    /*Qualified=*/true);
-    return Name;
+    if (llvm::isTimeTraceVerbose()) {
+      auto Loc = SourceMgr.getExpansionLoc(Function->getLocation());
+      M.File = SourceMgr.getFilename(Loc);
+      M.Line = SourceMgr.getExpansionLineNumber(Loc);
+    }
+    return M;
   });
 
   // If we're performing recursive template instantiation, create our own
diff --git a/clang/test/Driver/ftime-trace-sections.cpp b/clang/test/Driver/ftime-trace-sections.cpp
index 0c16052bc0c3a..da7109b9d81a6 100644
--- a/clang/test/Driver/ftime-trace-sections.cpp
+++ b/clang/test/Driver/ftime-trace-sections.cpp
@@ -1,5 +1,5 @@
 // RUN: rm -rf %t && mkdir %t && cd %t
-// RUN: %clangxx -S -ftime-trace -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: %python %S/ftime-trace-sections.py < out.json
 
 template <typename T>
diff --git a/clang/test/Driver/ftime-trace.cpp b/clang/test/Driver/ftime-trace.cpp
index 5fe63de915a71..60c5885704b58 100644
--- a/clang/test/Driver/ftime-trace.cpp
+++ b/clang/test/Driver/ftime-trace.cpp
@@ -1,18 +1,18 @@
 // RUN: rm -rf %t && mkdir -p %t && cd %t
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=new-name.json -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=new-name.json -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat new-name.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
 // RUN: mkdir dir1 dir2
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir1 -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir1 -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat dir1/out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
-// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir2/ -ftime-trace-granularity=0 -o out %s
+// RUN: %clangxx -S -no-canonical-prefixes -ftime-trace=dir2/ -ftime-trace-granularity=0 -ftime-trace-verbose -o out %s
 // RUN: cat dir2/out.json \
 // RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 // RUN:   | FileCheck %s
@@ -34,32 +34,33 @@
 // RUN: mkdir d e f && cp %s d/a.cpp && touch d/b.c
 
 /// TODO: Support -fno-integrated-as.
-// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -fintegrated-as d/a.cpp -o e/a.o 2>&1 | FileCheck %s --check-prefix=COMPILE1
-// COMPILE1: -cc1{{.*}} "-ftime-trace=e/a.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose -fintegrated-as d/a.cpp -o e/a.o 2>&1 | FileCheck %s --check-prefix=COMPILE1
+// COMPILE1: -cc1{{.*}} "-ftime-trace=e/a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
-// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=COMPILE2
-// COMPILE2: -cc1{{.*}} "-ftime-trace=f/a.json" "-ftime-trace-granularity=0"
-// COMPILE2: -cc1{{.*}} "-ftime-trace=f/b.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -c -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=COMPILE2
+// COMPILE2: -cc1{{.*}} "-ftime-trace=f/a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// COMPILE2: -cc1{{.*}} "-ftime-trace=f/b.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
 /// -o specifies the link output. Create ${output}-${basename}.json.
-// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -o e/x 2>&1 | FileCheck %s --check-prefix=LINK1
-// LINK1: -cc1{{.*}} "-ftime-trace=e/x-a.json" "-ftime-trace-granularity=0"
-// LINK1: -cc1{{.*}} "-ftime-trace=e/x-b.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o e/x 2>&1 | FileCheck %s --check-prefix=LINK1
+// LINK1: -cc1{{.*}} "-ftime-trace=e/x-a.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// LINK1: -cc1{{.*}} "-ftime-trace=e/x-b.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
 /// -dumpdir is f/g, not ending with a path separator. We create f/g${basename}.json.
-// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 d/a.cpp d/b.c -o e/x -dumpdir f/g 2>&1 | FileCheck %s --check-prefix=LINK2
-// LINK2: -cc1{{.*}} "-ftime-trace=f/ga.json" "-ftime-trace-granularity=0"
-// LINK2: -cc1{{.*}} "-ftime-trace=f/gb.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -ftime-trace -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o e/x -dumpdir f/g 2>&1 | FileCheck %s --check-prefix=LINK2
+// LINK2: -cc1{{.*}} "-ftime-trace=f/ga.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// LINK2: -cc1{{.*}} "-ftime-trace=f/gb.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
-// RUN: %clang -### -ftime-trace=e -ftime-trace-granularity=0 d/a.cpp d/b.c -o f/x -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=LINK3
-// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}a-{{[^.]*}}.json" "-ftime-trace-granularity=0"
-// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}b-{{[^.]*}}.json" "-ftime-trace-granularity=0"
+// RUN: %clang -### -ftime-trace=e -ftime-trace-granularity=0 -ftime-trace-verbose d/a.cpp d/b.c -o f/x -dumpdir f/ 2>&1 | FileCheck %s --check-prefix=LINK3
+// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}a-{{[^.]*}}.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
+// LINK3: -cc1{{.*}} "-ftime-trace=e{{/|\\\\}}b-{{[^.]*}}.json" "-ftime-trace-granularity=0" "-ftime-trace-verbose"
 
-// RUN: %clang -### -ftime-trace -ftime-trace=e -ftime-trace-granularity=1 -xassembler d/a.cpp 2>&1 | \
+// RUN: %clang -### -ftime-trace -ftime-trace=e -ftime-trace-granularity=1 -ftime-trace-verbose -xassembler d/a.cpp 2>&1 | \
 // RUN:   FileCheck %s --check-prefix=UNUSED
 // UNUSED:      warning: argument unused during compilation: '-ftime-trace'
 // UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace=e'
 // UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace-granularity=1'
+// UNUSED-NEXT: warning: argument unused during compilation: '-ftime-trace-verbose'
 // UNUSED-NOT:  warning:
 
 template <typename T>
diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp
index c2ccb47a15bc8..f5e5fad36573e 100644
--- a/clang/tools/driver/cc1_main.cpp
+++ b/clang/tools/driver/cc1_main.cpp
@@ -241,7 +241,8 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
 
   if (!Clang->getFrontendOpts().TimeTracePath.empty()) {
     llvm::timeTraceProfilerInitialize(
-        Clang->getFrontendOpts().TimeTraceGranularity, Argv0);
+        Clang->getFrontendOpts().TimeTraceGranularity, Argv0,
+        Clang->getFrontendOpts().TimeTraceVerbose);
   }
   // --print-supported-cpus takes priority over the actual compilation.
   if (Clang->getFrontendOpts().PrintSupportedCPUs)
diff --git a/clang/unittests/Support/TimeProfilerTest.cpp b/clang/unittests/Support/TimeProfilerTest.cpp
index 5f3950ff033f1..56d880cffde61 100644
--- a/clang/unittests/Support/TimeProfilerTest.cpp
+++ b/clang/unittests/Support/TimeProfilerTest.cpp
@@ -10,11 +10,15 @@
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Lex/PreprocessorOptions.h"
 
+#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/JSON.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/TimeProfiler.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include <stack>
 
 #include "gtest/gtest.h"
+#include <tuple>
 
 using namespace clang;
 using namespace llvm;
@@ -23,7 +27,8 @@ namespace {
 
 // Should be called before testing.
 void setupProfiler() {
-  timeTraceProfilerInitialize(/*TimeTraceGranularity=*/0, "test");
+  timeTraceProfilerInitialize(/*TimeTraceGranularity=*/0, "test",
+                              /*TimeTraceVerbose=*/true);
 }
 
 // Should be called after `compileFromString()`.
@@ -38,14 +43,24 @@ std::string teardownProfiler() {
 
 // Returns true if code compiles successfully.
 // We only parse AST here. This is enough for constexpr evaluation.
-bool compileFromString(StringRef Code, StringRef Standard, StringRef FileName) {
+bool compileFromString(StringRef Code, StringRef Standard, StringRef File,
+                       llvm::StringMap<std::string> Headers = {}) {
   CompilerInstance Compiler;
   Compiler.createDiagnostics();
 
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> FS(
+      new llvm::vfs::InMemoryFileSystem());
+  FS->addFile(File, 0, MemoryBuffer::getMemBuffer(Code));
+  for (const auto &Header : Headers) {
+    FS->addFile(Header.getKey(), 0,
+                MemoryBuffer::getMemBuffer(Header.getValue()));
+  }
+  llvm::IntrusiveRefCntPtr<FileManager> Files(
+      new FileManager(FileSystemOptions(), FS));
+  Compiler.setFileManager(Files.get());
+
   auto Invocation = std::make_shared<CompilerInvocation>();
-  Invocation->getPreprocessorOpts().addRemappedFile(
-      FileName, MemoryBuffer::getMemBuffer(Code).release());
-  const char *Args[] = {Standard.data(), FileName.data()};
+  std::vector<const char *> Args = {Standard.data(), File.data()};
   CompilerInvocation::CreateFromArgs(*Invocation, Args,
                                      Compiler.getDiagnostics());
   Compiler.setInvocation(std::move(Invocation));
@@ -60,13 +75,28 @@ bool compileFromString(StringRef Code, StringRef Standard, StringRef FileName) {
   return Compiler.ExecuteAction(Action);
 }
 
+std::string GetMetadata(json::Object *Event) {
+  std::string Metadata;
+  llvm::raw_string_ostream OS(Metadata);
+  if (json::Object *Args = Event->getObject("args")) {
+    if (auto Detail = Args->getString("detail"))
+      OS << Detail;
+    // Use only filename to not include os-specific path separators.
+    if (auto File = Args->getString("file"))
+      OS << ", " << llvm::sys::path::filename(*File);
+    if (auto Line = Args->getInteger("line"))
+      OS << ":" << *Line;
+  }
+  return Metadata;
+}
+
 // Returns pretty-printed trace graph.
 std::string buildTraceGraph(StringRef Json) {
   struct EventRecord {
     int64_t TimestampBegin;
     int64_t TimestampEnd;
-    StringRef Name;
-    StringRef Detail;
+    std::string Name;
+    std::string Metadata;
   };
   std::vector<EventRecord> Events;
 
@@ -81,10 +111,13 @@ std::string buildTraceGraph(StringRef Json) {
     int64_t TimestampBegin = TraceEventObj->getInteger("ts").value_or(0);
     int64_t TimestampEnd =
         TimestampBegin + TraceEventObj->getInteger("dur").value_or(0);
-    StringRef Name = TraceEventObj->getString("name").value_or("");
-    StringRef Detail = "";
-    if (json::Object *Args = TraceEventObj->getObject("args"))
-      Detail = Args->getString("detail").value_or("");
+    std::string Name = TraceEventObj->getString("name").value_or("").str();
+    std::string Metadata = GetMetadata(TraceEventObj);
+
+    // Source events are asynchronous events and may not perfectly nest the
+    // synchronous events. Skip testing them.
+    if (Name == "Source")
+      continue;
 
     // This is a "summary" event, like "Total PerformPendingInstantiations",
     // skip it
@@ -92,7 +125,7 @@ std::string buildTraceGraph(StringRef Json) {
       continue;
 
     Events.emplace_back(
-        EventRecord{TimestampBegin, TimestampEnd, Name, Detail});
+        EventRecord{TimestampBegin, TimestampEnd, Name, Metadata});
   }
 
   // There can be nested events that are very fast, for example:
@@ -132,9 +165,9 @@ std::string buildTraceGraph(StringRef Json) {
       Stream << "| ";
     }
     Stream.write(Event.Name.data(), Event.Name.size());
-    if (!Event.Detail.empty()) {
+    if (!Event.Metadata.empty()) {
       Stream << " (";
-      Stream.write(Event.Detail.data(), Event.Detail.size());
+      Stream.write(Event.Metadata.data(), Event.Metadata.size());
       Stream << ")";
     }
     Stream << "\n";
@@ -145,7 +178,7 @@ std::string buildTraceGraph(StringRef Json) {
 } // namespace
 
 TEST(TimeProfilerTest, ConstantEvaluationCxx20) {
-  constexpr StringRef Code = R"(
+  std::string Code = R"(
 void print(double value);
 
 namespace slow_namespace {
@@ -175,8 +208,7 @@ constexpr int slow_init_list[] = {1, 1, 2, 3, 5, 8, 13, 21}; // 25th line
   setupProfiler();
   ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc"));
   std::string Json = teardownProfiler();
-  std::string TraceGraph = buildTraceGraph(Json);
-  ASSERT_TRUE(TraceGraph == R"(
+  ASSERT_EQ(R"(
 Frontend
 | ParseDeclarationOrFunctionDefinition (test.cc:2:1)
 | ParseDeclarationOrFunctionDefinition (test.cc:6:1)
@@ -202,14 +234,54 @@ Frontend
 | ParseDeclarationOrFunctionDefinition (test.cc:25:1)
 | | EvaluateAsInitializer (slow_init_list)
 | PerformPendingInstantiations
-)");
+)",
+            buildTraceGraph(Json));
+}
+
+TEST(TimeProfilerTest, TemplateInstantiations) {
+  std::string B_H = R"(
+    template <typename T>
+    T fooB(T t) {
+      return T();
+    }
 
-  // NOTE: If this test is failing, run this test with
-  // `llvm::errs() << TraceGraph;` and change the assert above.
+    #define MacroTemp(x) template <typename T> void foo##x(T) { T(); }
+  )";
+
+  std::string A_H = R"(
+    #include "b.h"
+
+    MacroTemp(MTA)
+
+    template <typename T>
+    void fooA(T t) { fooB(t); fooMTA(t); }
+  )";
+  std::string Code = R"(
+    #include "a.h"
+    void user() { fooA(0); }
+  )";
+
+  setupProfiler();
+  ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc",
+                                /*Headers=*/{{"a.h", A_H}, {"b.h", B_H}}));
+  std::string Json = teardownProfiler();
+  ASSERT_EQ(R"(
+Frontend
+| ParseFunctionDefinition (fooB)
+| ParseFunctionDefinition (fooMTA)
+| ParseFunctionDefinition (fooA)
+| ParseDeclarationOrFunctionDefinition (test.cc:3:5)
+| | ParseFunctionDefinition (user)
+| PerformPendingInstantiations
+| | InstantiateFunction (fooA<int>, a.h:7)
+| | | InstantiateFunction (fooB<int>, b.h:3)
+| | | InstantiateFunction (fooMTA<int>, a.h:4)
+)",
+            buildTraceGraph(Json));
 }
 
 TEST(TimeProfilerTest, ConstantEvaluationC99) {
-  constexpr StringRef Code = R"(
+  std::string Code = R"(
 struct {
   short quantval[4]; // 3rd line
 } value;
@@ -218,15 +290,12 @@ struct {
   setupProfiler();
   ASSERT_TRUE(compileFromString(Code, "-std=c99", "test.c"));
   std::string Json = teardownProfiler();
-  std::string TraceGraph = buildTraceGraph(Json);
-  ASSERT_TRUE(TraceGraph == R"(
+  ASSERT_EQ(R"(
 Frontend
 | ParseDeclarationOrFunctionDefinition (test.c:2:1)
 | | isIntegerConstantExpr (<test.c:3:18>)
 | | EvaluateKnownConstIntCheckOverflow (<test.c:3:18>)
 | PerformPendingInstantiations
-)");
-
-  // NOTE: If this test is failing, run this test with
-  // `llvm::errs() << TraceGraph;` and change the assert above.
+)",
+            buildTraceGraph(Json));
 }
diff --git a/llvm/include/llvm/Support/TimeProfiler.h b/llvm/include/llvm/Support/TimeProfiler.h
index 31f7df10916db..9e2ba31991f54 100644
--- a/llvm/include/llvm/Support/TimeProfiler.h
+++ b/llvm/include/llvm/Support/TimeProfiler.h
@@ -83,16 +83,28 @@ namespace llvm {
 
 class raw_pwrite_stream;
 
+struct TimeTraceMetadata {
+  std::string Detail;
+  // Source file and line number information for the event.
+  std::string File;
+  int Line = 0;
+
+  bool isEmpty() const { return Detail.empty() && File.empty(); }
+};
+
 struct TimeTraceProfiler;
 TimeTraceProfiler *getTimeTraceProfilerInstance();
 
+bool isTimeTraceVerbose();
+
 struct TimeTraceProfilerEntry;
 
 /// Initialize the time trace profiler.
 /// This sets up the global \p TimeTraceProfilerInstance
 /// variable to be the profiler instance.
 void timeTraceProfilerInitialize(unsigned TimeTraceGranularity,
-                                 StringRef ProcName);
+                                 StringRef ProcName,
+                                 bool TimeTraceVerbose = false);
 
 /// Cleanup the time trace profiler, if it was initialized.
 void timeTraceProfilerCleanup();
@@ -128,6 +140,10 @@ TimeTraceProfilerEntry *
 timeTraceProfilerBegin(StringRef Name,
                        llvm::function_ref<std::string()> Detail);
 
+TimeTraceProfilerEntry *
+timeTraceProfilerBegin(StringRef Name,
+                       llvm::function_ref<TimeTraceMetadata()> MetaData);
+
 /// Manually begin a time section, with the given \p Name and \p Detail.
 /// This starts Async Events having \p Name as a category which is shown
 /// separately from other traces. See
@@ -164,6 +180,11 @@ class TimeTraceScope {
     if (getTimeTraceProfilerInstance() != nullptr)
       Entry = timeTraceProfilerBegin(Name, Detail);
   }
+  TimeTraceScope(StringRef Name,
+                 llvm::function_ref<TimeTraceMetadata()> Metadata) {
+    if (getTimeTraceProfilerInstance() != nullptr)
+      Entry = timeTraceProfilerBegin(Name, Metadata);
+  }
   ~TimeTraceScope() {
     if (getTimeTraceProfilerInstance() != nullptr)
       timeTraceProfilerEnd(Entry);
diff --git a/llvm/lib/Support/TimeProfiler.cpp b/llvm/lib/Support/TimeProfiler.cpp
index 9612db7d30f98..c2014028ddadc 100644
--- a/llvm/lib/Support/TimeProfiler.cpp
+++ b/llvm/lib/Support/TimeProfiler.cpp
@@ -73,12 +73,20 @@ struct llvm::TimeTraceProfilerEntry {
   const TimePointType Start;
   TimePointType End;
   const std::string Name;
-  const std::string Detail;
+  TimeTraceMetadata Metadata;
+
   const bool AsyncEvent = false;
   TimeTraceProfilerEntry(TimePointType &&S, TimePointType &&E, std::string &&N,
                          std::string &&Dt, bool Ae)
+      : Start(std::move(S)), End(std::move(E)), Name(std::move(N)), Metadata(),
+        AsyncEvent(Ae) {
+    Metadata.Detail = std::move(Dt);
+  }
+
+  TimeTraceProfilerEntry(TimePointType &&S, TimePointType &&E, std::string &&N,
+                         TimeTraceMetadata &&Mt, bool Ae)
       : Start(std::move(S)), End(std::move(E)), Name(std::move(N)),
-        Detail(std::move(Dt)), AsyncEvent(Ae) {}
+        Metadata(std::move(Mt)), AsyncEvent(Ae) {}
 
   // Calculate timings for FlameGraph. Cast time points to microsecond precision
   // rather than casting duration. This avoids truncation issues causing inner
@@ -97,10 +105,12 @@ struct llvm::TimeTraceProfilerEntry {
 };
 
 struct llvm::TimeTraceProfiler {
-  TimeTraceProfiler(unsigned TimeTraceGranularity = 0, StringRef ProcName = "")
+  TimeTraceProfiler(unsigned TimeTraceGranularity = 0, StringRef ProcName = "",
+                    bool TimeTraceVerbose = false)
       : BeginningOfTime(system_clock::now()), StartTime(ClockType::now()),
         ProcName(ProcName), Pid(sys::Process::getProcessId()),
-        Tid(llvm::get_threadid()), TimeTraceGranularity(TimeTraceGranularity) {
+        Tid(llvm::get_threadid()), TimeTraceGranularity(TimeTraceGranularity),
+        TimeTraceVerbose(TimeTraceVerbose) {
     llvm::get_thread_name(ThreadName);
   }
 
@@ -113,6 +123,15 @@ struct llvm::TimeTraceProfiler {
     return Stack.back().get();
   }
 
+  TimeTraceProfilerEntry *
+  begin(std::string Name, llvm::function_ref<TimeTraceMetadata()> Metadata,
+        bool AsyncEvent = false) {
+    Stack.emplace_back(std::make_unique<TimeTraceProfilerEntry>(
+        ClockType::now(), TimePointType(), std::move(Name), Metadata(),
+        AsyncEvent));
+    return Stack.back().get();
+  }
+
   void end() {
     assert(!Stack.empty() && "Must call begin() first");
     end(*Stack.back());
@@ -184,8 +203,15 @@ struct llvm::TimeTraceProfiler {
           J.attribute("dur", DurUs);
         }
         J.attribute("name", E.Name);
-        if (!E.Detail.empty()) {
-          J.attributeObject("args", [&] { J.attribute("detail", E.Detail); });
+        if (!E.Metadata.isEmpty()) {
+          J.attributeObject("args", [&] {
+            if (!E.Metadata.Detail.empty())
+              J.attribute("detail", E.Metadata.Detail);
+            if (!E.Metadata.File.empty())
+              J.attribute("file", E.Metadata.File);
+            if (E.Metadata.Line > 0)
+              J.attribute("line", E.Metadata.Line);
+          });
         }
       });
 
@@ -307,14 +333,25 @@ struct llvm::TimeTraceProfiler {
 
   // Minimum time granularity (in microseconds)
   const unsigned TimeTraceGranularity;
+
+  // Make time trace capture verbose event details (e.g. source filenames). This
+  // can increase the size of the output by 2-3 times.
+  const bool TimeTraceVerbose;
 };
 
+bool llvm::isTimeTraceVerbose() {
+  return getTimeTraceProfilerInstance() &&
+         getTimeTraceProfilerInstance()->TimeTraceVerbose;
+}
+
 void llvm::timeTraceProfilerInitialize(unsigned TimeTraceGranularity,
-                                       StringRef ProcName) {
+                                       StringRef ProcName,
+                                       bool TimeTraceVerbose) {
   assert(TimeTraceProfilerInstance == nullptr &&
          "Profiler should not be initialized");
   TimeTraceProfilerInstance = new TimeTraceProfiler(
-      TimeTraceGranularity, llvm::sys::path::filename(ProcName));
+      TimeTraceGranularity, llvm::sys::path::filename(ProcName),
+      TimeTraceVerbose);
 }
 
 // Removes all TimeTraceProfilerInstances.
@@ -381,6 +418,14 @@ llvm::timeTraceProfilerBegin(StringRef Name,
   return nullptr;
 }
 
+TimeTraceProfilerEntry *
+llvm::timeTraceProfilerBegin(StringRef Name,
+                             llvm::function_ref<TimeTraceMetadata()> Metadata) {
+  if (TimeTraceProfilerInstance != nullptr)
+    return TimeTraceProfilerInstance->begin(std::string(Name), Metadata, false);
+  return nullptr;
+}
+
 TimeTraceProfilerEntry *llvm::timeTraceAsyncProfilerBegin(StringRef Name,
                                                           StringRef Detail) {
   if (TimeTraceProfilerInstance != nullptr)

From 6db2465ce74141f378a290cfa3b56eb69dc379cb Mon Sep 17 00:00:00 2001
From: serge-sans-paille <serge.guelton@telecom-bretagne.eu>
Date: Sun, 21 Jul 2024 18:57:56 +0000
Subject: [PATCH 348/486] =?UTF-8?q?[Flang][Runtime]=20Fix=20implicit=20con?=
 =?UTF-8?q?version=20warning=20when=20targeting=2032bit=E2=80=A6=20(#99465?=
 =?UTF-8?q?)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… architecture

On 32 bit systems, TypeParameterValue is 64bit wide while CFI_index_t is
32bit wide.
---
 flang/runtime/derived.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/flang/runtime/derived.cpp b/flang/runtime/derived.cpp
index 0d9e033df4e27..659f54fa344bb 100644
--- a/flang/runtime/derived.cpp
+++ b/flang/runtime/derived.cpp
@@ -23,10 +23,9 @@ static RT_API_ATTRS void GetComponentExtents(SubscriptValue (&extents)[maxRank],
     const typeInfo::Component &comp, const Descriptor &derivedInstance) {
   const typeInfo::Value *bounds{comp.bounds()};
   for (int dim{0}; dim < comp.rank(); ++dim) {
-    SubscriptValue lb{bounds[2 * dim].GetValue(&derivedInstance).value_or(0)};
-    SubscriptValue ub{
-        bounds[2 * dim + 1].GetValue(&derivedInstance).value_or(0)};
-    extents[dim] = ub >= lb ? ub - lb + 1 : 0;
+    auto lb{bounds[2 * dim].GetValue(&derivedInstance).value_or(0)};
+    auto ub{bounds[2 * dim + 1].GetValue(&derivedInstance).value_or(0)};
+    extents[dim] = ub >= lb ? static_cast<SubscriptValue>(ub - lb + 1) : 0;
   }
 }
 

From 9539a7796094ff5fb59d9c685140ea2e214b945c Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 12:04:47 -0700
Subject: [PATCH 349/486] [MC] Export llvm::WinCOFFObjectWriter and access it
 from MCWinCOFFStreamer

Similar to commit 28fcafb50274be2520117eacb0a886adafefe59d (2011) for
MachObjectWriter. MCWinCOFFStreamer can now access WinCOFFObjectWriter
directly without holding object file format specific inforamtion in
MCAssembler (e.g. IncrementalLinkerCompatible).
---
 llvm/include/llvm/MC/MCAssembler.h           |  8 ---
 llvm/include/llvm/MC/MCWinCOFFObjectWriter.h | 31 +++++++++++
 llvm/include/llvm/MC/MCWinCOFFStreamer.h     |  3 ++
 llvm/lib/MC/MCAssembler.cpp                  |  1 -
 llvm/lib/MC/MCWinCOFFStreamer.cpp            |  7 ++-
 llvm/lib/MC/WinCOFFObjectWriter.cpp          | 56 ++++++--------------
 llvm/lib/MC/XCOFFObjectWriter.cpp            |  2 -
 7 files changed, 57 insertions(+), 51 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 93892ce0e832e..5ffe2ca5fabc3 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -84,7 +84,6 @@ class MCAssembler {
   bool HasLayout = false;
   bool RelaxAll = false;
   bool SubsectionsViaSymbols = false;
-  bool IncrementalLinkerCompatible = false;
 
   SectionListType Sections;
 
@@ -298,13 +297,6 @@ class MCAssembler {
   bool getSubsectionsViaSymbols() const { return SubsectionsViaSymbols; }
   void setSubsectionsViaSymbols(bool Value) { SubsectionsViaSymbols = Value; }
 
-  bool isIncrementalLinkerCompatible() const {
-    return IncrementalLinkerCompatible;
-  }
-  void setIncrementalLinkerCompatible(bool Value) {
-    IncrementalLinkerCompatible = Value;
-  }
-
   bool hasLayout() const { return HasLayout; }
   bool getRelaxAll() const { return RelaxAll; }
   void setRelaxAll(bool Value) { RelaxAll = Value; }
diff --git a/llvm/include/llvm/MC/MCWinCOFFObjectWriter.h b/llvm/include/llvm/MC/MCWinCOFFObjectWriter.h
index 307800e73c687..49498871d741d 100644
--- a/llvm/include/llvm/MC/MCWinCOFFObjectWriter.h
+++ b/llvm/include/llvm/MC/MCWinCOFFObjectWriter.h
@@ -43,6 +43,37 @@ class MCWinCOFFObjectTargetWriter : public MCObjectTargetWriter {
   virtual bool recordRelocation(const MCFixup &) const { return true; }
 };
 
+class WinCOFFWriter;
+
+class WinCOFFObjectWriter : public MCObjectWriter {
+  friend class WinCOFFWriter;
+
+  std::unique_ptr<MCWinCOFFObjectTargetWriter> TargetObjectWriter;
+  std::unique_ptr<WinCOFFWriter> ObjWriter, DwoWriter;
+  bool IncrementalLinkerCompatible = false;
+
+public:
+  WinCOFFObjectWriter(std::unique_ptr<MCWinCOFFObjectTargetWriter> MOTW,
+                      raw_pwrite_stream &OS);
+  WinCOFFObjectWriter(std::unique_ptr<MCWinCOFFObjectTargetWriter> MOTW,
+                      raw_pwrite_stream &OS, raw_pwrite_stream &DwoOS);
+
+  // MCObjectWriter interface implementation.
+  void reset() override;
+  void setIncrementalLinkerCompatible(bool Value) {
+    IncrementalLinkerCompatible = Value;
+  }
+  void executePostLayoutBinding(MCAssembler &Asm) override;
+  bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
+                                              const MCSymbol &SymA,
+                                              const MCFragment &FB, bool InSet,
+                                              bool IsPCRel) const override;
+  void recordRelocation(MCAssembler &Asm, const MCFragment *Fragment,
+                        const MCFixup &Fixup, MCValue Target,
+                        uint64_t &FixedValue) override;
+  uint64_t writeObject(MCAssembler &Asm) override;
+};
+
 /// Construct a new Win COFF writer instance.
 ///
 /// \param MOTW - The target specific WinCOFF writer subclass.
diff --git a/llvm/include/llvm/MC/MCWinCOFFStreamer.h b/llvm/include/llvm/MC/MCWinCOFFStreamer.h
index 893a70f74ee3b..5c39d80538944 100644
--- a/llvm/include/llvm/MC/MCWinCOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCWinCOFFStreamer.h
@@ -22,6 +22,7 @@ class MCSection;
 class MCSubtargetInfo;
 class MCSymbol;
 class StringRef;
+class WinCOFFObjectWriter;
 class raw_pwrite_stream;
 
 class MCWinCOFFStreamer : public MCObjectStreamer {
@@ -36,6 +37,8 @@ class MCWinCOFFStreamer : public MCObjectStreamer {
     MCObjectStreamer::reset();
   }
 
+  WinCOFFObjectWriter &getWriter();
+
   /// \name MCStreamer interface
   /// \{
 
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index ed654674c3854..852fa76f84ccc 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -94,7 +94,6 @@ MCAssembler::~MCAssembler() = default;
 void MCAssembler::reset() {
   RelaxAll = false;
   SubsectionsViaSymbols = false;
-  IncrementalLinkerCompatible = false;
   Sections.clear();
   Symbols.clear();
   LinkerOptions.clear();
diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp
index a14d3bcf37f3f..47360a1d26258 100644
--- a/llvm/lib/MC/MCWinCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp
@@ -28,6 +28,7 @@
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSymbolCOFF.h"
 #include "llvm/MC/MCTargetOptions.h"
+#include "llvm/MC/MCWinCOFFObjectWriter.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
@@ -49,7 +50,11 @@ MCWinCOFFStreamer::MCWinCOFFStreamer(MCContext &Context,
       CurSymbol(nullptr) {
   auto *TO = Context.getTargetOptions();
   if (TO && TO->MCIncrementalLinkerCompatible)
-    getAssembler().setIncrementalLinkerCompatible(true);
+    getWriter().setIncrementalLinkerCompatible(true);
+}
+
+WinCOFFObjectWriter &MCWinCOFFStreamer::getWriter() {
+  return static_cast<WinCOFFObjectWriter &>(getAssembler().getWriter());
 }
 
 void MCWinCOFFStreamer::emitInstToData(const MCInst &Inst,
diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp
index 7ba38be7edba9..1747413dc3714 100644
--- a/llvm/lib/MC/WinCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp
@@ -119,10 +119,9 @@ class COFFSection {
 
   SmallVector<COFFSymbol *, 1> OffsetSymbols;
 };
+} // namespace
 
-class WinCOFFObjectWriter;
-
-class WinCOFFWriter {
+class llvm::WinCOFFWriter {
   WinCOFFObjectWriter &OWriter;
   support::endian::Writer W;
 
@@ -196,41 +195,19 @@ class WinCOFFWriter {
   void assignFileOffsets(MCAssembler &Asm);
 };
 
-class WinCOFFObjectWriter : public MCObjectWriter {
-  friend class WinCOFFWriter;
-
-  std::unique_ptr<MCWinCOFFObjectTargetWriter> TargetObjectWriter;
-  std::unique_ptr<WinCOFFWriter> ObjWriter, DwoWriter;
-
-public:
-  WinCOFFObjectWriter(std::unique_ptr<MCWinCOFFObjectTargetWriter> MOTW,
-                      raw_pwrite_stream &OS)
-      : TargetObjectWriter(std::move(MOTW)),
-        ObjWriter(std::make_unique<WinCOFFWriter>(*this, OS,
-                                                  WinCOFFWriter::AllSections)) {
-  }
-  WinCOFFObjectWriter(std::unique_ptr<MCWinCOFFObjectTargetWriter> MOTW,
-                      raw_pwrite_stream &OS, raw_pwrite_stream &DwoOS)
-      : TargetObjectWriter(std::move(MOTW)),
-        ObjWriter(std::make_unique<WinCOFFWriter>(*this, OS,
-                                                  WinCOFFWriter::NonDwoOnly)),
-        DwoWriter(std::make_unique<WinCOFFWriter>(*this, DwoOS,
-                                                  WinCOFFWriter::DwoOnly)) {}
-
-  // MCObjectWriter interface implementation.
-  void reset() override;
-  void executePostLayoutBinding(MCAssembler &Asm) override;
-  bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
-                                              const MCSymbol &SymA,
-                                              const MCFragment &FB, bool InSet,
-                                              bool IsPCRel) const override;
-  void recordRelocation(MCAssembler &Asm, const MCFragment *Fragment,
-                        const MCFixup &Fixup, MCValue Target,
-                        uint64_t &FixedValue) override;
-  uint64_t writeObject(MCAssembler &Asm) override;
-};
-
-} // end anonymous namespace
+WinCOFFObjectWriter::WinCOFFObjectWriter(
+    std::unique_ptr<MCWinCOFFObjectTargetWriter> MOTW, raw_pwrite_stream &OS)
+    : TargetObjectWriter(std::move(MOTW)),
+      ObjWriter(std::make_unique<WinCOFFWriter>(*this, OS,
+                                                WinCOFFWriter::AllSections)) {}
+WinCOFFObjectWriter::WinCOFFObjectWriter(
+    std::unique_ptr<MCWinCOFFObjectTargetWriter> MOTW, raw_pwrite_stream &OS,
+    raw_pwrite_stream &DwoOS)
+    : TargetObjectWriter(std::move(MOTW)),
+      ObjWriter(std::make_unique<WinCOFFWriter>(*this, OS,
+                                                WinCOFFWriter::NonDwoOnly)),
+      DwoWriter(std::make_unique<WinCOFFWriter>(*this, DwoOS,
+                                                WinCOFFWriter::DwoOnly)) {}
 
 static bool isDwoSection(const MCSection &Sec) {
   return Sec.getName().ends_with(".dwo");
@@ -1125,7 +1102,7 @@ uint64_t WinCOFFWriter::writeObject(MCAssembler &Asm) {
 
   // MS LINK expects to be able to use this timestamp to implement their
   // /INCREMENTAL feature.
-  if (Asm.isIncrementalLinkerCompatible()) {
+  if (OWriter.IncrementalLinkerCompatible) {
     Header.TimeDateStamp = getTime();
   } else {
     // Have deterministic output if /INCREMENTAL isn't needed. Also matches GNU.
@@ -1174,6 +1151,7 @@ uint64_t WinCOFFWriter::writeObject(MCAssembler &Asm) {
 // MCObjectWriter interface implementations
 
 void WinCOFFObjectWriter::reset() {
+  IncrementalLinkerCompatible = false;
   ObjWriter->reset();
   if (DwoWriter)
     DwoWriter->reset();
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index 2735142c576b1..ba9894f18b000 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -825,8 +825,6 @@ uint64_t XCOFFObjectWriter::writeObject(MCAssembler &Asm) {
   // We always emit a timestamp of 0 for reproducibility, so ensure incremental
   // linking is not enabled, in case, like with Windows COFF, such a timestamp
   // is incompatible with incremental linking of XCOFF.
-  if (Asm.isIncrementalLinkerCompatible())
-    report_fatal_error("Incremental linking not supported for XCOFF.");
 
   finalizeSectionInfo();
   uint64_t StartOffset = W.OS.tell();

From ffcd7e98d93cda157b4ab9dddd0aa42f364db576 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 12:48:09 -0700
Subject: [PATCH 350/486] [MC] Export llvm::SPIRVObjectTargetWriter and drop
 reliance on Mach-o specific VersionInfo

---
 llvm/include/llvm/MC/MCSPIRVObjectWriter.h | 28 ++++++++++++++++
 llvm/lib/MC/SPIRVObjectWriter.cpp          | 38 +++++-----------------
 llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp  |  5 +--
 3 files changed, 40 insertions(+), 31 deletions(-)

diff --git a/llvm/include/llvm/MC/MCSPIRVObjectWriter.h b/llvm/include/llvm/MC/MCSPIRVObjectWriter.h
index a8baf96b83842..18bdb8be0145e 100644
--- a/llvm/include/llvm/MC/MCSPIRVObjectWriter.h
+++ b/llvm/include/llvm/MC/MCSPIRVObjectWriter.h
@@ -10,6 +10,8 @@
 #define LLVM_MC_MCSPIRVOBJECTWRITER_H
 
 #include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCValue.h"
+#include "llvm/Support/EndianStream.h"
 #include "llvm/Support/raw_ostream.h"
 #include <memory>
 
@@ -26,6 +28,32 @@ class MCSPIRVObjectTargetWriter : public MCObjectTargetWriter {
   }
 };
 
+class SPIRVObjectWriter final : public MCObjectWriter {
+  support::endian::Writer W;
+  std::unique_ptr<MCSPIRVObjectTargetWriter> TargetObjectWriter;
+
+  struct VersionInfoType {
+    unsigned Major = 0;
+    unsigned Minor = 0;
+    unsigned Bound = 0;
+  } VersionInfo;
+
+public:
+  SPIRVObjectWriter(std::unique_ptr<MCSPIRVObjectTargetWriter> MOTW,
+                    raw_pwrite_stream &OS)
+      : W(OS, llvm::endianness::little), TargetObjectWriter(std::move(MOTW)) {}
+
+  void setBuildVersion(unsigned Major, unsigned Minor, unsigned Bound);
+
+private:
+  void recordRelocation(MCAssembler &Asm, const MCFragment *Fragment,
+                        const MCFixup &Fixup, MCValue Target,
+                        uint64_t &FixedValue) override {}
+
+  uint64_t writeObject(MCAssembler &Asm) override;
+  void writeHeader(const MCAssembler &Asm);
+};
+
 /// Construct a new SPIR-V writer instance.
 ///
 /// \param MOTW - The target specific SPIR-V writer subclass.
diff --git a/llvm/lib/MC/SPIRVObjectWriter.cpp b/llvm/lib/MC/SPIRVObjectWriter.cpp
index 91bde7ab62c1e..ef3d4b029e43e 100644
--- a/llvm/lib/MC/SPIRVObjectWriter.cpp
+++ b/llvm/lib/MC/SPIRVObjectWriter.cpp
@@ -14,47 +14,27 @@
 
 using namespace llvm;
 
-namespace {
-class SPIRVObjectWriter : public MCObjectWriter {
-  ::support::endian::Writer W;
-
-  /// The target specific SPIR-V writer instance.
-  std::unique_ptr<MCSPIRVObjectTargetWriter> TargetObjectWriter;
-
-public:
-  SPIRVObjectWriter(std::unique_ptr<MCSPIRVObjectTargetWriter> MOTW,
-                    raw_pwrite_stream &OS)
-      : W(OS, llvm::endianness::little), TargetObjectWriter(std::move(MOTW)) {}
-
-  ~SPIRVObjectWriter() override {}
-
-private:
-  void recordRelocation(MCAssembler &Asm, const MCFragment *Fragment,
-                        const MCFixup &Fixup, MCValue Target,
-                        uint64_t &FixedValue) override {}
-
-  uint64_t writeObject(MCAssembler &Asm) override;
-  void writeHeader(const MCAssembler &Asm);
-};
-} // namespace
-
 void SPIRVObjectWriter::writeHeader(const MCAssembler &Asm) {
   constexpr uint32_t MagicNumber = 0x07230203;
   constexpr uint32_t GeneratorID = 43;
   constexpr uint32_t GeneratorMagicNumber =
       (GeneratorID << 16) | (LLVM_VERSION_MAJOR);
   constexpr uint32_t Schema = 0;
-  const MCAssembler::VersionInfoType &VIT = Asm.getVersionInfo();
-  uint32_t VersionNumber = 0 | (VIT.Major << 16) | (VIT.Minor << 8);
-  uint32_t Bound = VIT.Update;
 
   W.write<uint32_t>(MagicNumber);
-  W.write<uint32_t>(VersionNumber);
+  W.write<uint32_t>((VersionInfo.Major << 16) | (VersionInfo.Minor << 8));
   W.write<uint32_t>(GeneratorMagicNumber);
-  W.write<uint32_t>(Bound);
+  W.write<uint32_t>(VersionInfo.Bound);
   W.write<uint32_t>(Schema);
 }
 
+void SPIRVObjectWriter::setBuildVersion(unsigned Major, unsigned Minor,
+                                        unsigned Bound) {
+  VersionInfo.Major = Major;
+  VersionInfo.Minor = Minor;
+  VersionInfo.Bound = Bound;
+}
+
 uint64_t SPIRVObjectWriter::writeObject(MCAssembler &Asm) {
   uint64_t StartOffset = W.OS.tell();
   writeHeader(Asm);
diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
index 3206c264f99d3..7fe8e11aaa420 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
@@ -32,6 +32,7 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCObjectStreamer.h"
+#include "llvm/MC/MCSPIRVObjectWriter.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/TargetRegistry.h"
@@ -116,8 +117,8 @@ void SPIRVAsmPrinter::emitEndOfAsmFile(Module &M) {
   // number and number of generated OpLabels
   unsigned Bound = 2 * (ST->getBound() + 1) + NLabels;
   if (MCAssembler *Asm = OutStreamer->getAssemblerPtr())
-    Asm->setBuildVersion(static_cast<MachO::PlatformType>(0), Major, Minor,
-                         Bound, VersionTuple(Major, Minor, 0, Bound));
+    static_cast<SPIRVObjectWriter &>(Asm->getWriter())
+        .setBuildVersion(Major, Minor, Bound);
 }
 
 void SPIRVAsmPrinter::emitFunctionHeader() {

From c1562374c8f24e5f873490639420b9c732b7e33d Mon Sep 17 00:00:00 2001
From: Job Henandez Lara <hj93@protonmail.com>
Date: Sun, 21 Jul 2024 12:55:11 -0700
Subject: [PATCH 351/486] [libc][math][c23] Add entrypoints and tests for
 dsqrt{l,f128} (#99815)

---
 libc/config/darwin/arm/entrypoints.txt      |  1 +
 libc/config/darwin/x86_64/entrypoints.txt   |  1 +
 libc/config/linux/aarch64/entrypoints.txt   |  2 ++
 libc/config/linux/arm/entrypoints.txt       |  1 +
 libc/config/linux/riscv/entrypoints.txt     |  2 ++
 libc/config/linux/x86_64/entrypoints.txt    |  2 ++
 libc/docs/math/index.rst                    |  2 +-
 libc/spec/llvm_libc_ext.td                  |  2 ++
 libc/spec/stdc.td                           |  3 +++
 libc/src/math/CMakeLists.txt                |  3 +++
 libc/src/math/dsqrtf128.h                   | 21 +++++++++++++++++
 libc/src/math/dsqrtl.h                      | 20 +++++++++++++++++
 libc/src/math/generic/CMakeLists.txt        | 25 +++++++++++++++++++++
 libc/src/math/generic/dsqrtf128.cpp         | 20 +++++++++++++++++
 libc/src/math/generic/dsqrtl.cpp            | 20 +++++++++++++++++
 libc/test/src/math/CMakeLists.txt           | 13 +++++++++++
 libc/test/src/math/dsqrtl_test.cpp          | 13 +++++++++++
 libc/test/src/math/smoke/CMakeLists.txt     | 24 ++++++++++++++++++++
 libc/test/src/math/smoke/dsqrtf128_test.cpp | 13 +++++++++++
 libc/test/src/math/smoke/dsqrtl_test.cpp    | 13 +++++++++++
 libc/utils/MPFRWrapper/MPFRUtils.cpp        |  6 +++++
 21 files changed, 206 insertions(+), 1 deletion(-)
 create mode 100644 libc/src/math/dsqrtf128.h
 create mode 100644 libc/src/math/dsqrtl.h
 create mode 100644 libc/src/math/generic/dsqrtf128.cpp
 create mode 100644 libc/src/math/generic/dsqrtl.cpp
 create mode 100644 libc/test/src/math/dsqrtl_test.cpp
 create mode 100644 libc/test/src/math/smoke/dsqrtf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/dsqrtl_test.cpp

diff --git a/libc/config/darwin/arm/entrypoints.txt b/libc/config/darwin/arm/entrypoints.txt
index 32a08f20b328f..ea5c7b537bbec 100644
--- a/libc/config/darwin/arm/entrypoints.txt
+++ b/libc/config/darwin/arm/entrypoints.txt
@@ -135,6 +135,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.cos
     libc.src.math.cosf
     libc.src.math.cospif
+    libc.src.math.dsqrtl
     libc.src.math.erff
     libc.src.math.exp
     libc.src.math.expf
diff --git a/libc/config/darwin/x86_64/entrypoints.txt b/libc/config/darwin/x86_64/entrypoints.txt
index 02912decadcf7..1a7353172d464 100644
--- a/libc/config/darwin/x86_64/entrypoints.txt
+++ b/libc/config/darwin/x86_64/entrypoints.txt
@@ -119,6 +119,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     #libc.src.math.ceill
     #libc.src.math.coshf
     #libc.src.math.cosf
+    #libc.src.math.dsqrtl
     #libc.src.math.expf
     #libc.src.math.exp2f
     #libc.src.math.expm1f
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 8afd3fb67197e..0be6f884f0368 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -359,6 +359,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.coshf
     libc.src.math.cospif
     libc.src.math.dmull
+    libc.src.math.dsqrtl
     libc.src.math.erff
     libc.src.math.exp
     libc.src.math.exp10
@@ -598,6 +599,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     # math.h C23 _Float128 entrypoints
     libc.src.math.ceilf128
     libc.src.math.copysignf128
+    libc.src.math.dsqrtf128
     libc.src.math.fabsf128
     libc.src.math.fdimf128
     libc.src.math.floorf128
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index 0d09e4c22953c..55f118395c22e 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -227,6 +227,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.cos
     libc.src.math.cosf
     libc.src.math.coshf
+    libc.src.math.dsqrtl
     libc.src.math.erff
     libc.src.math.exp
     libc.src.math.exp10
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 54a382eccb546..ea3f36604e45d 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -377,6 +377,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.cosf
     libc.src.math.coshf
     libc.src.math.cospif
+    libc.src.math.dsqrtl
     libc.src.math.erff
     libc.src.math.exp
     libc.src.math.exp10
@@ -549,6 +550,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.ceilf128
     libc.src.math.copysignf128
     libc.src.math.dmulf128
+    libc.src.math.dsqrtf128
     libc.src.math.fabsf128
     libc.src.math.fdimf128
     libc.src.math.floorf128
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 736908809d96c..7c422bad9f01d 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -382,6 +382,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.coshf
     libc.src.math.cospif
     libc.src.math.dmull
+    libc.src.math.dsqrtl
     libc.src.math.erff
     libc.src.math.exp
     libc.src.math.exp10
@@ -643,6 +644,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.ceilf128
     libc.src.math.copysignf128
     libc.src.math.dmulf128
+    libc.src.math.dsqrtf128
     libc.src.math.fabsf128
     libc.src.math.fdimf128
     libc.src.math.floorf128
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index 5cf9d8a42929e..f287c16fd01e2 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -278,7 +278,7 @@ Higher Math Functions
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | cospi     | |check|          |                 |                        |                      |                        | 7.12.4.12              | F.10.1.12                  |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| dsqrt     | N/A              | N/A             |                        | N/A                  |                        | 7.12.14.6              | F.10.11                    |
+| dsqrt     | N/A              | N/A             |   |check|              | N/A                  |       |check|\*        | 7.12.14.6              | F.10.11                    |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | erf       | |check|          |                 |                        |                      |                        | 7.12.8.1               | F.10.5.1                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/spec/llvm_libc_ext.td b/libc/spec/llvm_libc_ext.td
index 6291a9ceec60d..f3a8862574ac5 100644
--- a/libc/spec/llvm_libc_ext.td
+++ b/libc/spec/llvm_libc_ext.td
@@ -57,6 +57,8 @@ def LLVMLibcExt : StandardSpec<"llvm_libc_ext"> {
       [], // Types
       [], // Enumerations
       [
+          GuardedFunctionSpec<"dsqrtf128", RetValSpec<DoubleType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+	  
           GuardedFunctionSpec<"f16add", RetValSpec<Float16Type>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"f16addf", RetValSpec<Float16Type>, [ArgSpec<FloatType>, ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"f16addl", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 437d6d04faf96..0aae65308d33a 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -717,6 +717,9 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"canonicalizel", RetValSpec<IntType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
           GuardedFunctionSpec<"canonicalizef16", RetValSpec<IntType>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"canonicalizef128", RetValSpec<IntType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+	  
+          FunctionSpec<"dsqrtl", RetValSpec<DoubleType>, [ArgSpec<LongDoubleType>]>,
+	
 
           GuardedFunctionSpec<"totalorderf16", RetValSpec<IntType>, [ArgSpec<Float16Ptr>, ArgSpec<Float16Ptr>], "LIBC_TYPES_HAS_FLOAT16">,
 
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 82dd23186187b..25aef3f72e3cd 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -89,6 +89,9 @@ add_math_entrypoint_object(cospif)
 add_math_entrypoint_object(dmull)
 add_math_entrypoint_object(dmulf128)
 
+add_math_entrypoint_object(dsqrtl)
+add_math_entrypoint_object(dsqrtf128)
+
 add_math_entrypoint_object(erf)
 add_math_entrypoint_object(erff)
 
diff --git a/libc/src/math/dsqrtf128.h b/libc/src/math/dsqrtf128.h
new file mode 100644
index 0000000000000..97103392e3fef
--- /dev/null
+++ b/libc/src/math/dsqrtf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for dsqrtf128 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_DSQRTF128_H
+#define LLVM_LIBC_SRC_MATH_DSQRTF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+double dsqrtf128(float128 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_DSQRTF128_H
diff --git a/libc/src/math/dsqrtl.h b/libc/src/math/dsqrtl.h
new file mode 100644
index 0000000000000..7bf0255697da4
--- /dev/null
+++ b/libc/src/math/dsqrtl.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for dsqrtl ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_DSQRTL_H
+#define LLVM_LIBC_SRC_MATH_DSQRTL_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+double dsqrtl(long double x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_DSQRTL_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 7712a6d1ae527..0513aa6cebd2d 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -129,6 +129,31 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+  dsqrtl
+  SRCS
+    dsqrtl.cpp
+  HDRS
+    ../dsqrtl.h
+  DEPENDS
+    libc.src.__support.FPUtil.generic.sqrt
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  dsqrtf128
+  SRCS
+    dsqrtf128.cpp
+  HDRS
+    ../dsqrtf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.generic.sqrt
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_header_library(
   range_reduction
   HDRS
diff --git a/libc/src/math/generic/dsqrtf128.cpp b/libc/src/math/generic/dsqrtf128.cpp
new file mode 100644
index 0000000000000..ad8339309b0f3
--- /dev/null
+++ b/libc/src/math/generic/dsqrtf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of dsqrtf128 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/dsqrtf128.h"
+#include "src/__support/FPUtil/generic/sqrt.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(double, dsqrtf128, (float128 x)) {
+  return fputil::sqrt<double>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/dsqrtl.cpp b/libc/src/math/generic/dsqrtl.cpp
new file mode 100644
index 0000000000000..bf1dae9161460
--- /dev/null
+++ b/libc/src/math/generic/dsqrtl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of dsqrtl function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/dsqrtl.h"
+#include "src/__support/FPUtil/generic/sqrt.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(double, dsqrtl, (long double x)) {
+  return fputil::sqrt<double>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 586338b2213de..3ad5d98858165 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -2271,6 +2271,19 @@ add_fp_unittest(
     libc.src.math.fsqrtl
 )
 
+add_fp_unittest(
+  dsqrtl_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    dsqrtl_test.cpp
+  HDRS
+    SqrtTest.h
+  DEPENDS
+    libc.src.math.dsqrtl
+)
+
 add_fp_unittest(
   cbrtf_test
   NEED_MPFR
diff --git a/libc/test/src/math/dsqrtl_test.cpp b/libc/test/src/math/dsqrtl_test.cpp
new file mode 100644
index 0000000000000..c178e389a57cd
--- /dev/null
+++ b/libc/test/src/math/dsqrtl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for dsqrtl ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SqrtTest.h"
+
+#include "src/math/dsqrtl.h"
+
+LIST_NARROWING_SQRT_TESTS(double, long double, LIBC_NAMESPACE::dsqrtl)
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index f7caa03d11646..1b3c51739c0fe 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -3996,6 +3996,30 @@ add_fp_unittest(
     libc.src.math.fsqrtf128
 )
 
+add_fp_unittest(
+  dsqrtl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    dsqrtl_test.cpp
+  HDRS
+    SqrtTest.h
+  DEPENDS
+    libc.src.math.dsqrtl
+)
+
+add_fp_unittest(
+  dsqrtf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    dsqrtf128_test.cpp
+  HDRS
+    SqrtTest.h
+  DEPENDS
+    libc.src.math.dsqrtf128
+)
+
 add_fp_unittest(
   sin_test
   SUITE
diff --git a/libc/test/src/math/smoke/dsqrtf128_test.cpp b/libc/test/src/math/smoke/dsqrtf128_test.cpp
new file mode 100644
index 0000000000000..6f98d1cfc6042
--- /dev/null
+++ b/libc/test/src/math/smoke/dsqrtf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for dsqrtf128 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SqrtTest.h"
+
+#include "src/math/dsqrtf128.h"
+
+LIST_NARROWING_SQRT_TESTS(double, float128, LIBC_NAMESPACE::dsqrtf128)
diff --git a/libc/test/src/math/smoke/dsqrtl_test.cpp b/libc/test/src/math/smoke/dsqrtl_test.cpp
new file mode 100644
index 0000000000000..c178e389a57cd
--- /dev/null
+++ b/libc/test/src/math/smoke/dsqrtl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for dsqrtl ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SqrtTest.h"
+
+#include "src/math/dsqrtl.h"
+
+LIST_NARROWING_SQRT_TESTS(double, long double, LIBC_NAMESPACE::dsqrtl)
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp
index d2c28c19ab176..f88ee2af35c52 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.cpp
+++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp
@@ -893,6 +893,9 @@ template void explain_unary_operation_single_output_error(Operation op, double,
 template void explain_unary_operation_single_output_error(Operation op,
                                                           long double, float,
                                                           double, RoundingMode);
+template void explain_unary_operation_single_output_error(Operation op,
+                                                          long double, double,
+                                                          double, RoundingMode);
 
 #ifdef LIBC_TYPES_HAS_FLOAT16
 template void explain_unary_operation_single_output_error(Operation op, float16,
@@ -1118,6 +1121,9 @@ template bool compare_unary_operation_single_output(Operation, double, float,
 template bool compare_unary_operation_single_output(Operation, long double,
                                                     float, double,
                                                     RoundingMode);
+template bool compare_unary_operation_single_output(Operation, long double,
+                                                    double, double,
+                                                    RoundingMode);
 #ifdef LIBC_TYPES_HAS_FLOAT16
 template bool compare_unary_operation_single_output(Operation, float16, float16,
                                                     double, RoundingMode);

From 09a399a1ddbef1e5e51b77fd6bd1792d34697187 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 13:03:21 -0700
Subject: [PATCH 352/486] [MC] Move VersionInfo to MachObjectWriter

---
 llvm/include/llvm/MC/MCAssembler.h        | 60 -----------------------
 llvm/include/llvm/MC/MCMachObjectWriter.h | 52 ++++++++++++++++++++
 llvm/lib/MC/MCAssembler.cpp               |  9 +---
 llvm/lib/MC/MCMachOStreamer.cpp           | 10 ++--
 llvm/lib/MC/MachObjectWriter.cpp          | 10 ++--
 5 files changed, 63 insertions(+), 78 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 5ffe2ca5fabc3..3852bdf9777d6 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -15,12 +15,9 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/ADT/iterator_range.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/SMLoc.h"
-#include "llvm/Support/VersionTuple.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
@@ -58,22 +55,6 @@ class MCAssembler {
   using SectionListType = SmallVector<MCSection *, 0>;
   using const_iterator = pointee_iterator<SectionListType::const_iterator>;
 
-  /// MachO specific deployment target version info.
-  // A Major version of 0 indicates that no version information was supplied
-  // and so the corresponding load command should not be emitted.
-  using VersionInfoType = struct {
-    bool EmitBuildVersion;
-    union {
-      MCVersionMinType Type;          ///< Used when EmitBuildVersion==false.
-      MachO::PlatformType Platform;   ///< Used when EmitBuildVersion==true.
-    } TypeOrPlatform;
-    unsigned Major;
-    unsigned Minor;
-    unsigned Update;
-    /// An optional version of the SDK that was used to build the source.
-    VersionTuple SDKVersion;
-  };
-
 private:
   MCContext &Context;
 
@@ -120,9 +101,6 @@ class MCAssembler {
   // which flags to be set.
   unsigned ELFHeaderEFlags = 0;
 
-  VersionInfoType VersionInfo;
-  VersionInfoType DarwinTargetVariantVersionInfo;
-
   /// Evaluate a fixup to a relocatable expression and the value which should be
   /// placed into the fixup.
   ///
@@ -226,44 +204,6 @@ class MCAssembler {
   unsigned getELFHeaderEFlags() const { return ELFHeaderEFlags; }
   void setELFHeaderEFlags(unsigned Flags) { ELFHeaderEFlags = Flags; }
 
-  /// MachO deployment target version information.
-  const VersionInfoType &getVersionInfo() const { return VersionInfo; }
-  void setVersionMin(MCVersionMinType Type, unsigned Major, unsigned Minor,
-                     unsigned Update,
-                     VersionTuple SDKVersion = VersionTuple()) {
-    VersionInfo.EmitBuildVersion = false;
-    VersionInfo.TypeOrPlatform.Type = Type;
-    VersionInfo.Major = Major;
-    VersionInfo.Minor = Minor;
-    VersionInfo.Update = Update;
-    VersionInfo.SDKVersion = SDKVersion;
-  }
-  void setBuildVersion(MachO::PlatformType Platform, unsigned Major,
-                       unsigned Minor, unsigned Update,
-                       VersionTuple SDKVersion = VersionTuple()) {
-    VersionInfo.EmitBuildVersion = true;
-    VersionInfo.TypeOrPlatform.Platform = Platform;
-    VersionInfo.Major = Major;
-    VersionInfo.Minor = Minor;
-    VersionInfo.Update = Update;
-    VersionInfo.SDKVersion = SDKVersion;
-  }
-
-  const VersionInfoType &getDarwinTargetVariantVersionInfo() const {
-    return DarwinTargetVariantVersionInfo;
-  }
-  void setDarwinTargetVariantBuildVersion(MachO::PlatformType Platform,
-                                          unsigned Major, unsigned Minor,
-                                          unsigned Update,
-                                          VersionTuple SDKVersion) {
-    DarwinTargetVariantVersionInfo.EmitBuildVersion = true;
-    DarwinTargetVariantVersionInfo.TypeOrPlatform.Platform = Platform;
-    DarwinTargetVariantVersionInfo.Major = Major;
-    DarwinTargetVariantVersionInfo.Minor = Minor;
-    DarwinTargetVariantVersionInfo.Update = Update;
-    DarwinTargetVariantVersionInfo.SDKVersion = SDKVersion;
-  }
-
   /// Reuse an assembler instance
   ///
   void reset();
diff --git a/llvm/include/llvm/MC/MCMachObjectWriter.h b/llvm/include/llvm/MC/MCMachObjectWriter.h
index 60e3e28819724..919b7217f2ea6 100644
--- a/llvm/include/llvm/MC/MCMachObjectWriter.h
+++ b/llvm/include/llvm/MC/MCMachObjectWriter.h
@@ -12,12 +12,14 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/MachO.h"
+#include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/StringTableBuilder.h"
 #include "llvm/Support/EndianStream.h"
+#include "llvm/Support/VersionTuple.h"
 #include <cstdint>
 #include <memory>
 #include <string>
@@ -89,6 +91,21 @@ class MachObjectWriter : public MCObjectWriter {
     MCSymbol *End;
   };
 
+  // A Major version of 0 indicates that no version information was supplied
+  // and so the corresponding load command should not be emitted.
+  using VersionInfoType = struct {
+    bool EmitBuildVersion;
+    union {
+      MCVersionMinType Type;        ///< Used when EmitBuildVersion==false.
+      MachO::PlatformType Platform; ///< Used when EmitBuildVersion==true.
+    } TypeOrPlatform;
+    unsigned Major;
+    unsigned Minor;
+    unsigned Update;
+    /// An optional version of the SDK that was used to build the source.
+    VersionTuple SDKVersion;
+  };
+
 private:
   /// Helper struct for containing some precomputed information on symbols.
   struct MachSymbolData {
@@ -144,6 +161,9 @@ class MachObjectWriter : public MCObjectWriter {
   // Used to communicate Linker Optimization Hint information.
   MCLOHContainer LOHContainer;
 
+  VersionInfoType VersionInfo{};
+  VersionInfoType TargetVariantVersionInfo{};
+
   MachSymbolData *findSymbolData(const MCSymbol &Sym);
 
   void writeWithPadding(StringRef Str, uint64_t Size);
@@ -197,6 +217,38 @@ class MachObjectWriter : public MCObjectWriter {
 
   bool doesSymbolRequireExternRelocation(const MCSymbol &S);
 
+  /// Mach-O deployment target version information.
+  void setVersionMin(MCVersionMinType Type, unsigned Major, unsigned Minor,
+                     unsigned Update,
+                     VersionTuple SDKVersion = VersionTuple()) {
+    VersionInfo.EmitBuildVersion = false;
+    VersionInfo.TypeOrPlatform.Type = Type;
+    VersionInfo.Major = Major;
+    VersionInfo.Minor = Minor;
+    VersionInfo.Update = Update;
+    VersionInfo.SDKVersion = SDKVersion;
+  }
+  void setBuildVersion(MachO::PlatformType Platform, unsigned Major,
+                       unsigned Minor, unsigned Update,
+                       VersionTuple SDKVersion = VersionTuple()) {
+    VersionInfo.EmitBuildVersion = true;
+    VersionInfo.TypeOrPlatform.Platform = Platform;
+    VersionInfo.Major = Major;
+    VersionInfo.Minor = Minor;
+    VersionInfo.Update = Update;
+    VersionInfo.SDKVersion = SDKVersion;
+  }
+  void setTargetVariantBuildVersion(MachO::PlatformType Platform,
+                                    unsigned Major, unsigned Minor,
+                                    unsigned Update, VersionTuple SDKVersion) {
+    TargetVariantVersionInfo.EmitBuildVersion = true;
+    TargetVariantVersionInfo.TypeOrPlatform.Platform = Platform;
+    TargetVariantVersionInfo.Major = Major;
+    TargetVariantVersionInfo.Minor = Minor;
+    TargetVariantVersionInfo.Update = Update;
+    TargetVariantVersionInfo.SDKVersion = SDKVersion;
+  }
+
   /// @}
 
   /// \name Target Writer Proxy Accessors
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 852fa76f84ccc..e731dfccc0560 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -84,10 +84,7 @@ MCAssembler::MCAssembler(MCContext &Context,
                          std::unique_ptr<MCCodeEmitter> Emitter,
                          std::unique_ptr<MCObjectWriter> Writer)
     : Context(Context), Backend(std::move(Backend)),
-      Emitter(std::move(Emitter)), Writer(std::move(Writer)) {
-  VersionInfo.Major = 0; // Major version == 0 for "none specified"
-  DarwinTargetVariantVersionInfo.Major = 0;
-}
+      Emitter(std::move(Emitter)), Writer(std::move(Writer)) {}
 
 MCAssembler::~MCAssembler() = default;
 
@@ -101,10 +98,6 @@ void MCAssembler::reset() {
   ThumbFuncs.clear();
   BundleAlignSize = 0;
   ELFHeaderEFlags = 0;
-  VersionInfo.Major = 0;
-  VersionInfo.SDKVersion = VersionTuple();
-  DarwinTargetVariantVersionInfo.Major = 0;
-  DarwinTargetVariantVersionInfo.SDKVersion = VersionTuple();
 
   // reset objects owned by us
   if (getBackendPtr())
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index f83e3096f3238..1d5d1c6ee72f1 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -252,21 +252,21 @@ void MCMachOStreamer::emitDataRegion(MCDataRegionType Kind) {
 void MCMachOStreamer::emitVersionMin(MCVersionMinType Kind, unsigned Major,
                                      unsigned Minor, unsigned Update,
                                      VersionTuple SDKVersion) {
-  getAssembler().setVersionMin(Kind, Major, Minor, Update, SDKVersion);
+  getWriter().setVersionMin(Kind, Major, Minor, Update, SDKVersion);
 }
 
 void MCMachOStreamer::emitBuildVersion(unsigned Platform, unsigned Major,
                                        unsigned Minor, unsigned Update,
                                        VersionTuple SDKVersion) {
-  getAssembler().setBuildVersion((MachO::PlatformType)Platform, Major, Minor,
-                                 Update, SDKVersion);
+  getWriter().setBuildVersion((MachO::PlatformType)Platform, Major, Minor,
+                              Update, SDKVersion);
 }
 
 void MCMachOStreamer::emitDarwinTargetVariantBuildVersion(
     unsigned Platform, unsigned Major, unsigned Minor, unsigned Update,
     VersionTuple SDKVersion) {
-  getAssembler().setDarwinTargetVariantBuildVersion(
-      (MachO::PlatformType)Platform, Major, Minor, Update, SDKVersion);
+  getWriter().setTargetVariantBuildVersion((MachO::PlatformType)Platform, Major,
+                                           Minor, Update, SDKVersion);
 }
 
 void MCMachOStreamer::emitThumbFunc(MCSymbol *Symbol) {
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 3c67f16e3d102..cc29c1ae6ccd6 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -56,6 +56,10 @@ void MachObjectWriter::reset() {
   ExternalSymbolData.clear();
   UndefinedSymbolData.clear();
   LOHContainer.reset();
+  VersionInfo.Major = 0;
+  VersionInfo.SDKVersion = VersionTuple();
+  TargetVariantVersionInfo.Major = 0;
+  TargetVariantVersionInfo.SDKVersion = VersionTuple();
   MCObjectWriter::reset();
 }
 
@@ -803,7 +807,6 @@ uint64_t MachObjectWriter::writeObject(MCAssembler &Asm) {
   }
 
   unsigned NumSections = Asm.end() - Asm.begin();
-  const MCAssembler::VersionInfoType &VersionInfo = Asm.getVersionInfo();
 
   // The section data starts after the header, the segment load command (and
   // section headers) and the symbol table.
@@ -821,9 +824,6 @@ uint64_t MachObjectWriter::writeObject(MCAssembler &Asm) {
       LoadCommandsSize += sizeof(MachO::version_min_command);
   }
 
-  const MCAssembler::VersionInfoType &TargetVariantVersionInfo =
-      Asm.getDarwinTargetVariantVersionInfo();
-
   // Add the target variant version info load command size, if used.
   if (TargetVariantVersionInfo.Major != 0) {
     ++NumLoadCommands;
@@ -928,7 +928,7 @@ uint64_t MachObjectWriter::writeObject(MCAssembler &Asm) {
 
   // Write out the deployment target information, if it's available.
   auto EmitDeploymentTargetVersion =
-      [&](const MCAssembler::VersionInfoType &VersionInfo) {
+      [&](const VersionInfoType &VersionInfo) {
         auto EncodeVersion = [](VersionTuple V) -> uint32_t {
           assert(!V.empty() && "empty version");
           unsigned Update = V.getSubminor().value_or(0);

From 5c83498984dd5d278c40d6650d3f9205b5131550 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 21 Jul 2024 13:11:25 -0700
Subject: [PATCH 353/486] [Transforms] Use range-based for loops (NFC) (#99607)

---
 llvm/lib/Transforms/Utils/LoopUnroll.cpp                    | 4 ++--
 llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 0afa1ef780504..a0406111ecbf3 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -1091,8 +1091,8 @@ MDNode *llvm::GetUnrollMetadata(MDNode *LoopID, StringRef Name) {
   assert(LoopID->getNumOperands() > 0 && "requires at least one operand");
   assert(LoopID->getOperand(0) == LoopID && "invalid loop id");
 
-  for (unsigned i = 1, e = LoopID->getNumOperands(); i < e; ++i) {
-    MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
+  for (const MDOperand &MDO : llvm::drop_begin(LoopID->operands())) {
+    MDNode *MD = dyn_cast<MDNode>(MDO);
     if (!MD)
       continue;
 
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index f54eebb2874ab..cafec165f6d6f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -261,20 +261,20 @@ void LoopVectorizeHints::getHintsFromMetadata() {
   assert(LoopID->getNumOperands() > 0 && "requires at least one operand");
   assert(LoopID->getOperand(0) == LoopID && "invalid loop id");
 
-  for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
+  for (const MDOperand &MDO : llvm::drop_begin(LoopID->operands())) {
     const MDString *S = nullptr;
     SmallVector<Metadata *, 4> Args;
 
     // The expected hint is either a MDString or a MDNode with the first
     // operand a MDString.
-    if (const MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i))) {
+    if (const MDNode *MD = dyn_cast<MDNode>(MDO)) {
       if (!MD || MD->getNumOperands() == 0)
         continue;
       S = dyn_cast<MDString>(MD->getOperand(0));
       for (unsigned i = 1, ie = MD->getNumOperands(); i < ie; ++i)
         Args.push_back(MD->getOperand(i));
     } else {
-      S = dyn_cast<MDString>(LoopID->getOperand(i));
+      S = dyn_cast<MDString>(MDO);
       assert(Args.size() == 0 && "too many arguments for MDString");
     }
 

From dcebe297367f48a71a5ecea3bcdd54671195da1b Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sun, 21 Jul 2024 13:14:18 -0700
Subject: [PATCH 354/486] [clang-format] Fix a bug in annotating StartOfName
 (#99791)

Fixes #99758.
---
 clang/lib/Format/TokenAnnotator.cpp           |  4 +++-
 clang/unittests/Format/TokenAnnotatorTest.cpp | 14 ++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index db66911f00f63..06c34bd45eb31 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -2625,8 +2625,10 @@ class AnnotatingParser {
       return false;
 
     // int a or auto a.
-    if (PreviousNotConst->isOneOf(tok::identifier, tok::kw_auto))
+    if (PreviousNotConst->isOneOf(tok::identifier, tok::kw_auto) &&
+        PreviousNotConst->isNot(TT_StatementAttributeLikeMacro)) {
       return true;
+    }
 
     // *a or &a or &&a.
     if (PreviousNotConst->is(TT_PointerOrReference))
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index f70424c3ee060..e7bc15d7e1dfa 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -2125,6 +2125,13 @@ TEST_F(TokenAnnotatorTest, UnderstandsTrailingReturnArrow) {
   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
   EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
 
+  auto Style = getLLVMStyle();
+  Style.StatementAttributeLikeMacros.push_back("emit");
+  Tokens = annotate("emit foo()->bar;", Style);
+  ASSERT_EQ(Tokens.size(), 8u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_StatementAttributeLikeMacro);
+  EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
+
   // Mixed
   Tokens = annotate("auto f() -> int { auto a = b()->c; }");
   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
@@ -2950,6 +2957,13 @@ TEST_F(TokenAnnotatorTest, StartOfName) {
   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
   EXPECT_TOKEN(Tokens[0], tok::at, TT_ObjCDecl);
   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_StartOfName);
+
+  auto Style = getLLVMStyle();
+  Style.StatementAttributeLikeMacros.push_back("emit");
+  Tokens = annotate("emit foo = 0;", Style);
+  ASSERT_EQ(Tokens.size(), 6u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_StatementAttributeLikeMacro);
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
 }
 
 TEST_F(TokenAnnotatorTest, BraceKind) {

From 568845ae50650420532f4e2c6b22f5b302fcec38 Mon Sep 17 00:00:00 2001
From: Alexander Belyaev <32522095+pifon2a@users.noreply.github.com>
Date: Sun, 21 Jul 2024 23:08:04 +0200
Subject: [PATCH 355/486] [mlir] Add a ValueSemantics trait. (#99493)

We need to distinguish ShapedTypes with and without value semantics.
This is needed for downstream users to define their custom vector and
tensor
types that can work with the arith/math dialect.

RFC https://discourse.llvm.org/t/rfc-mlir-types-with-encoding/80189
---
 mlir/include/mlir/IR/BuiltinTypes.h           |  5 ++++
 mlir/include/mlir/IR/BuiltinTypes.td          | 16 +++++++++---
 mlir/include/mlir/IR/CommonTypeConstraints.td | 26 +++++++++++++++----
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/mlir/include/mlir/IR/BuiltinTypes.h b/mlir/include/mlir/IR/BuiltinTypes.h
index 5579b138668d2..f04668ed9142c 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.h
+++ b/mlir/include/mlir/IR/BuiltinTypes.h
@@ -39,6 +39,11 @@ struct IntegerTypeStorage;
 struct TupleTypeStorage;
 } // namespace detail
 
+/// Type trait indicating that the type has value semantics.
+template <typename ConcreteType>
+class ValueSemantics
+    : public TypeTrait::TraitBase<ConcreteType, ValueSemantics> {};
+
 //===----------------------------------------------------------------------===//
 // FloatType
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/BuiltinTypes.td b/mlir/include/mlir/IR/BuiltinTypes.td
index 4cade83dd3c32..91aece3a543f0 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.td
+++ b/mlir/include/mlir/IR/BuiltinTypes.td
@@ -30,6 +30,15 @@ class Builtin_Type<string name, string typeMnemonic, list<Trait> traits = [],
   let typeName = "builtin." # typeMnemonic;
 }
 
+//===----------------------------------------------------------------------===//
+// Traits
+//===----------------------------------------------------------------------===//
+
+/// Type trait indicating that the type has value semantics.
+def ValueSemantics : NativeTypeTrait<"ValueSemantics"> {
+  let cppNamespace = "::mlir";
+}
+
 //===----------------------------------------------------------------------===//
 // ComplexType
 //===----------------------------------------------------------------------===//
@@ -745,7 +754,7 @@ def Builtin_Opaque : Builtin_Type<"Opaque", "opaque"> {
 //===----------------------------------------------------------------------===//
 
 def Builtin_RankedTensor : Builtin_Type<"RankedTensor", "tensor", [
-    ShapedTypeInterface
+    ShapedTypeInterface, ValueSemantics
   ], "TensorType"> {
   let summary = "Multi-dimensional array with a fixed number of dimensions";
   let description = [{
@@ -1001,7 +1010,7 @@ def Builtin_UnrankedMemRef : Builtin_Type<"UnrankedMemRef", "unranked_memref", [
 //===----------------------------------------------------------------------===//
 
 def Builtin_UnrankedTensor : Builtin_Type<"UnrankedTensor", "unranked_tensor", [
-    ShapedTypeInterface
+    ShapedTypeInterface, ValueSemantics
   ], "TensorType"> {
   let summary = "Multi-dimensional array with unknown dimensions";
   let description = [{
@@ -1049,7 +1058,8 @@ def Builtin_UnrankedTensor : Builtin_Type<"UnrankedTensor", "unranked_tensor", [
 // VectorType
 //===----------------------------------------------------------------------===//
 
-def Builtin_Vector : Builtin_Type<"Vector", "vector", [ShapedTypeInterface], "Type"> {
+def Builtin_Vector : Builtin_Type<"Vector", "vector",
+    [ShapedTypeInterface, ValueSemantics], "Type"> {
   let summary = "Multi-dimensional SIMD vector type";
   let description = [{
     Syntax:
diff --git a/mlir/include/mlir/IR/CommonTypeConstraints.td b/mlir/include/mlir/IR/CommonTypeConstraints.td
index af4f13dc09360..9b524f02e5d26 100644
--- a/mlir/include/mlir/IR/CommonTypeConstraints.td
+++ b/mlir/include/mlir/IR/CommonTypeConstraints.td
@@ -89,6 +89,9 @@ def HasStaticShapePred :
 // Whether a type is a TupleType.
 def IsTupleTypePred : CPred<"::llvm::isa<::mlir::TupleType>($_self)">;
 
+// Whether a type has a ValueSemantics trait.
+def HasValueSemanticsPred : CPred<"$_self.hasTrait<::mlir::ValueSemantics>()">;
+
 //===----------------------------------------------------------------------===//
 // Type definitions
 //===----------------------------------------------------------------------===//
@@ -403,6 +406,11 @@ class HasRankGreaterOrEqualPred<int rank> : And<[
     CPred<[{::llvm::cast<::mlir::ShapedType>($_self).getRank() >= }] # rank>
 ]>;
 
+// Container with value semantics.
+class ValueSemanticsContainerOf<list<Type> allowedTypes> :
+  ShapedContainerType<allowedTypes, HasValueSemanticsPred,
+  "container with value semantics">;
+
 // Vector types.
 
 class VectorOf<list<Type> allowedTypes> :
@@ -842,10 +850,18 @@ class NestedTupleOf<list<Type> allowedTypes> :
 // Common type constraints
 //===----------------------------------------------------------------------===//
 // Type constraint for types that are "like" some type or set of types T, that is
-// they're either a T, a vector of Ts, or a tensor of Ts
+// they're either a T, a vector of Ts, or a tensor of Ts.
 class TypeOrContainer<Type allowedType, string name> : TypeConstraint<Or<[
-  allowedType.predicate, VectorOf<[allowedType]>.predicate,
-  TensorOf<[allowedType]>.predicate]>,
+  allowedType.predicate,
+  ValueSemanticsContainerOf<[allowedType]>.predicate]>,
+  name>;
+
+// Type constraint for types that are "like" some type or set of types T, that is
+// they're either a T or a mapable container of Ts.
+class TypeOrValueSemanticsContainer<Type allowedType, string name>
+    : TypeConstraint<Or<[
+  allowedType.predicate,
+  ValueSemanticsContainerOf<[allowedType]>.predicate]>,
   name>;
 
 // Temporary constraint to allow gradual transition to supporting 0-D vectors.
@@ -864,8 +880,8 @@ def BoolLikeOfAnyRank : TypeOrContainerOfAnyRank<I1, "bool-like">;
 
 // Type constraint for signless-integer-like types: signless integers, indices,
 // vectors of signless integers or indices, tensors of signless integers.
-def SignlessIntegerLike : TypeOrContainer<AnySignlessIntegerOrIndex,
-    "signless-integer-like">;
+def SignlessIntegerLike : TypeOrValueSemanticsContainer<
+    AnySignlessIntegerOrIndex, "signless-integer-like">;
 
 def SignlessIntegerLikeOfAnyRank : TypeOrContainerOfAnyRank<
     AnySignlessIntegerOrIndex,

From a964f2e8a1e98d5ac1fa379ec52b6713ae5cb48a Mon Sep 17 00:00:00 2001
From: jameshu15869 <55058507+jameshu15869@users.noreply.github.com>
Date: Sun, 21 Jul 2024 16:40:01 -0500
Subject: [PATCH 356/486] [libc] Improve Benchmark UI (#99796)

This PR changes the output to resemble Google Benchmark. e.g.

```
Running Suite: LlvmLibcIsAlNumGpuBenchmark
Benchmark            |  Cycles |     Min |     Max | Iterations |   Time (ns) |   Stddev |  Threads |
-----------------------------------------------------------------------------------------------------
IsAlnum              |      92 |      76 |     482 |         23 |       86500 |       76 |       64 |
IsAlnumSingleThread  |      87 |      76 |     302 |         20 |       72000 |       49 |        1 |
IsAlnumSingleWave    |      87 |      76 |     302 |         20 |       72000 |       49 |       32 |
IsAlnumCapital       |      89 |      76 |     299 |         17 |       78500 |       52 |       64 |
IsAlnumNotAlnum      |      87 |      76 |     303 |         20 |       76000 |       49 |       64 |
```
---
 libc/benchmarks/gpu/CMakeLists.txt       |  6 ++-
 libc/benchmarks/gpu/LibcGpuBenchmark.cpp | 57 +++++++++++++++++++-----
 libc/benchmarks/gpu/LibcGpuBenchmark.h   | 17 ++++---
 3 files changed, 62 insertions(+), 18 deletions(-)

diff --git a/libc/benchmarks/gpu/CMakeLists.txt b/libc/benchmarks/gpu/CMakeLists.txt
index 29e27724e1ab3..69518acff3a5d 100644
--- a/libc/benchmarks/gpu/CMakeLists.txt
+++ b/libc/benchmarks/gpu/CMakeLists.txt
@@ -7,7 +7,7 @@ function(add_benchmark benchmark_name)
     "BENCHMARK"
     "" # Optional arguments
     "" # Single value arguments
-    "LINK_LIBRARIES" # Multi-value arguments
+    "LINK_LIBRARIES;DEPENDS" # Multi-value arguments
     ${ARGN}
   )
   
@@ -20,6 +20,9 @@ function(add_benchmark benchmark_name)
     LINK_LIBRARIES
       LibcGpuBenchmark.hermetic
       ${BENCHMARK_LINK_LIBRARIES}
+    DEPENDS
+      libc.src.stdio.printf
+      ${BENCHMARK_DEPENDS}
     ${BENCHMARK_UNPARSED_ARGUMENTS}
   )
   get_fq_target_name(${benchmark_name} fq_target_name)
@@ -55,6 +58,7 @@ add_unittest_framework_library(
     libc.src.__support.fixedvector
     libc.src.time.clock
     libc.benchmarks.gpu.timing.timing
+    libc.src.stdio.printf
 )
 
 add_subdirectory(src)
diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
index c926d8efd7db2..59de18c20417d 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
@@ -7,6 +7,7 @@
 #include "src/__support/GPU/utils.h"
 #include "src/__support/fixedvector.h"
 #include "src/__support/macros/config.h"
+#include "src/stdio/printf.h"
 #include "src/time/gpu/time_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
@@ -73,10 +74,16 @@ struct AtomicBenchmarkSums {
 };
 
 AtomicBenchmarkSums all_results;
+const char *header_format_string =
+    "Benchmark            |  Cycles |     Min |     Max | Iterations |        "
+    "Time |   Stddev |  Threads |\n";
+const char *output_format_string =
+    "%-20s |%8ld |%8ld |%8ld |%11ld |%9ld %2s |%9ld |%9d |\n";
+
+constexpr auto GREEN = "\033[32m";
+constexpr auto RESET = "\033[0m";
 
 void print_results(Benchmark *b) {
-  constexpr auto GREEN = "\033[32m";
-  constexpr auto RESET = "\033[0m";
 
   BenchmarkResult result;
   cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
@@ -92,21 +99,51 @@ void print_results(Benchmark *b) {
       all_results.samples_sum.load(cpp::MemoryOrder::RELAXED) / num_threads;
   result.total_iterations =
       all_results.iterations_sum.load(cpp::MemoryOrder::RELAXED) / num_threads;
-  result.total_time =
+  const uint64_t duration_ns =
       all_results.time_sum.load(cpp::MemoryOrder::RELAXED) / num_threads;
+  const uint64_t duration_us = duration_ns / 1000;
+  const uint64_t duration_ms = duration_ns / (1000 * 1000);
+  uint64_t converted_duration = duration_ns;
+  cpp::string time_unit;
+  if (duration_ms != 0) {
+    converted_duration = duration_ms;
+    time_unit = cpp::string("ms");
+  } else if (duration_us != 0) {
+    converted_duration = duration_us;
+    time_unit = cpp::string("us");
+  } else {
+    converted_duration = duration_ns;
+    time_unit = cpp::string("ns");
+  }
+  result.total_time = converted_duration;
+  // result.total_time =
+  //     all_results.time_sum.load(cpp::MemoryOrder::RELAXED) / num_threads;
   cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
 
-  log << GREEN << "[ RUN      ] " << RESET << b->get_name() << '\n';
-  log << GREEN << "[       OK ] " << RESET << b->get_name() << ": "
-      << result.cycles << " cycles, " << result.min << " min, " << result.max
-      << " max, " << result.total_iterations << " iterations, "
-      << result.total_time << " ns, "
-      << static_cast<uint64_t>(result.standard_deviation)
-      << " stddev (num threads: " << num_threads << ")\n";
+  LIBC_NAMESPACE::printf(
+      output_format_string, b->get_test_name().data(), result.cycles,
+      result.min, result.max, result.total_iterations, result.total_time,
+      time_unit.data(), static_cast<uint64_t>(result.standard_deviation),
+      num_threads);
+}
+
+void print_header() {
+  LIBC_NAMESPACE::printf("%s", GREEN);
+  LIBC_NAMESPACE::printf("Running Suite: %-10s\n",
+                         benchmarks[0]->get_suite_name().data());
+  LIBC_NAMESPACE::printf("%s", RESET);
+  LIBC_NAMESPACE::printf(header_format_string);
+  LIBC_NAMESPACE::printf(
+      "---------------------------------------------------------------------"
+      "--------------------------------\n");
 }
 
 void Benchmark::run_benchmarks() {
   uint64_t id = gpu::get_thread_id();
+
+  if (id == 0)
+    print_header();
+
   gpu::sync_threads();
 
   for (Benchmark *b : benchmarks) {
diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.h b/libc/benchmarks/gpu/LibcGpuBenchmark.h
index 29d7ba8b0a132..c07fab9ccfbe3 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.h
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.h
@@ -81,17 +81,20 @@ BenchmarkResult benchmark(const BenchmarkOptions &options,
 
 class Benchmark {
   const cpp::function<uint64_t(void)> func;
-  const cpp::string_view name;
+  const cpp::string_view suite_name;
+  const cpp::string_view test_name;
   const uint8_t flags;
 
 public:
-  Benchmark(cpp::function<uint64_t(void)> func, char const *name, uint8_t flags)
-      : func(func), name(name), flags(flags) {
+  Benchmark(cpp::function<uint64_t(void)> func, char const *suite_name,
+            char const *test_name, uint8_t flags)
+      : func(func), suite_name(suite_name), test_name(test_name), flags(flags) {
     add_benchmark(this);
   }
 
   static void run_benchmarks();
-  const cpp::string_view get_name() const { return name; }
+  const cpp::string_view get_suite_name() const { return suite_name; }
+  const cpp::string_view get_test_name() const { return test_name; }
 
 protected:
   static void add_benchmark(Benchmark *benchmark);
@@ -107,16 +110,16 @@ class Benchmark {
 
 #define BENCHMARK(SuiteName, TestName, Func)                                   \
   LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance(     \
-      Func, #SuiteName "." #TestName, 0)
+      Func, #SuiteName, #TestName, 0)
 
 #define SINGLE_THREADED_BENCHMARK(SuiteName, TestName, Func)                   \
   LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance(     \
-      Func, #SuiteName "." #TestName,                                          \
+      Func, #SuiteName, #TestName,                                             \
       LIBC_NAMESPACE::benchmarks::BenchmarkFlags::SINGLE_THREADED)
 
 #define SINGLE_WAVE_BENCHMARK(SuiteName, TestName, Func)                       \
   LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance(     \
-      Func, #SuiteName "." #TestName,                                          \
+      Func, #SuiteName, #TestName,                                             \
       LIBC_NAMESPACE::benchmarks::BenchmarkFlags::SINGLE_WAVE)
 
 #endif

From 0387cd052b081d6bc9856ef756942a5df1a2a301 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sun, 21 Jul 2024 14:54:50 -0700
Subject: [PATCH 357/486] [clang-format] Fix a bug in annotating
 FunctionAnnotationRParen (#99802)

Fixes #37906.
---
 clang/lib/Format/TokenAnnotator.cpp           | 6 +++---
 clang/unittests/Format/FormatTest.cpp         | 1 +
 clang/unittests/Format/TokenAnnotatorTest.cpp | 4 ++++
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 06c34bd45eb31..21924a8fe17d1 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -2467,9 +2467,9 @@ class AnnotatingParser {
         Current.setType(TT_CastRParen);
       if (Current.MatchingParen && Current.Next &&
           !Current.Next->isBinaryOperator() &&
-          !Current.Next->isOneOf(tok::semi, tok::colon, tok::l_brace,
-                                 tok::comma, tok::period, tok::arrow,
-                                 tok::coloncolon, tok::kw_noexcept)) {
+          !Current.Next->isOneOf(
+              tok::semi, tok::colon, tok::l_brace, tok::l_paren, tok::comma,
+              tok::period, tok::arrow, tok::coloncolon, tok::kw_noexcept)) {
         if (FormatToken *AfterParen = Current.MatchingParen->Next;
             AfterParen && AfterParen->isNot(tok::caret)) {
           // Make sure this isn't the return type of an Obj-C block declaration.
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index d01ce137b8fcb..39fcbab3447a7 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -8742,6 +8742,7 @@ TEST_F(FormatTest, FunctionAnnotations) {
                "    << abc;");
   verifyFormat("MACRO(abc)::function() // wrap\n"
                "    << abc;");
+  verifyFormat("FOO(bar)();", getLLVMStyleWithColumns(0));
 }
 
 TEST_F(FormatTest, BreaksDesireably) {
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index e7bc15d7e1dfa..b01ca322505b1 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -1930,6 +1930,10 @@ TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) {
                     "A(T) noexcept;");
   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
+
+  Tokens = annotate("FOO(bar)();");
+  ASSERT_EQ(Tokens.size(), 8u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_Unknown);
 }
 
 TEST_F(TokenAnnotatorTest, UnderstandsFunctionDeclarationNames) {

From 1a47f3f3db66589c11f8ddacfeaecc03fb80c510 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 21 Jul 2024 15:42:27 -0700
Subject: [PATCH 358/486] [MC] Compute fragment offsets eagerly

This builds on top of commit 9d0754ada5dbbc0c009bcc2f7824488419cc5530
("[MC] Relax fragments eagerly") and relaxes fragments eagerly to
eliminate MCSection::HasLayout and `getFragmentOffset` overhead.

Note: The removed `#ifndef NDEBUG` code (disabled by default) in
X86AsmBackend::finishLayout was problematic, as (a) !NDEBUG and NDEBUG
builds evaluated fragment offsets at different times before this patch
(b) one iteration might not be sufficient to converge. There might be
some edge cases that it did not handle. Anyhow, this patch probably
makes it work for more cases.
---
 llvm/include/llvm/MC/MCAsmBackend.h           |  5 +-
 llvm/include/llvm/MC/MCAssembler.h            |  4 +-
 llvm/include/llvm/MC/MCSection.h              |  5 --
 llvm/lib/MC/MCAssembler.cpp                   | 77 ++++++++++---------
 llvm/lib/MC/MCSection.cpp                     |  4 +-
 .../MCTargetDesc/HexagonAsmBackend.cpp        |  4 +-
 .../Target/X86/MCTargetDesc/X86AsmBackend.cpp | 26 ++-----
 7 files changed, 54 insertions(+), 71 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index f91c8e1fc9af3..736f44686689b 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -217,8 +217,9 @@ class MCAsmBackend {
   virtual bool writeNopData(raw_ostream &OS, uint64_t Count,
                             const MCSubtargetInfo *STI) const = 0;
 
-  /// Give backend an opportunity to finish layout after relaxation
-  virtual void finishLayout(MCAssembler const &Asm) const {}
+  // Return true if fragment offsets have been adjusted and an extra layout
+  // iteration is needed.
+  virtual bool finishLayout(const MCAssembler &Asm) const { return false; }
 
   /// Handle any target-specific assembler flags. By default, do nothing.
   virtual void handleAssemblerFlag(MCAssemblerFlag Flag) {}
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 3852bdf9777d6..3fb681cd09cfa 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -126,6 +126,7 @@ class MCAssembler {
   /// Check whether the given fragment needs relaxation.
   bool fragmentNeedsRelaxation(const MCRelaxableFragment *IF) const;
 
+  void layoutSection(MCSection &Sec);
   /// Perform one layout iteration and return true if any offsets
   /// were adjusted.
   bool layoutOnce();
@@ -172,10 +173,9 @@ class MCAssembler {
   uint64_t computeFragmentSize(const MCFragment &F) const;
 
   void layoutBundle(MCFragment *Prev, MCFragment *F) const;
-  void ensureValid(MCSection &Sec) const;
 
   // Get the offset of the given fragment inside its containing section.
-  uint64_t getFragmentOffset(const MCFragment &F) const;
+  uint64_t getFragmentOffset(const MCFragment &F) const { return F.Offset; }
 
   uint64_t getSectionAddressSize(const MCSection &Sec) const;
   uint64_t getSectionFileSize(const MCSection &Sec) const;
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index dcdcd094fa17b..1289d6f6f9f65 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -99,8 +99,6 @@ class MCSection {
   /// Whether this section has had instructions emitted into it.
   bool HasInstructions : 1;
 
-  bool HasLayout : 1;
-
   bool IsRegistered : 1;
 
   bool IsText : 1;
@@ -169,9 +167,6 @@ class MCSection {
   bool hasInstructions() const { return HasInstructions; }
   void setHasInstructions(bool Value) { HasInstructions = Value; }
 
-  bool hasLayout() const { return HasLayout; }
-  void setHasLayout(bool Value) { HasLayout = Value; }
-
   bool isRegistered() const { return IsRegistered; }
   void setIsRegistered(bool Value) { IsRegistered = Value; }
 
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index e731dfccc0560..a8652d2d82f8d 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -438,28 +438,6 @@ void MCAssembler::layoutBundle(MCFragment *Prev, MCFragment *F) const {
       DF->Offset = EF->Offset;
 }
 
-void MCAssembler::ensureValid(MCSection &Sec) const {
-  if (Sec.hasLayout())
-    return;
-  Sec.setHasLayout(true);
-  MCFragment *Prev = nullptr;
-  uint64_t Offset = 0;
-  for (MCFragment &F : Sec) {
-    F.Offset = Offset;
-    if (isBundlingEnabled() && F.hasInstructions()) {
-      layoutBundle(Prev, &F);
-      Offset = F.Offset;
-    }
-    Offset += computeFragmentSize(F);
-    Prev = &F;
-  }
-}
-
-uint64_t MCAssembler::getFragmentOffset(const MCFragment &F) const {
-  ensureValid(*F.getParent());
-  return F.Offset;
-}
-
 // Simple getSymbolOffset helper for the non-variable case.
 static bool getLabelOffset(const MCAssembler &Asm, const MCSymbol &S,
                            bool ReportError, uint64_t &Val) {
@@ -944,22 +922,20 @@ void MCAssembler::layout() {
 
   // Layout until everything fits.
   this->HasLayout = true;
+  for (MCSection &Sec : *this)
+    layoutSection(Sec);
   while (layoutOnce()) {
-    if (getContext().hadError())
-      return;
-    // Size of fragments in one section can depend on the size of fragments in
-    // another. If any fragment has changed size, we have to re-layout (and
-    // as a result possibly further relax) all.
-    for (MCSection &Sec : *this)
-      Sec.setHasLayout(false);
   }
 
   DEBUG_WITH_TYPE("mc-dump", {
       errs() << "assembler backend - post-relaxation\n--\n";
       dump(); });
 
-  // Finalize the layout, including fragment lowering.
-  getBackend().finishLayout(*this);
+  // Some targets might want to adjust fragment offsets. If so, perform another
+  // layout loop.
+  if (getBackend().finishLayout(*this))
+    for (MCSection &Sec : *this)
+      layoutSection(Sec);
 
   DEBUG_WITH_TYPE("mc-dump", {
       errs() << "assembler backend - final-layout\n--\n";
@@ -1312,15 +1288,42 @@ bool MCAssembler::relaxFragment(MCFragment &F) {
   }
 }
 
+void MCAssembler::layoutSection(MCSection &Sec) {
+  MCFragment *Prev = nullptr;
+  uint64_t Offset = 0;
+  for (MCFragment &F : Sec) {
+    F.Offset = Offset;
+    if (LLVM_UNLIKELY(isBundlingEnabled())) {
+      if (F.hasInstructions()) {
+        layoutBundle(Prev, &F);
+        Offset = F.Offset;
+      }
+      Prev = &F;
+    }
+    Offset += computeFragmentSize(F);
+  }
+}
+
 bool MCAssembler::layoutOnce() {
   ++stats::RelaxationSteps;
 
-  bool Changed = false;
-  for (MCSection &Sec : *this)
-    for (MCFragment &Frag : Sec)
-      if (relaxFragment(Frag))
-        Changed = true;
-  return Changed;
+  // Size of fragments in one section can depend on the size of fragments in
+  // another. If any fragment has changed size, we have to re-layout (and
+  // as a result possibly further relax) all.
+  bool ChangedAny = false;
+  for (MCSection &Sec : *this) {
+    for (;;) {
+      bool Changed = false;
+      for (MCFragment &F : Sec)
+        if (relaxFragment(F))
+          Changed = true;
+      ChangedAny |= Changed;
+      if (!Changed)
+        break;
+      layoutSection(Sec);
+    }
+  }
+  return ChangedAny;
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp
index 8c2ee5635a49c..97e87a41c8ce5 100644
--- a/llvm/lib/MC/MCSection.cpp
+++ b/llvm/lib/MC/MCSection.cpp
@@ -23,8 +23,8 @@ using namespace llvm;
 MCSection::MCSection(SectionVariant V, StringRef Name, bool IsText,
                      bool IsVirtual, MCSymbol *Begin)
     : Begin(Begin), BundleGroupBeforeFirstInst(false), HasInstructions(false),
-      HasLayout(false), IsRegistered(false), IsText(IsText),
-      IsVirtual(IsVirtual), Name(Name), Variant(V) {
+      IsRegistered(false), IsText(IsText), IsVirtual(IsVirtual), Name(Name),
+      Variant(V) {
   DummyFragment.setParent(this);
   // The initial subsection number is 0. Create a fragment list.
   CurFragList = &Subsections.emplace_back(0u, FragList{}).second;
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
index 6acc37e599f2e..1570493b765ca 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
@@ -702,7 +702,7 @@ class HexagonAsmBackend : public MCAsmBackend {
     return true;
   }
 
-  void finishLayout(MCAssembler const &Asm) const override {
+  bool finishLayout(const MCAssembler &Asm) const override {
     SmallVector<MCFragment *> Frags;
     for (MCSection &Sec : Asm) {
       Frags.clear();
@@ -747,7 +747,6 @@ class HexagonAsmBackend : public MCAsmBackend {
               //assert(!Error);
               (void)Error;
               ReplaceInstruction(Asm.getEmitter(), RF, Inst);
-              Sec.setHasLayout(false);
               Size = 0; // Only look back one instruction
               break;
             }
@@ -757,6 +756,7 @@ class HexagonAsmBackend : public MCAsmBackend {
         }
       }
     }
+    return true;
   }
 }; // class HexagonAsmBackend
 
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 09a6d57afc0ba..a3ef11b2cab45 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -201,7 +201,7 @@ class X86AsmBackend : public MCAsmBackend {
   bool padInstructionEncoding(MCRelaxableFragment &RF, MCCodeEmitter &Emitter,
                               unsigned &RemainingSize) const;
 
-  void finishLayout(const MCAssembler &Asm) const override;
+  bool finishLayout(const MCAssembler &Asm) const override;
 
   unsigned getMaximumNopSize(const MCSubtargetInfo &STI) const override;
 
@@ -856,7 +856,7 @@ bool X86AsmBackend::padInstructionEncoding(MCRelaxableFragment &RF,
   return Changed;
 }
 
-void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
+bool X86AsmBackend::finishLayout(const MCAssembler &Asm) const {
   // See if we can further relax some instructions to cut down on the number of
   // nop bytes required for code alignment.  The actual win is in reducing
   // instruction count, not number of bytes.  Modern X86-64 can easily end up
@@ -864,7 +864,7 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
   // (i.e. eliminate nops) even at the cost of increasing the size and
   // complexity of others.
   if (!X86PadForAlign && !X86PadForBranchAlign)
-    return;
+    return false;
 
   // The processed regions are delimitered by LabeledFragments. -g may have more
   // MCSymbols and therefore different relaxation results. X86PadForAlign is
@@ -911,9 +911,6 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
         continue;
       }
 
-#ifndef NDEBUG
-      const uint64_t OrigOffset = Asm.getFragmentOffset(F);
-#endif
       const uint64_t OrigSize = Asm.computeFragmentSize(F);
 
       // To keep the effects local, prefer to relax instructions closest to
@@ -926,8 +923,7 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
         // Give the backend a chance to play any tricks it wishes to increase
         // the encoding size of the given instruction.  Target independent code
         // will try further relaxation, but target's may play further tricks.
-        if (padInstructionEncoding(RF, Asm.getEmitter(), RemainingSize))
-          Sec.setHasLayout(false);
+        padInstructionEncoding(RF, Asm.getEmitter(), RemainingSize);
 
         // If we have an instruction which hasn't been fully relaxed, we can't
         // skip past it and insert bytes before it.  Changing its starting
@@ -944,14 +940,6 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
       if (F.getKind() == MCFragment::FT_BoundaryAlign)
         cast<MCBoundaryAlignFragment>(F).setSize(RemainingSize);
 
-#ifndef NDEBUG
-      const uint64_t FinalOffset = Asm.getFragmentOffset(F);
-      const uint64_t FinalSize = Asm.computeFragmentSize(F);
-      assert(OrigOffset + OrigSize == FinalOffset + FinalSize &&
-             "can't move start of next fragment!");
-      assert(FinalSize == RemainingSize && "inconsistent size computation?");
-#endif
-
       // If we're looking at a boundary align, make sure we don't try to pad
       // its target instructions for some following directive.  Doing so would
       // break the alignment of the current boundary align.
@@ -965,11 +953,7 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
     }
   }
 
-  // The layout is done. Mark every fragment as valid.
-  for (MCSection &Section : Asm) {
-    Asm.getFragmentOffset(*Section.curFragList()->Tail);
-    Asm.computeFragmentSize(*Section.curFragList()->Tail);
-  }
+  return true;
 }
 
 unsigned X86AsmBackend::getMaximumNopSize(const MCSubtargetInfo &STI) const {

From 74a1ca504bf60f02431140ee72dbe1c158556237 Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Sun, 21 Jul 2024 19:27:06 -0400
Subject: [PATCH 359/486] [libc][math] Update getpayload and
 fmul/fadd/fsub/ffma with NaN inputs. (#99812)

---
 libc/config/linux/riscv/entrypoints.txt     |  2 +
 libc/src/__support/FPUtil/BasicOperations.h | 13 ++++-
 libc/src/__support/FPUtil/CMakeLists.txt    | 57 +++++++++++----------
 libc/src/__support/FPUtil/generic/FMA.h     | 45 +++++++++-------
 libc/src/__support/FPUtil/generic/add_sub.h | 26 +++++-----
 libc/src/__support/FPUtil/generic/div.h     | 20 ++++----
 libc/src/__support/FPUtil/generic/mul.h     | 20 ++++----
 libc/test/src/math/smoke/AddTest.h          | 19 +------
 libc/test/src/math/smoke/DivTest.h          | 19 +------
 libc/test/src/math/smoke/MulTest.h          | 19 +------
 libc/test/src/math/smoke/SubTest.h          | 19 +------
 11 files changed, 111 insertions(+), 148 deletions(-)

diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index ea3f36604e45d..61a1ef351a4c6 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -377,6 +377,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.cosf
     libc.src.math.coshf
     libc.src.math.cospif
+    libc.src.math.dmull
     libc.src.math.dsqrtl
     libc.src.math.erff
     libc.src.math.exp
@@ -434,6 +435,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fmodf
     libc.src.math.fmodl
     libc.src.math.fmul
+    libc.src.math.fmull
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h
index a963a92bfb074..3b7d7a5c249ae 100644
--- a/libc/src/__support/FPUtil/BasicOperations.h
+++ b/libc/src/__support/FPUtil/BasicOperations.h
@@ -11,8 +11,8 @@
 
 #include "FEnvImpl.h"
 #include "FPBits.h"
+#include "dyadic_float.h"
 
-#include "FEnvImpl.h"
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
@@ -269,12 +269,21 @@ totalordermag(T x, T y) {
 template <typename T>
 LIBC_INLINE cpp::enable_if_t<cpp::is_floating_point_v<T>, T> getpayload(T x) {
   using FPBits = FPBits<T>;
+  using StorageType = typename FPBits::StorageType;
   FPBits x_bits(x);
 
   if (!x_bits.is_nan())
     return T(-1.0);
 
-  return T(x_bits.uintval() & (FPBits::FRACTION_MASK >> 1));
+  StorageType payload = x_bits.uintval() & (FPBits::FRACTION_MASK >> 1);
+
+  if constexpr (is_big_int_v<StorageType>) {
+    DyadicFloat<FPBits::STORAGE_LEN> payload_dfloat(Sign::POS, 0, payload);
+
+    return static_cast<T>(payload_dfloat);
+  } else {
+    return static_cast<T>(payload);
+  }
 }
 
 template <bool IsSignaling, typename T>
diff --git a/libc/src/__support/FPUtil/CMakeLists.txt b/libc/src/__support/FPUtil/CMakeLists.txt
index 793d3a121c742..8804f3a4d5e23 100644
--- a/libc/src/__support/FPUtil/CMakeLists.txt
+++ b/libc/src/__support/FPUtil/CMakeLists.txt
@@ -75,19 +75,6 @@ add_header_library(
     libc.src.__support.common
 )
 
-add_header_library(
-  basic_operations
-  HDRS
-    BasicOperations.h
-  DEPENDS
-    .fp_bits
-    .fenv_impl
-    libc.src.__support.CPP.type_traits
-    libc.src.__support.uint128
-    libc.src.__support.common
-    libc.src.__support.macros.optimization
-)
-
 add_header_library(
   division_and_remainder_operations
   HDRS
@@ -113,21 +100,6 @@ add_header_library(
 )
 
 
-add_header_library(
-  hypot
-  HDRS
-    Hypot.h
-  DEPENDS
-    .basic_operations
-    .fenv_impl
-    .fp_bits
-    .rounding_mode
-    libc.src.__support.common
-    libc.src.__support.CPP.bit
-    libc.src.__support.CPP.type_traits
-    libc.src.__support.uint128
-)
-
 add_header_library(
   sqrt
   HDRS
@@ -208,6 +180,35 @@ add_header_library(
     libc.src.__support.macros.optimization
 )
 
+add_header_library(
+  basic_operations
+  HDRS
+    BasicOperations.h
+  DEPENDS
+    .dyadic_float
+    .fp_bits
+    .fenv_impl
+    libc.src.__support.CPP.type_traits
+    libc.src.__support.uint128
+    libc.src.__support.common
+    libc.src.__support.macros.optimization
+)
+
+add_header_library(
+  hypot
+  HDRS
+    Hypot.h
+  DEPENDS
+    .basic_operations
+    .fenv_impl
+    .fp_bits
+    .rounding_mode
+    libc.src.__support.common
+    libc.src.__support.CPP.bit
+    libc.src.__support.CPP.type_traits
+    libc.src.__support.uint128
+)
+
 add_header_library(
   manipulation_functions
   HDRS
diff --git a/libc/src/__support/FPUtil/generic/FMA.h b/libc/src/__support/FPUtil/generic/FMA.h
index 337301d86d919..e5683c8ff61ea 100644
--- a/libc/src/__support/FPUtil/generic/FMA.h
+++ b/libc/src/__support/FPUtil/generic/FMA.h
@@ -129,27 +129,27 @@ fma(InType x, InType y, InType z) {
         raise_except_if_required(FE_INVALID);
 
       if (x_bits.is_quiet_nan()) {
-        InStorageType x_payload = static_cast<InStorageType>(getpayload(x));
-        if ((x_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
-          return OutFPBits::quiet_nan(x_bits.sign(),
-                                      static_cast<OutStorageType>(x_payload))
-              .get_val();
+        InStorageType x_payload = x_bits.get_mantissa();
+        x_payload >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
+        return OutFPBits::quiet_nan(x_bits.sign(),
+                                    static_cast<OutStorageType>(x_payload))
+            .get_val();
       }
 
       if (y_bits.is_quiet_nan()) {
-        InStorageType y_payload = static_cast<InStorageType>(getpayload(y));
-        if ((y_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
-          return OutFPBits::quiet_nan(y_bits.sign(),
-                                      static_cast<OutStorageType>(y_payload))
-              .get_val();
+        InStorageType y_payload = y_bits.get_mantissa();
+        y_payload >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
+        return OutFPBits::quiet_nan(y_bits.sign(),
+                                    static_cast<OutStorageType>(y_payload))
+            .get_val();
       }
 
       if (z_bits.is_quiet_nan()) {
-        InStorageType z_payload = static_cast<InStorageType>(getpayload(z));
-        if ((z_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
-          return OutFPBits::quiet_nan(z_bits.sign(),
-                                      static_cast<OutStorageType>(z_payload))
-              .get_val();
+        InStorageType z_payload = z_bits.get_mantissa();
+        z_payload >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
+        return OutFPBits::quiet_nan(z_bits.sign(),
+                                    static_cast<OutStorageType>(z_payload))
+            .get_val();
       }
 
       return OutFPBits::quiet_nan().get_val();
@@ -163,18 +163,27 @@ fma(InType x, InType y, InType z) {
   int y_exp = 0;
   int z_exp = 0;
 
+  // Denormal scaling = 2^(fraction length).
+  constexpr InStorageType IMPLICIT_MASK =
+      InFPBits::SIG_MASK - InFPBits::FRACTION_MASK;
+
+  constexpr InType DENORMAL_SCALING =
+      InFPBits::create_value(
+          Sign::POS, InFPBits::FRACTION_LEN + InFPBits::EXP_BIAS, IMPLICIT_MASK)
+          .get_val();
+
   // Normalize denormal inputs.
   if (LIBC_UNLIKELY(InFPBits(x).is_subnormal())) {
     x_exp -= InFPBits::FRACTION_LEN;
-    x *= InType(InStorageType(1) << InFPBits::FRACTION_LEN);
+    x *= DENORMAL_SCALING;
   }
   if (LIBC_UNLIKELY(InFPBits(y).is_subnormal())) {
     y_exp -= InFPBits::FRACTION_LEN;
-    y *= InType(InStorageType(1) << InFPBits::FRACTION_LEN);
+    y *= DENORMAL_SCALING;
   }
   if (LIBC_UNLIKELY(InFPBits(z).is_subnormal())) {
     z_exp -= InFPBits::FRACTION_LEN;
-    z *= InType(InStorageType(1) << InFPBits::FRACTION_LEN);
+    z *= DENORMAL_SCALING;
   }
 
   x_bits = InFPBits(x);
diff --git a/libc/src/__support/FPUtil/generic/add_sub.h b/libc/src/__support/FPUtil/generic/add_sub.h
index ec20a8723b704..850db3f83209e 100644
--- a/libc/src/__support/FPUtil/generic/add_sub.h
+++ b/libc/src/__support/FPUtil/generic/add_sub.h
@@ -56,19 +56,19 @@ add_or_sub(InType x, InType y) {
         raise_except_if_required(FE_INVALID);
 
       if (x_bits.is_quiet_nan()) {
-        InStorageType x_payload = static_cast<InStorageType>(getpayload(x));
-        if ((x_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
-          return OutFPBits::quiet_nan(x_bits.sign(),
-                                      static_cast<OutStorageType>(x_payload))
-              .get_val();
+        InStorageType x_payload = x_bits.get_mantissa();
+        x_payload >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
+        return OutFPBits::quiet_nan(x_bits.sign(),
+                                    static_cast<OutStorageType>(x_payload))
+            .get_val();
       }
 
       if (y_bits.is_quiet_nan()) {
-        InStorageType y_payload = static_cast<InStorageType>(getpayload(y));
-        if ((y_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
-          return OutFPBits::quiet_nan(y_bits.sign(),
-                                      static_cast<OutStorageType>(y_payload))
-              .get_val();
+        InStorageType y_payload = y_bits.get_mantissa();
+        y_payload >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
+        return OutFPBits::quiet_nan(y_bits.sign(),
+                                    static_cast<OutStorageType>(y_payload))
+            .get_val();
       }
 
       return OutFPBits::quiet_nan().get_val();
@@ -174,10 +174,12 @@ add_or_sub(InType x, InType y) {
     else
       aligned_min_mant_sticky = true;
 
+    InStorageType min_mant_sticky(static_cast<int>(aligned_min_mant_sticky));
+
     if (is_effectively_add)
-      result_mant = max_mant + (aligned_min_mant | aligned_min_mant_sticky);
+      result_mant = max_mant + (aligned_min_mant | min_mant_sticky);
     else
-      result_mant = max_mant - (aligned_min_mant | aligned_min_mant_sticky);
+      result_mant = max_mant - (aligned_min_mant | min_mant_sticky);
   }
 
   int result_exp = max_bits.get_exponent() - RESULT_FRACTION_LEN;
diff --git a/libc/src/__support/FPUtil/generic/div.h b/libc/src/__support/FPUtil/generic/div.h
index 27545786aea17..dad1772fce750 100644
--- a/libc/src/__support/FPUtil/generic/div.h
+++ b/libc/src/__support/FPUtil/generic/div.h
@@ -49,19 +49,19 @@ div(InType x, InType y) {
         raise_except_if_required(FE_INVALID);
 
       if (x_bits.is_quiet_nan()) {
-        InStorageType x_payload = static_cast<InStorageType>(getpayload(x));
-        if ((x_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
-          return OutFPBits::quiet_nan(x_bits.sign(),
-                                      static_cast<OutStorageType>(x_payload))
-              .get_val();
+        InStorageType x_payload = x_bits.get_mantissa();
+        x_payload >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
+        return OutFPBits::quiet_nan(x_bits.sign(),
+                                    static_cast<OutStorageType>(x_payload))
+            .get_val();
       }
 
       if (y_bits.is_quiet_nan()) {
-        InStorageType y_payload = static_cast<InStorageType>(getpayload(y));
-        if ((y_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
-          return OutFPBits::quiet_nan(y_bits.sign(),
-                                      static_cast<OutStorageType>(y_payload))
-              .get_val();
+        InStorageType y_payload = y_bits.get_mantissa();
+        y_payload >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
+        return OutFPBits::quiet_nan(y_bits.sign(),
+                                    static_cast<OutStorageType>(y_payload))
+            .get_val();
       }
 
       return OutFPBits::quiet_nan().get_val();
diff --git a/libc/src/__support/FPUtil/generic/mul.h b/libc/src/__support/FPUtil/generic/mul.h
index 02fc69c6cb1ba..20d9a77792762 100644
--- a/libc/src/__support/FPUtil/generic/mul.h
+++ b/libc/src/__support/FPUtil/generic/mul.h
@@ -50,19 +50,19 @@ mul(InType x, InType y) {
         raise_except_if_required(FE_INVALID);
 
       if (x_bits.is_quiet_nan()) {
-        InStorageType x_payload = static_cast<InStorageType>(getpayload(x));
-        if ((x_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
-          return OutFPBits::quiet_nan(x_bits.sign(),
-                                      static_cast<OutStorageType>(x_payload))
-              .get_val();
+        InStorageType x_payload = x_bits.get_mantissa();
+        x_payload >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
+        return OutFPBits::quiet_nan(x_bits.sign(),
+                                    static_cast<OutStorageType>(x_payload))
+            .get_val();
       }
 
       if (y_bits.is_quiet_nan()) {
-        InStorageType y_payload = static_cast<InStorageType>(getpayload(y));
-        if ((y_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
-          return OutFPBits::quiet_nan(y_bits.sign(),
-                                      static_cast<OutStorageType>(y_payload))
-              .get_val();
+        InStorageType y_payload = y_bits.get_mantissa();
+        y_payload >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
+        return OutFPBits::quiet_nan(y_bits.sign(),
+                                    static_cast<OutStorageType>(y_payload))
+            .get_val();
       }
 
       return OutFPBits::quiet_nan().get_val();
diff --git a/libc/test/src/math/smoke/AddTest.h b/libc/test/src/math/smoke/AddTest.h
index c713c5a88763c..0b7e395a22d4c 100644
--- a/libc/test/src/math/smoke/AddTest.h
+++ b/libc/test/src/math/smoke/AddTest.h
@@ -38,23 +38,8 @@ class AddTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(sNaN, sNaN), FE_INVALID);
 
     InType qnan_42 = InFPBits::quiet_nan(Sign::POS, 0x42).get_val();
-    EXPECT_FP_EQ(InType(0x42.0p+0),
-                 LIBC_NAMESPACE::fputil::getpayload(func(qnan_42, zero)));
-    EXPECT_FP_EQ(InType(0x42.0p+0),
-                 LIBC_NAMESPACE::fputil::getpayload(func(zero, qnan_42)));
-
-    if constexpr (sizeof(OutType) < sizeof(InType)) {
-      InStorageType max_payload = InFPBits::FRACTION_MASK >> 1;
-      InType qnan_max = InFPBits::quiet_nan(Sign::POS, max_payload).get_val();
-      EXPECT_FP_EQ(zero,
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_max, zero)));
-      EXPECT_FP_EQ(zero,
-                   LIBC_NAMESPACE::fputil::getpayload(func(zero, qnan_max)));
-      EXPECT_FP_EQ(InType(0x42.0p+0),
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_max, qnan_42)));
-      EXPECT_FP_EQ(InType(0x42.0p+0),
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_42, qnan_max)));
-    }
+    EXPECT_FP_IS_NAN(func(qnan_42, zero));
+    EXPECT_FP_IS_NAN(func(zero, qnan_42));
 
     EXPECT_FP_EQ(inf, func(inf, zero));
     EXPECT_FP_EQ(neg_inf, func(neg_inf, zero));
diff --git a/libc/test/src/math/smoke/DivTest.h b/libc/test/src/math/smoke/DivTest.h
index 71cfb326b97cc..b30fc17aac1d5 100644
--- a/libc/test/src/math/smoke/DivTest.h
+++ b/libc/test/src/math/smoke/DivTest.h
@@ -36,23 +36,8 @@ class DivTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(sNaN, sNaN), FE_INVALID);
 
     InType qnan_42 = InFPBits::quiet_nan(Sign::POS, 0x42).get_val();
-    EXPECT_FP_EQ(InType(0x42.0p+0),
-                 LIBC_NAMESPACE::fputil::getpayload(func(qnan_42, zero)));
-    EXPECT_FP_EQ(InType(0x42.0p+0),
-                 LIBC_NAMESPACE::fputil::getpayload(func(zero, qnan_42)));
-
-    if constexpr (sizeof(OutType) < sizeof(InType)) {
-      InStorageType max_payload = InFPBits::FRACTION_MASK >> 1;
-      InType qnan_max = InFPBits::quiet_nan(Sign::POS, max_payload).get_val();
-      EXPECT_FP_EQ(zero,
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_max, zero)));
-      EXPECT_FP_EQ(zero,
-                   LIBC_NAMESPACE::fputil::getpayload(func(zero, qnan_max)));
-      EXPECT_FP_EQ(InType(0x42.0p+0),
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_max, qnan_42)));
-      EXPECT_FP_EQ(InType(0x42.0p+0),
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_42, qnan_max)));
-    }
+    EXPECT_FP_IS_NAN(func(qnan_42, zero));
+    EXPECT_FP_IS_NAN(func(zero, qnan_42));
 
     EXPECT_FP_EQ(inf, func(inf, zero));
     EXPECT_FP_EQ(neg_inf, func(neg_inf, zero));
diff --git a/libc/test/src/math/smoke/MulTest.h b/libc/test/src/math/smoke/MulTest.h
index e2298eaeeb216..0c847e39687b7 100644
--- a/libc/test/src/math/smoke/MulTest.h
+++ b/libc/test/src/math/smoke/MulTest.h
@@ -38,23 +38,8 @@ class MulTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(sNaN, sNaN), FE_INVALID);
 
     InType qnan_42 = InFPBits::quiet_nan(Sign::POS, 0x42).get_val();
-    EXPECT_FP_EQ(InType(0x42.0p+0),
-                 LIBC_NAMESPACE::fputil::getpayload(func(qnan_42, zero)));
-    EXPECT_FP_EQ(InType(0x42.0p+0),
-                 LIBC_NAMESPACE::fputil::getpayload(func(zero, qnan_42)));
-
-    if constexpr (sizeof(OutType) < sizeof(InType)) {
-      InStorageType max_payload = InFPBits::FRACTION_MASK >> 1;
-      InType qnan_max = InFPBits::quiet_nan(Sign::POS, max_payload).get_val();
-      EXPECT_FP_EQ(zero,
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_max, zero)));
-      EXPECT_FP_EQ(zero,
-                   LIBC_NAMESPACE::fputil::getpayload(func(zero, qnan_max)));
-      EXPECT_FP_EQ(InType(0x42.0p+0),
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_max, qnan_42)));
-      EXPECT_FP_EQ(InType(0x42.0p+0),
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_42, qnan_max)));
-    }
+    EXPECT_FP_IS_NAN(func(qnan_42, zero));
+    EXPECT_FP_IS_NAN(func(zero, qnan_42));
 
     EXPECT_FP_EQ(inf, func(inf, InType(1.0)));
     EXPECT_FP_EQ(neg_inf, func(neg_inf, InType(1.0)));
diff --git a/libc/test/src/math/smoke/SubTest.h b/libc/test/src/math/smoke/SubTest.h
index a4b3822aca43c..e5e04996affa8 100644
--- a/libc/test/src/math/smoke/SubTest.h
+++ b/libc/test/src/math/smoke/SubTest.h
@@ -37,23 +37,8 @@ class SubTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(sNaN, sNaN), FE_INVALID);
 
     InType qnan_42 = InFPBits::quiet_nan(Sign::POS, 0x42).get_val();
-    EXPECT_FP_EQ(InType(0x42.0p+0),
-                 LIBC_NAMESPACE::fputil::getpayload(func(qnan_42, zero)));
-    EXPECT_FP_EQ(InType(0x42.0p+0),
-                 LIBC_NAMESPACE::fputil::getpayload(func(zero, qnan_42)));
-
-    if constexpr (sizeof(OutType) < sizeof(InType)) {
-      InStorageType max_payload = InFPBits::FRACTION_MASK >> 1;
-      InType qnan_max = InFPBits::quiet_nan(Sign::POS, max_payload).get_val();
-      EXPECT_FP_EQ(zero,
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_max, zero)));
-      EXPECT_FP_EQ(zero,
-                   LIBC_NAMESPACE::fputil::getpayload(func(zero, qnan_max)));
-      EXPECT_FP_EQ(InType(0x42.0p+0),
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_max, qnan_42)));
-      EXPECT_FP_EQ(InType(0x42.0p+0),
-                   LIBC_NAMESPACE::fputil::getpayload(func(qnan_42, qnan_max)));
-    }
+    EXPECT_FP_IS_NAN(func(qnan_42, zero));
+    EXPECT_FP_IS_NAN(func(zero, qnan_42));
 
     EXPECT_FP_EQ(inf, func(inf, zero));
     EXPECT_FP_EQ(neg_inf, func(neg_inf, zero));

From eaa07c00b10b2105fabed61a95d2a5e0971bc02e Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Mon, 22 Jul 2024 02:21:05 +0200
Subject: [PATCH 360/486] [libc] Fix statvfs test case when SYS_statfs64 is
 used (#99827)

When SYS_statfs64 is used, struct statfs64 is used instead of struct statfs. This patch adds a define to select the appropriate struct, similar to how it's done internally.

This patch also enables fstatvfs and statvfs on riscv, which would not be compiled without this change.
---
 libc/config/linux/riscv/entrypoints.txt           |  4 ++++
 libc/config/linux/riscv/headers.txt               |  1 +
 libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp | 12 +++++++++---
 libc/test/src/sys/statvfs/linux/statvfs_test.cpp  | 10 ++++++++--
 4 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 61a1ef351a4c6..6c557b7a69cb6 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -266,6 +266,10 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.sys.stat.mkdirat
     libc.src.sys.stat.stat
 
+    # sys/statvfs.h
+    libc.src.sys.statvfs.fstatvfs
+    libc.src.sys.statvfs.statvfs
+
     # sys/utsname.h entrypoints
     libc.src.sys.utsname.uname
 
diff --git a/libc/config/linux/riscv/headers.txt b/libc/config/linux/riscv/headers.txt
index da203e9850603..4bb8d23ab961a 100644
--- a/libc/config/linux/riscv/headers.txt
+++ b/libc/config/linux/riscv/headers.txt
@@ -44,6 +44,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.sys_select
     libc.include.sys_socket
     libc.include.sys_stat
+    libc.include.sys_statvfs
     libc.include.sys_syscall
     libc.include.sys_time
     libc.include.sys_types
diff --git a/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp b/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
index 0895c33167151..8cb5f867453e4 100644
--- a/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
+++ b/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
@@ -9,10 +9,16 @@
 #include <linux/magic.h>
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
 
+#ifdef SYS_statfs64
+using StatFs = statfs64;
+#else
+using StatFs = statfs;
+#endif
+
 namespace LIBC_NAMESPACE_DECL {
-static int fstatfs(int fd, struct statfs *buf) {
+static int fstatfs(int fd, StatFs *buf) {
   using namespace statfs_utils;
-  if (cpp::optional<LinuxStatFs> result = linux_fstatfs(fd)) {
+  if (cpp::optional<StatFs> result = linux_fstatfs(fd)) {
     *buf = *result;
     return 0;
   }
@@ -29,7 +35,7 @@ struct PathFD {
 };
 
 TEST(LlvmLibcSysStatvfsTest, FstatfsBasic) {
-  struct statfs buf;
+  StatFs buf;
   ASSERT_THAT(LIBC_NAMESPACE::fstatfs(PathFD("/"), &buf), Succeeds());
   ASSERT_THAT(LIBC_NAMESPACE::fstatfs(PathFD("/proc"), &buf), Succeeds());
   ASSERT_EQ(buf.f_type, static_cast<decltype(buf.f_type)>(PROC_SUPER_MAGIC));
diff --git a/libc/test/src/sys/statvfs/linux/statvfs_test.cpp b/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
index 6719c1ab26865..5329adb54d64d 100644
--- a/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
+++ b/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
@@ -6,8 +6,14 @@
 #include <linux/magic.h>
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
 
+#ifdef SYS_statfs64
+using StatFs = statfs64;
+#else
+using StatFs = statfs;
+#endif
+
 namespace LIBC_NAMESPACE_DECL {
-static int statfs(const char *path, struct statfs *buf) {
+static int statfs(const char *path, StatFs *buf) {
   using namespace statfs_utils;
   if (cpp::optional<LinuxStatFs> result = linux_statfs(path)) {
     *buf = *result;
@@ -18,7 +24,7 @@ static int statfs(const char *path, struct statfs *buf) {
 } // namespace LIBC_NAMESPACE_DECL
 
 TEST(LlvmLibcSysStatfsTest, StatfsBasic) {
-  struct statfs buf;
+  StatFs buf;
   ASSERT_THAT(LIBC_NAMESPACE::statfs("/", &buf), Succeeds());
   ASSERT_THAT(LIBC_NAMESPACE::statfs("/proc", &buf), Succeeds());
   ASSERT_EQ(buf.f_type, static_cast<decltype(buf.f_type)>(PROC_SUPER_MAGIC));

From 43213002b99e32d618f2afbbaaeb2ff8dfc84e33 Mon Sep 17 00:00:00 2001
From: Chen Zheng <czhengsz@cn.ibm.com>
Date: Mon, 22 Jul 2024 08:51:51 +0800
Subject: [PATCH 361/486] [PowerPC] Support -fpatchable-function-entry (#92997)

For now only PPC big endian Linux 32 and 64 bit are supported.

PPC little endian Linux has XRAY support for 64-bit.
PPC AIX has different patchable function entry implementations.

Fixes #63220
Fixes #57031
---
 clang/include/clang/Basic/Attr.td             |  2 +-
 clang/include/clang/Basic/AttrDocs.td         |  3 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |  2 +-
 clang/lib/Driver/ToolChains/Clang.cpp         |  4 +-
 clang/lib/Sema/SemaDeclAttr.cpp               |  4 ++
 clang/test/Driver/fpatchable-function-entry.c |  9 ++-
 .../Sema/patchable-function-entry-attr.cpp    |  5 ++
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp     | 23 +++++++-
 .../PowerPC/patchable-function-entry.ll       | 58 +++++++++++++++++++
 9 files changed, 103 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/patchable-function-entry.ll

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 1293d0ddbc117..4825979a974d2 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -902,7 +902,7 @@ def PatchableFunctionEntry
     : InheritableAttr,
       TargetSpecificAttr<TargetArch<
           ["aarch64", "aarch64_be", "loongarch32", "loongarch64", "riscv32",
-           "riscv64", "x86", "x86_64"]>> {
+           "riscv64", "x86", "x86_64", "ppc", "ppc64"]>> {
   let Spellings = [GCC<"patchable_function_entry">];
   let Subjects = SubjectList<[Function, ObjCMethod]>;
   let Args = [UnsignedArgument<"Count">, DefaultIntArgument<"Offset", 0>];
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 09cf4f80bd999..99738812c8157 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -5800,7 +5800,8 @@ takes precedence over the command line option ``-fpatchable-function-entry=N,M``
 ``M`` defaults to 0 if omitted.
 
 This attribute is only supported on
-aarch64/aarch64-be/loongarch32/loongarch64/riscv32/riscv64/i386/x86-64 targets.
+aarch64/aarch64-be/loongarch32/loongarch64/riscv32/riscv64/i386/x86-64/ppc/ppc64 targets.
+For ppc/ppc64 targets, AIX is still not supported.
 }];
 }
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index d60f32674ca3a..dd9dc912ef4ca 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -3492,7 +3492,7 @@ def err_attr_tlsmodel_arg : Error<"tls_model must be \"global-dynamic\", "
 
 def err_attr_codemodel_arg : Error<"code model '%0' is not supported on this target">;
 
-def err_aix_attr_unsupported_tls_model : Error<"TLS model '%0' is not yet supported on AIX">;
+def err_aix_attr_unsupported : Error<"%0 attribute is not yet supported on AIX">;
 
 def err_tls_var_aligned_over_maximum : Error<
   "alignment (%0) of thread-local variable %1 is greater than the maximum supported "
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 364e55a628c34..601f7a355297d 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6666,7 +6666,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     StringRef S0 = A->getValue(), S = S0;
     unsigned Size, Offset = 0;
     if (!Triple.isAArch64() && !Triple.isLoongArch() && !Triple.isRISCV() &&
-        !Triple.isX86())
+        !Triple.isX86() &&
+        !(!Triple.isOSAIX() && (Triple.getArch() == llvm::Triple::ppc ||
+                                Triple.getArch() == llvm::Triple::ppc64)))
       D.Diag(diag::err_drv_unsupported_opt_for_target)
           << A->getAsString(Args) << TripleStr;
     else if (S.consumeInteger(10, Size) ||
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 50e7be9ea6020..5fd8622c90dd8 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -5247,6 +5247,10 @@ static void handleXRayLogArgsAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
 static void handlePatchableFunctionEntryAttr(Sema &S, Decl *D,
                                              const ParsedAttr &AL) {
+  if (S.Context.getTargetInfo().getTriple().isOSAIX()) {
+    S.Diag(AL.getLoc(), diag::err_aix_attr_unsupported) << AL;
+    return;
+  }
   uint32_t Count = 0, Offset = 0;
   if (!S.checkUInt32Argument(AL, AL.getArgAsExpr(0), Count, 0, true))
     return;
diff --git a/clang/test/Driver/fpatchable-function-entry.c b/clang/test/Driver/fpatchable-function-entry.c
index ab04fd39ffa1c..5f07ca99a69de 100644
--- a/clang/test/Driver/fpatchable-function-entry.c
+++ b/clang/test/Driver/fpatchable-function-entry.c
@@ -6,6 +6,8 @@
 // RUN: %clang --target=loongarch64 %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s
 // RUN: %clang --target=riscv32 %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s
 // RUN: %clang --target=riscv64 %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s
+// RUN: %clang --target=powerpc-unknown-linux-gnu %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s
+// RUN: %clang --target=powerpc64-unknown-linux-gnu %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s
 // CHECK: "-fpatchable-function-entry=1"
 
 // RUN: %clang --target=aarch64 -fsyntax-only %s -fpatchable-function-entry=1,1 -c -### 2>&1 | FileCheck --check-prefix=11 %s
@@ -13,8 +15,11 @@
 // RUN: %clang --target=aarch64 -fsyntax-only %s -fpatchable-function-entry=2,1 -c -### 2>&1 | FileCheck --check-prefix=21 %s
 // 21: "-fpatchable-function-entry=2" "-fpatchable-function-entry-offset=1"
 
-// RUN: not %clang --target=ppc64 -fsyntax-only %s -fpatchable-function-entry=1 2>&1 | FileCheck --check-prefix=TARGET %s
-// TARGET: error: unsupported option '-fpatchable-function-entry=1' for target 'ppc64'
+// RUN: not %clang --target=powerpc64-ibm-aix-xcoff -fsyntax-only %s -fpatchable-function-entry=1 2>&1 | FileCheck --check-prefix=AIX64 %s
+// AIX64: error: unsupported option '-fpatchable-function-entry=1' for target 'powerpc64-ibm-aix-xcoff'
+
+// RUN: not %clang --target=powerpc-ibm-aix-xcoff -fsyntax-only %s -fpatchable-function-entry=1 2>&1 | FileCheck --check-prefix=AIX32 %s
+// AIX32: error: unsupported option '-fpatchable-function-entry=1' for target 'powerpc-ibm-aix-xcoff'
 
 // RUN: not %clang --target=x86_64 -fsyntax-only %s -fpatchable-function-entry=1,0, 2>&1 | FileCheck --check-prefix=EXCESS %s
 // EXCESS: error: invalid argument '1,0,' to -fpatchable-function-entry=
diff --git a/clang/test/Sema/patchable-function-entry-attr.cpp b/clang/test/Sema/patchable-function-entry-attr.cpp
index 9134c851da588..bd4d57a7e3093 100644
--- a/clang/test/Sema/patchable-function-entry-attr.cpp
+++ b/clang/test/Sema/patchable-function-entry-attr.cpp
@@ -6,9 +6,14 @@
 // RUN: %clang_cc1 -triple loongarch64 -fsyntax-only -verify=silence %s
 // RUN: %clang_cc1 -triple riscv32 -fsyntax-only -verify=silence %s
 // RUN: %clang_cc1 -triple riscv64 -fsyntax-only -verify=silence %s
+// RUN: %clang_cc1 -triple powerpc-unknown-linux-gnu -fsyntax-only -verify=silence %s
+// RUN: %clang_cc1 -triple powerpc64-unknown-linux-gnu -fsyntax-only -verify=silence %s
 // RUN: %clang_cc1 -triple ppc64le -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple powerpc64-ibm-aix-xcoff -fsyntax-only -verify=AIX %s
+// RUN: %clang_cc1 -triple powerpc-ibm-aix-xcoff -fsyntax-only -verify=AIX %s
 
 // silence-no-diagnostics
 
+// AIX-error@+2 {{'patchable_function_entry' attribute is not yet supported on AIX}}
 // expected-warning@+1 {{unknown attribute 'patchable_function_entry' ignored}}
 [[gnu::patchable_function_entry(0)]] void f();
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index d74143b487880..753f0d586a0d9 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -909,6 +909,23 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
   // Lower multi-instruction pseudo operations.
   switch (MI->getOpcode()) {
   default: break;
+  case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
+    assert(!Subtarget->isAIXABI() &&
+           "AIX does not support patchable function entry!");
+    // PATCHABLE_FUNCTION_ENTER on little endian is for XRAY support which is
+    // handled in PPCLinuxAsmPrinter.
+    if (MAI->isLittleEndian())
+      return;
+    const Function &F = MF->getFunction();
+    unsigned Num = 0;
+    (void)F.getFnAttribute("patchable-function-entry")
+        .getValueAsString()
+        .getAsInteger(10, Num);
+    if (!Num)
+      return;
+    emitNops(Num);
+    return;
+  }
   case TargetOpcode::DBG_VALUE:
     llvm_unreachable("Should be handled target independently");
   case TargetOpcode::STACKMAP:
@@ -1780,7 +1797,7 @@ void PPCLinuxAsmPrinter::emitInstruction(const MachineInstr *MI) {
 
   switch (MI->getOpcode()) {
   default:
-    return PPCAsmPrinter::emitInstruction(MI);
+    break;
   case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
     // .begin:
     //   b .end # lis 0, FuncId[16..32]
@@ -1793,6 +1810,9 @@ void PPCLinuxAsmPrinter::emitInstruction(const MachineInstr *MI) {
     //
     // Update compiler-rt/lib/xray/xray_powerpc64.cc accordingly when number
     // of instructions change.
+    // XRAY is only supported on PPC Linux little endian.
+    if (!MAI->isLittleEndian())
+      break;
     MCSymbol *BeginOfSled = OutContext.createTempSymbol();
     MCSymbol *EndOfSled = OutContext.createTempSymbol();
     OutStreamer->emitLabel(BeginOfSled);
@@ -1909,6 +1929,7 @@ void PPCLinuxAsmPrinter::emitInstruction(const MachineInstr *MI) {
     llvm_unreachable("Tail call is handled in the normal case. See comments "
                      "around this assert.");
   }
+  return PPCAsmPrinter::emitInstruction(MI);
 }
 
 void PPCLinuxAsmPrinter::emitStartOfAsmFile(Module &M) {
diff --git a/llvm/test/CodeGen/PowerPC/patchable-function-entry.ll b/llvm/test/CodeGen/PowerPC/patchable-function-entry.ll
new file mode 100644
index 0000000000000..0c2d2829a6d4b
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/patchable-function-entry.ll
@@ -0,0 +1,58 @@
+; RUN: llc -mtriple=powerpc %s -o - | FileCheck %s --check-prefixes=CHECK,PPC32
+; RUN: llc -mtriple=powerpc64 %s -o - | FileCheck %s --check-prefixes=CHECK,PPC64
+
+@a = global i32 0, align 4
+
+define void @f0() {
+; CHECK-LABEL: f0:
+; CHECK-NOT:   nop
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    blr
+; CHECK-NOT:   .section    __patchable_function_entries
+  ret void
+}
+
+define void @f1() "patchable-function-entry"="0" {
+; CHECK-LABEL: f1:
+; CHECK-NOT:   nop
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    blr
+; CHECK-NOT:   .section    __patchable_function_entries
+  ret void
+}
+
+define void @f2() "patchable-function-entry"="1" {
+; CHECK-LABEL: f2:
+; CHECK-LABEL-NEXT:  .Lfunc_begin2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    blr
+; CHECK:       .section    __patchable_function_entries
+; PPC32:       .p2align    2, 0x0
+; PPC64:       .p2align    3, 0x0
+; PPC32-NEXT:  .long   .Lfunc_begin2
+; PPC64-NEXT:  .quad   .Lfunc_begin2
+  ret void
+}
+
+define i32 @f3() "patchable-function-entry"="1" "patchable-function-prefix"="2" {
+; CHECK-LABEL: .Ltmp0:
+; CHECK-COUNT-2: nop
+; CHECK-LABEL: f3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nop
+; PPC32:         lis 3, a@ha
+; PPC32-NEXT:    lwz 3, a@l(3)
+; PPC64:         addis 3, 2, .LC0@toc@ha
+; PPC64-NEXT:    ld 3, .LC0@toc@l(3)
+; PPC64-NEXT:    lwz 3, 0(3)
+; CHECK:         blr
+; CHECK:       .section    __patchable_function_entries
+; PPC32:       .p2align    2, 0x0
+; PPC64:       .p2align    3, 0x0
+; PPC32-NEXT:  .long   .Ltmp0
+; PPC64-NEXT:  .quad   .Ltmp0
+entry:
+  %0 = load i32, ptr @a, align 4
+  ret i32 %0
+}

From d15ada24b1fbbd72776022383a5c557a1a056413 Mon Sep 17 00:00:00 2001
From: Oliver Hunt <oliver@apple.com>
Date: Sun, 21 Jul 2024 18:59:33 -0700
Subject: [PATCH 362/486] [PAC] Incorrect codegen for constant global init with
 polymorphic MI (#99741)

Fixes an error where we use the wrong authentication schema for
secondary vtable pointers in constant initialized globals of types with
multiple polymorphic base classes.
---
 clang/lib/CodeGen/CGExprConstant.cpp          |   3 +-
 .../ptrauth-global-constant-initializers.cpp  | 234 ++++++++++++++++++
 2 files changed, 235 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/CodeGenCXX/ptrauth-global-constant-initializers.cpp

diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 7c65fccb60855..f22321f0e738a 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -814,8 +814,7 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
       llvm::Constant *VTableAddressPoint =
           CGM.getCXXABI().getVTableAddressPoint(BaseSubobject(CD, Offset),
                                                 VTableClass);
-      if (auto Authentication =
-              CGM.getVTablePointerAuthentication(VTableClass)) {
+      if (auto Authentication = CGM.getVTablePointerAuthentication(CD)) {
         VTableAddressPoint = Emitter.tryEmitConstantSignedPointer(
             VTableAddressPoint, *Authentication);
         if (!VTableAddressPoint)
diff --git a/clang/test/CodeGenCXX/ptrauth-global-constant-initializers.cpp b/clang/test/CodeGenCXX/ptrauth-global-constant-initializers.cpp
new file mode 100644
index 0000000000000..f0c3ea83d8958
--- /dev/null
+++ b/clang/test/CodeGenCXX/ptrauth-global-constant-initializers.cpp
@@ -0,0 +1,234 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios -fptrauth-calls -fno-rtti -fptrauth-vtable-pointer-type-discrimination -fptrauth-vtable-pointer-address-discrimination -emit-llvm -o - %s | FileCheck %s
+
+// CHECK: %struct.Base1 = type { ptr }
+// CHECK: %struct.Base2 = type { ptr }
+// CHECK: %struct.Derived1 = type { %struct.Base1, %struct.Base2 }
+// CHECK: %struct.Derived2 = type { %struct.Base2, %struct.Base1 }
+// CHECK: %struct.Derived3 = type { %struct.Base1, %struct.Base2 }
+
+// CHECK: @_ZTV5Base1 = linkonce_odr unnamed_addr constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr ptrauth (ptr @_ZN5Base11aEv, i32 0, i64 [[BASE1_A_DISC:38871]], ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV5Base1, i32 0, i32 0, i32 2))] }, align 8
+// CHECK: @g_b1 = global %struct.Base1 { ptr ptrauth (ptr getelementptr inbounds inrange(-16, 8) ({ [3 x ptr] }, ptr @_ZTV5Base1, i32 0, i32 0, i32 2), i32 2, i64 [[BASE1_VTABLE_DISC:6511]], ptr @g_b1) }, align 8
+// CHECK: @_ZTV5Base2 = linkonce_odr unnamed_addr constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr ptrauth (ptr @_ZN5Base21bEv, i32 0, i64 [[BASE2_B_DISC:27651]], ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV5Base2, i32 0, i32 0, i32 2))] }, align 8
+// CHECK: @g_b2 = global %struct.Base2 { ptr ptrauth (ptr getelementptr inbounds inrange(-16, 8) ({ [3 x ptr] }, ptr @_ZTV5Base2, i32 0, i32 0, i32 2), i32 2, i64 [[BASE2_VTABLE_DISC:63631]], ptr @g_b2) }, align 8
+// CHECK: @_ZTV8Derived1 = linkonce_odr unnamed_addr constant { [5 x ptr], [3 x ptr] } { [5 x ptr] [ptr null, ptr null, ptr ptrauth (ptr @_ZN5Base11aEv, i32 0, i64 [[BASE1_A_DISC]], ptr getelementptr inbounds ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived1, i32 0, i32 0, i32 2)), ptr ptrauth (ptr @_ZN8Derived11cEv, i32 0, i64 [[DERIVED1_C_DISC:54092]], ptr getelementptr inbounds ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived1, i32 0, i32 0, i32 3)), ptr ptrauth (ptr @_ZN8Derived11dEv, i32 0, i64 [[DERIVED1_D_DISC:37391]], ptr getelementptr inbounds ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived1, i32 0, i32 0, i32 4))], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr ptrauth (ptr @_ZN5Base21bEv, i32 0, i64 [[BASE2_B_DISC]], ptr getelementptr inbounds ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived1, i32 0, i32 1, i32 2))] }, align 8
+// CHECK: @g_d1 = global { ptr, ptr } { ptr ptrauth (ptr getelementptr inbounds inrange(-16, 24) ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived1, i32 0, i32 0, i32 2), i32 2, i64 [[BASE1_VTABLE_DISC]], ptr @g_d1), ptr ptrauth (ptr getelementptr inbounds inrange(-16, 8) ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived1, i32 0, i32 1, i32 2), i32 2, i64 [[BASE2_VTABLE_DISC]], ptr getelementptr inbounds ({ ptr, ptr }, ptr @g_d1, i32 0, i32 1)) }, align 8
+// CHECK: @_ZTV8Derived2 = linkonce_odr unnamed_addr constant { [5 x ptr], [3 x ptr] } { [5 x ptr] [ptr null, ptr null, ptr ptrauth (ptr @_ZN5Base21bEv, i32 0, i64 [[BASE2_B_DISC]], ptr getelementptr inbounds ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived2, i32 0, i32 0, i32 2)), ptr ptrauth (ptr @_ZN8Derived21cEv, i32 0, i64 [[DERIVED2_C_DISC:15537]], ptr getelementptr inbounds ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived2, i32 0, i32 0, i32 3)), ptr ptrauth (ptr @_ZN8Derived21eEv, i32 0, i64 209, ptr getelementptr inbounds ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived2, i32 0, i32 0, i32 4))], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr ptrauth (ptr @_ZN5Base11aEv, i32 0, i64 [[BASE1_A_DISC]], ptr getelementptr inbounds ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived2, i32 0, i32 1, i32 2))] }, align 8
+// CHECK: @g_d2 = global { ptr, ptr } { ptr ptrauth (ptr getelementptr inbounds inrange(-16, 24) ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived2, i32 0, i32 0, i32 2), i32 2, i64 [[BASE2_VTABLE_DISC]], ptr @g_d2), ptr ptrauth (ptr getelementptr inbounds inrange(-16, 8) ({ [5 x ptr], [3 x ptr] }, ptr @_ZTV8Derived2, i32 0, i32 1, i32 2), i32 2, i64 [[BASE1_VTABLE_DISC]], ptr getelementptr inbounds ({ ptr, ptr }, ptr @g_d2, i32 0, i32 1)) }, align 8
+// CHECK: @_ZTV8Derived3 = linkonce_odr unnamed_addr constant { [4 x ptr], [3 x ptr] } { [4 x ptr] [ptr null, ptr null, ptr ptrauth (ptr @_ZN5Base11aEv, i32 0, i64 [[BASE1_A_DISC]], ptr getelementptr inbounds ({ [4 x ptr], [3 x ptr] }, ptr @_ZTV8Derived3, i32 0, i32 0, i32 2)), ptr ptrauth (ptr @_ZN8Derived31iEv, i32 0, i64 [[DERIVED3_I_DISC:19084]], ptr getelementptr inbounds ({ [4 x ptr], [3 x ptr] }, ptr @_ZTV8Derived3, i32 0, i32 0, i32 3))], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr ptrauth (ptr @_ZN5Base21bEv, i32 0, i64 [[BASE2_B_DISC]], ptr getelementptr inbounds ({ [4 x ptr], [3 x ptr] }, ptr @_ZTV8Derived3, i32 0, i32 1, i32 2))] }, align 8
+// CHECK: @g_d3 = global { ptr, ptr } { ptr ptrauth (ptr getelementptr inbounds inrange(-16, 16) ({ [4 x ptr], [3 x ptr] }, ptr @_ZTV8Derived3, i32 0, i32 0, i32 2), i32 2, i64 [[BASE1_VTABLE_DISC]], ptr @g_d3), ptr ptrauth (ptr getelementptr inbounds inrange(-16, 8) ({ [4 x ptr], [3 x ptr] }, ptr @_ZTV8Derived3, i32 0, i32 1, i32 2), i32 2, i64 [[BASE2_VTABLE_DISC]], ptr getelementptr inbounds ({ ptr, ptr }, ptr @g_d3, i32 0, i32 1)) }, align 8
+// CHECK: @g_vb1 = global %struct.VirtualBase1 zeroinitializer, align 8
+// CHECK: @g_vb2 = global %struct.VirtualBase2 zeroinitializer, align 8
+// CHECK: @g_d4 = global %struct.Derived4 zeroinitializer, align 8
+// CHECK: @_ZTV12VirtualBase1 = linkonce_odr unnamed_addr constant { [6 x ptr] } { [6 x ptr] [ptr null, ptr null, ptr null, ptr null, ptr ptrauth (ptr @_ZN5Base11aEv, i32 0, i64 [[BASE1_A_DISC]], ptr getelementptr inbounds ({ [6 x ptr] }, ptr @_ZTV12VirtualBase1, i32 0, i32 0, i32 4)), ptr ptrauth (ptr @_ZN12VirtualBase11fEv, i32 0, i64 [[VIRTUALBASE1_F_DISC:7987]], ptr getelementptr inbounds ({ [6 x ptr] }, ptr @_ZTV12VirtualBase1, i32 0, i32 0, i32 5))] }, align 8
+// CHECK: @_ZTT12VirtualBase1 = linkonce_odr unnamed_addr constant [2 x ptr] [ptr ptrauth (ptr getelementptr inbounds inrange(-32, 16) ({ [6 x ptr] }, ptr @_ZTV12VirtualBase1, i32 0, i32 0, i32 4), i32 2), ptr ptrauth (ptr getelementptr inbounds inrange(-32, 16) ({ [6 x ptr] }, ptr @_ZTV12VirtualBase1, i32 0, i32 0, i32 4), i32 2)], align 8
+// CHECK: @_ZTV12VirtualBase2 = linkonce_odr unnamed_addr constant { [5 x ptr], [4 x ptr] } { [5 x ptr] [ptr inttoptr (i64 8 to ptr), ptr null, ptr null, ptr ptrauth (ptr @_ZN5Base21bEv, i32 0, i64 [[BASE2_B_DISC]], ptr getelementptr inbounds ({ [5 x ptr], [4 x ptr] }, ptr @_ZTV12VirtualBase2, i32 0, i32 0, i32 3)), ptr ptrauth (ptr @_ZN12VirtualBase21gEv, i32 0, i64 [[VIRTUALBASE2_G_DISC:51224]], ptr getelementptr inbounds ({ [5 x ptr], [4 x ptr] }, ptr @_ZTV12VirtualBase2, i32 0, i32 0, i32 4))], [4 x ptr] [ptr null, ptr inttoptr (i64 -8 to ptr), ptr null, ptr ptrauth (ptr @_ZN5Base11aEv, i32 0, i64 [[BASE1_A_DISC]], ptr getelementptr inbounds ({ [5 x ptr], [4 x ptr] }, ptr @_ZTV12VirtualBase2, i32 0, i32 1, i32 3))] }, align 8
+// CHECK: @_ZTT12VirtualBase2 = linkonce_odr unnamed_addr constant [2 x ptr] [ptr ptrauth (ptr getelementptr inbounds inrange(-24, 16) ({ [5 x ptr], [4 x ptr] }, ptr @_ZTV12VirtualBase2, i32 0, i32 0, i32 3), i32 2), ptr ptrauth (ptr getelementptr inbounds inrange(-24, 8) ({ [5 x ptr], [4 x ptr] }, ptr @_ZTV12VirtualBase2, i32 0, i32 1, i32 3), i32 2)], align 8
+// CHECK: @_ZTV8Derived4 = linkonce_odr unnamed_addr constant { [7 x ptr], [5 x ptr] } { [7 x ptr] [ptr null, ptr null, ptr null, ptr null, ptr ptrauth (ptr @_ZN5Base11aEv, i32 0, i64 [[BASE1_A_DISC]], ptr getelementptr inbounds ({ [7 x ptr], [5 x ptr] }, ptr @_ZTV8Derived4, i32 0, i32 0, i32 4)), ptr ptrauth (ptr @_ZN12VirtualBase11fEv, i32 0, i64 [[VIRTUALBASE1_F_DISC]], ptr getelementptr inbounds ({ [7 x ptr], [5 x ptr] }, ptr @_ZTV8Derived4, i32 0, i32 0, i32 5)), ptr ptrauth (ptr @_ZN8Derived41hEv, i32 0, i64 [[DERIVED4_H_DISC:31844]], ptr getelementptr inbounds ({ [7 x ptr], [5 x ptr] }, ptr @_ZTV8Derived4, i32 0, i32 0, i32 6))], [5 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr inttoptr (i64 -8 to ptr), ptr null, ptr ptrauth (ptr @_ZN5Base21bEv, i32 0, i64 [[BASE2_B_DISC]], ptr getelementptr inbounds ({ [7 x ptr], [5 x ptr] }, ptr @_ZTV8Derived4, i32 0, i32 1, i32 3)), ptr ptrauth (ptr @_ZN12VirtualBase21gEv, i32 0, i64 [[VIRTUALBASE2_G_DISC]], ptr getelementptr inbounds ({ [7 x ptr], [5 x ptr] }, ptr @_ZTV8Derived4, i32 0, i32 1, i32 4))] }, align 8
+// CHECK: @_ZTT8Derived4 = linkonce_odr unnamed_addr constant [7 x ptr] [ptr ptrauth (ptr getelementptr inbounds inrange(-32, 24) ({ [7 x ptr], [5 x ptr] }, ptr @_ZTV8Derived4, i32 0, i32 0, i32 4), i32 2), ptr ptrauth (ptr getelementptr inbounds inrange(-32, 16) ({ [6 x ptr] }, ptr @_ZTC8Derived40_12VirtualBase1, i32 0, i32 0, i32 4), i32 2), ptr ptrauth (ptr getelementptr inbounds inrange(-32, 16) ({ [6 x ptr] }, ptr @_ZTC8Derived40_12VirtualBase1, i32 0, i32 0, i32 4), i32 2), ptr ptrauth (ptr getelementptr inbounds inrange(-24, 16) ({ [5 x ptr], [4 x ptr] }, ptr @_ZTC8Derived48_12VirtualBase2, i32 0, i32 0, i32 3), i32 2), ptr ptrauth (ptr getelementptr inbounds inrange(-24, 8) ({ [5 x ptr], [4 x ptr] }, ptr @_ZTC8Derived48_12VirtualBase2, i32 0, i32 1, i32 3), i32 2), ptr ptrauth (ptr getelementptr inbounds inrange(-32, 24) ({ [7 x ptr], [5 x ptr] }, ptr @_ZTV8Derived4, i32 0, i32 0, i32 4), i32 2), ptr ptrauth (ptr getelementptr inbounds inrange(-24, 16) ({ [7 x ptr], [5 x ptr] }, ptr @_ZTV8Derived4, i32 0, i32 1, i32 3), i32 2)], align 8
+// CHECK: @_ZTC8Derived40_12VirtualBase1 = linkonce_odr unnamed_addr constant { [6 x ptr] } { [6 x ptr] [ptr null, ptr null, ptr null, ptr null, ptr ptrauth (ptr @_ZN5Base11aEv, i32 0, i64 [[BASE1_A_DISC]], ptr getelementptr inbounds ({ [6 x ptr] }, ptr @_ZTC8Derived40_12VirtualBase1, i32 0, i32 0, i32 4)), ptr ptrauth (ptr @_ZN12VirtualBase11fEv, i32 0, i64 [[VIRTUALBASE1_F_DISC]], ptr getelementptr inbounds ({ [6 x ptr] }, ptr @_ZTC8Derived40_12VirtualBase1, i32 0, i32 0, i32 5))] }, align 8
+// CHECK: @_ZTC8Derived48_12VirtualBase2 = linkonce_odr unnamed_addr constant { [5 x ptr], [4 x ptr] } { [5 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr null, ptr ptrauth (ptr @_ZN5Base21bEv, i32 0, i64 [[BASE2_B_DISC]], ptr getelementptr inbounds ({ [5 x ptr], [4 x ptr] }, ptr @_ZTC8Derived48_12VirtualBase2, i32 0, i32 0, i32 3)), ptr ptrauth (ptr @_ZN12VirtualBase21gEv, i32 0, i64 [[VIRTUALBASE2_G_DISC]], ptr getelementptr inbounds ({ [5 x ptr], [4 x ptr] }, ptr @_ZTC8Derived48_12VirtualBase2, i32 0, i32 0, i32 4))], [4 x ptr] [ptr null, ptr inttoptr (i64 8 to ptr), ptr null, ptr ptrauth (ptr @_ZN5Base11aEv, i32 0, i64 [[BASE1_A_DISC]], ptr getelementptr inbounds ({ [5 x ptr], [4 x ptr] }, ptr @_ZTC8Derived48_12VirtualBase2, i32 0, i32 1, i32 3))] }, align 8
+
+struct Base1 { virtual void a() {} };
+struct Base2 { virtual void b() {} };
+struct Derived1 : public Base1, public Base2 {
+  virtual void c() {}
+  virtual void d() {}
+};
+struct Derived2 : public Base2, public Base1 {
+  virtual void c() {}
+  virtual void e() {}
+};
+
+struct Derived3 : public Base1, public Base2 {
+  constexpr Derived3(){}
+  virtual void i() {}
+};
+
+Base1 g_b1;
+Base2 g_b2;
+Derived1 g_d1;
+Derived2 g_d2;
+Derived3 g_d3;
+
+extern "C" void test_basic_inheritance() {
+  Base1 g_b1;
+  Base2 g_b2;
+  Derived1 g_d1;
+  Derived2 g_d2;
+  Derived3 g_d3;
+}
+
+struct VirtualBase1 : virtual Base1 {
+  VirtualBase1(){}
+  virtual void f() {}
+};
+struct VirtualBase2 : virtual Base1, Base2 {
+  VirtualBase2(){}
+  virtual void g() {}
+};
+struct Derived4 : VirtualBase1, VirtualBase2 {
+  virtual void h() {}
+};
+struct Derived5 : VirtualBase2, VirtualBase1 {
+  virtual void h() {}
+};
+
+// CHECK-LABEL: define {{.*}} ptr @_ZN12VirtualBase1C1Ev
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+
+// CHECK-LABEL: define {{.*}} ptr @_ZN12VirtualBase2C1Ev
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+
+// CHECK-LABEL: define {{.*}} ptr @_ZN8Derived4C1Ev
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+
+// CHECK-LABEL: define {{.*}} ptr @_ZN8Derived5C1Ev
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+
+
+VirtualBase1 g_vb1;
+VirtualBase2 g_vb2;
+Derived4 g_d4;
+Derived5 g_d5;
+
+
+extern "C" void cross_check_vtables(Base1 *b1,
+                   Base2 *b2,
+                   Derived1 *d1,
+                   Derived2 *d2,
+                   Derived3 *d3,
+                   VirtualBase1 *vb1,
+                   VirtualBase2 *vb2,
+                   Derived4 *d4,
+                   Derived4 *d5) {
+  asm("; b1->a()" ::: "memory");
+  b1->a();
+  asm("; b2->b()" ::: "memory");
+  b2->b();
+  asm("; d1->a()" ::: "memory");
+  d1->a();
+  asm("; d1->c()" ::: "memory");
+  d1->c();
+  asm("; d2->a()" ::: "memory");
+  d2->a();
+  asm("; d2->c()" ::: "memory");
+  d2->c();
+  asm("; d3->a()" ::: "memory");
+  d3->a();
+  asm("; d3->b()" ::: "memory");
+  d3->b();
+  asm("; d3->i()" ::: "memory");
+  d3->i();
+  asm("; vb1->a()" ::: "memory");
+  vb1->a();
+  asm("; vb1->f()" ::: "memory");
+  vb1->f();
+  asm("; vb2->a()" ::: "memory");
+  vb2->a();
+  asm("; vb2->g()" ::: "memory");
+  vb2->g();
+  asm("; d4->a()" ::: "memory");
+  d4->a();
+  asm("; d4->b()" ::: "memory");
+  d4->b();
+  asm("; d4->f()" ::: "memory");
+  d4->f();
+  asm("; d4->g()" ::: "memory");
+  d4->g();
+  asm("; d4->h()" ::: "memory");
+  d4->h();
+  asm("; d5->a()" ::: "memory");
+  d5->a();
+  asm("; d5->b()" ::: "memory");
+  d5->b();
+  asm("; d5->f()" ::: "memory");
+  d5->f();
+  asm("; d5->g()" ::: "memory");
+  d5->g();
+  asm("; d5->h()" ::: "memory");
+  d5->h();
+}
+
+// CHECK-LABEL: define void @cross_check_vtables(
+// CHECK: "; b1->a()",
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_A_DISC]])
+// CHECK: "; b2->b()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_B_DISC]])
+// CHECK: "; d1->a()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_A_DISC]])
+// CHECK: "; d1->c()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[DERIVED1_C_DISC]])
+// CHECK: "; d2->a()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_A_DISC]])
+// CHECK: "; d2->c()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[DERIVED2_C_DISC]])
+// CHECK: "; d3->a()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_A_DISC]])
+// CHECK: "; d3->b()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_B_DISC]])
+// CHECK: "; d3->i()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[DERIVED3_I_DISC]])
+// CHECK: "; vb1->a()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_A_DISC]])
+// CHECK: "; vb1->f()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[VIRTUALBASE1_F_DISC]])
+// CHECK: "; vb2->a()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_A_DISC]])
+// CHECK: "; vb2->g()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[VIRTUALBASE2_G_DISC]])
+// CHECK: "; d4->a()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_A_DISC]])
+// CHECK: "; d4->b()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_B_DISC]])
+// CHECK: "; d4->f()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[VIRTUALBASE1_F_DISC]])
+// CHECK: "; d4->g()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[VIRTUALBASE2_G_DISC]])
+// CHECK: "; d4->h()"
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[DERIVED4_H_DISC]])
+
+// CHECK-LABEL: define {{.*}} ptr @_ZN5Base1C2Ev
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+
+// CHECK-LABEL: define {{.*}} ptr @_ZN5Base2C2Ev
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+
+// CHECK-LABEL: define {{.*}} ptr @_ZN8Derived1C2Ev
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+
+// CHECK-LABEL: define {{.*}} ptr @_ZN8Derived2C2Ev
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+
+// CHECK-LABEL: define {{.*}} ptr @_ZN8Derived3C2Ev
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE1_VTABLE_DISC]])
+// CHECK: call i64 @llvm.ptrauth.blend(i64 {{%.*}}, i64 [[BASE2_VTABLE_DISC]])
+

From 197b14223238716a84b423315242f8b10d8922dd Mon Sep 17 00:00:00 2001
From: jameshu15869 <55058507+jameshu15869@users.noreply.github.com>
Date: Sun, 21 Jul 2024 21:56:40 -0500
Subject: [PATCH 363/486] [libc] Add N Threads Benchmark Helper (#99834)

This PR adds a `BENCHMARK_N_THREADS()` helper to register benchmarks
with a specific number of threads. This PR replaces the flags used
originally to allow any amount of threads.
---
 libc/benchmarks/gpu/LibcGpuBenchmark.cpp |  5 +----
 libc/benchmarks/gpu/LibcGpuBenchmark.h   | 26 +++++++++++++-----------
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
index 59de18c20417d..ac139cc21b0b6 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
@@ -151,10 +151,7 @@ void Benchmark::run_benchmarks() {
       all_results.reset();
 
     gpu::sync_threads();
-    if (!b->flags ||
-        ((b->flags & BenchmarkFlags::SINGLE_THREADED) && id == 0) ||
-        ((b->flags & BenchmarkFlags::SINGLE_WAVE) &&
-         id < gpu::get_lane_size())) {
+    if (b->num_threads == static_cast<uint32_t>(-1) || id < b->num_threads) {
       auto current_result = b->run();
       all_results.update(current_result);
     }
diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.h b/libc/benchmarks/gpu/LibcGpuBenchmark.h
index c07fab9ccfbe3..f5cf4822f6fd3 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.h
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.h
@@ -74,8 +74,6 @@ struct BenchmarkResult {
   clock_t total_time = 0;
 };
 
-enum BenchmarkFlags { SINGLE_THREADED = 0x1, SINGLE_WAVE = 0x2 };
-
 BenchmarkResult benchmark(const BenchmarkOptions &options,
                           cpp::function<uint64_t(void)> wrapper_func);
 
@@ -83,12 +81,13 @@ class Benchmark {
   const cpp::function<uint64_t(void)> func;
   const cpp::string_view suite_name;
   const cpp::string_view test_name;
-  const uint8_t flags;
+  const uint32_t num_threads;
 
 public:
   Benchmark(cpp::function<uint64_t(void)> func, char const *suite_name,
-            char const *test_name, uint8_t flags)
-      : func(func), suite_name(suite_name), test_name(test_name), flags(flags) {
+            char const *test_name, uint32_t num_threads)
+      : func(func), suite_name(suite_name), test_name(test_name),
+        num_threads(num_threads) {
     add_benchmark(this);
   }
 
@@ -108,18 +107,21 @@ class Benchmark {
 } // namespace benchmarks
 } // namespace LIBC_NAMESPACE_DECL
 
+// Passing -1 indicates the benchmark should be run with as many threads as
+// allocated by the user in the benchmark's CMake.
 #define BENCHMARK(SuiteName, TestName, Func)                                   \
   LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance(     \
-      Func, #SuiteName, #TestName, 0)
+      Func, #SuiteName, #TestName, -1)
 
-#define SINGLE_THREADED_BENCHMARK(SuiteName, TestName, Func)                   \
+#define BENCHMARK_N_THREADS(SuiteName, TestName, Func, NumThreads)             \
   LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance(     \
-      Func, #SuiteName, #TestName,                                             \
-      LIBC_NAMESPACE::benchmarks::BenchmarkFlags::SINGLE_THREADED)
+      Func, #SuiteName, #TestName, NumThreads)
+
+#define SINGLE_THREADED_BENCHMARK(SuiteName, TestName, Func)                   \
+  BENCHMARK_N_THREADS(SuiteName, TestName, Func, 1)
 
 #define SINGLE_WAVE_BENCHMARK(SuiteName, TestName, Func)                       \
-  LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance(     \
-      Func, #SuiteName, #TestName,                                             \
-      LIBC_NAMESPACE::benchmarks::BenchmarkFlags::SINGLE_WAVE)
+  BENCHMARK_N_THREADS(SuiteName, TestName, Func,                               \
+                      LIBC_NAMESPACE::gpu::get_lane_size())
 
 #endif

From 324fea9baa902b2bff7b644fade080f98a8c543b Mon Sep 17 00:00:00 2001
From: donald chen <chenxunyu1993@gmail.com>
Date: Mon, 22 Jul 2024 11:10:27 +0800
Subject: [PATCH 364/486] [mlir][docs] Update documentation for canonicalize.
 (#99753)

Update canonicalize docs.
---
 mlir/docs/Canonicalization.md | 39 ++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/mlir/docs/Canonicalization.md b/mlir/docs/Canonicalization.md
index d1cba572af212..03fd174229afe 100644
--- a/mlir/docs/Canonicalization.md
+++ b/mlir/docs/Canonicalization.md
@@ -33,6 +33,10 @@ together.
 
 Some important things to think about w.r.t. canonicalization patterns:
 
+*   The goal of canonicalization is to make subsequent analyses and
+    optimizations more effective. Therefore, performance improvements are not
+    necessary for canonicalization.
+
 *   Pass pipelines should not rely on the canonicalizer pass for correctness.
     They should work correctly with all instances of the canonicalization pass
     removed.
@@ -51,6 +55,39 @@ Some important things to think about w.r.t. canonicalization patterns:
 *   It is always good to eliminate operations entirely when possible, e.g. by
     folding known identities (like "x + 0 = x").
 
+*   Pattens with expensive running time (i.e. have O(n) complexity) or
+    complicated cost models don't belong to canonicalization: since the
+    algorithm is executed iteratively until fixed-point we want patterns that
+    execute quickly (in particular their matching phase).
+
+*   Canonicalize shouldn't lose the semantic of original operation: the original
+    information should always be recoverable from the transformed IR.
+
+For example, a pattern that transform
+
+```
+  %transpose = linalg.transpose
+      ins(%input : tensor<1x2x3xf32>)
+      outs(%init1 : tensor<2x1x3xf32>)
+      dimensions = [1, 0, 2]
+  %out = linalg.transpose
+      ins(%tranpose: tensor<2x1x3xf32>)
+      outs(%init2 : tensor<3x1x2xf32>)
+      permutation = [2, 1, 0]
+```
+
+to
+
+```
+  %out= linalg.transpose
+      ins(%input : tensor<1x2x3xf32>)
+      outs(%init2: tensor<3x1x2xf32>)
+      permutation = [2, 0, 1]
+```
+
+is a good canonicalization pattern because it removes a redundant operation,
+making other analysis optimizations and more efficient.
+
 ## Globally Applied Rules
 
 These transformations are applied to all levels of IR:
@@ -189,7 +226,7 @@ each of the operands, returning the corresponding constant attribute. These
 operands are those that implement the `ConstantLike` trait. If any of the
 operands are non-constant, a null `Attribute` value is provided instead. For
 example, if MyOp provides three operands [`a`, `b`, `c`], but only `b` is
-constant then `adaptor` will return Attribute() for `getA()` and `getC()`, 
+constant then `adaptor` will return Attribute() for `getA()` and `getC()`,
 and b-value for `getB()`.
 
 Also above, is the use of `OpFoldResult`. This class represents the possible

From 76321b9f08ef31a2b8ca26f7522aee511a05f7a8 Mon Sep 17 00:00:00 2001
From: Connor Sughrue <55301806+cpsughrue@users.noreply.github.com>
Date: Sun, 21 Jul 2024 23:50:28 -0400
Subject: [PATCH 365/486] [llvm][Support] Implement raw_socket_stream::read
 with optional timeout (#92308)

This PR implements `raw_socket_stream::read`, which overloads the base
class `raw_fd_stream::read`. `raw_socket_stream::read` provides a way to
timeout the underlying `::read`. The timeout functionality was not added
to `raw_fd_stream::read` to avoid needlessly increasing compile times
and allow for convenient code reuse with `raw_socket_stream::accept`,
which also requires timeout functionality. This PR supports the module
build daemon and will help guarantee it never becomes a zombie process.
---
 llvm/include/llvm/Support/raw_socket_stream.h |  28 +++-
 llvm/lib/Support/raw_socket_stream.cpp        | 132 +++++++++++-------
 .../Support/raw_socket_stream_test.cpp        |  45 +++++-
 3 files changed, 144 insertions(+), 61 deletions(-)

diff --git a/llvm/include/llvm/Support/raw_socket_stream.h b/llvm/include/llvm/Support/raw_socket_stream.h
index eed865fb5af49..6c65a66dec9a4 100644
--- a/llvm/include/llvm/Support/raw_socket_stream.h
+++ b/llvm/include/llvm/Support/raw_socket_stream.h
@@ -92,13 +92,14 @@ class ListeningSocket {
   /// Accepts an incoming connection on the listening socket. This method can
   /// optionally either block until a connection is available or timeout after a
   /// specified amount of time has passed. By default the method will block
-  /// until the socket has recieved a connection.
+  /// until the socket has recieved a connection. If the accept timesout this
+  /// method will return std::errc:timed_out
   ///
   /// \param Timeout An optional timeout duration in milliseconds. Setting
-  /// Timeout to -1 causes accept to block indefinitely
+  /// Timeout to a negative number causes ::accept to block indefinitely
   ///
-  Expected<std::unique_ptr<raw_socket_stream>>
-  accept(std::chrono::milliseconds Timeout = std::chrono::milliseconds(-1));
+  Expected<std::unique_ptr<raw_socket_stream>> accept(
+      const std::chrono::milliseconds &Timeout = std::chrono::milliseconds(-1));
 
   /// Creates a listening socket bound to the specified file system path.
   /// Handles the socket creation, binding, and immediately starts listening for
@@ -124,11 +125,28 @@ class raw_socket_stream : public raw_fd_stream {
 
 public:
   raw_socket_stream(int SocketFD);
+  ~raw_socket_stream();
+
   /// Create a \p raw_socket_stream connected to the UNIX domain socket at \p
   /// SocketPath.
   static Expected<std::unique_ptr<raw_socket_stream>>
   createConnectedUnix(StringRef SocketPath);
-  ~raw_socket_stream();
+
+  /// Attempt to read from the raw_socket_stream's file descriptor.
+  ///
+  /// This method can optionally either block until data is read or an error has
+  /// occurred or timeout after a specified amount of time has passed. By
+  /// default the method will block until the socket has read data or
+  /// encountered an error. If the read times out this method will return
+  /// std::errc:timed_out
+  ///
+  /// \param Ptr The start of the buffer that will hold any read data
+  /// \param Size The number of bytes to be read
+  /// \param Timeout An optional timeout duration in milliseconds
+  ///
+  ssize_t read(
+      char *Ptr, size_t Size,
+      const std::chrono::milliseconds &Timeout = std::chrono::milliseconds(-1));
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Support/raw_socket_stream.cpp b/llvm/lib/Support/raw_socket_stream.cpp
index 4cd3d58b80198..04b3233084a41 100644
--- a/llvm/lib/Support/raw_socket_stream.cpp
+++ b/llvm/lib/Support/raw_socket_stream.cpp
@@ -18,6 +18,7 @@
 
 #include <atomic>
 #include <fcntl.h>
+#include <functional>
 #include <thread>
 
 #ifndef _WIN32
@@ -177,70 +178,89 @@ Expected<ListeningSocket> ListeningSocket::createUnix(StringRef SocketPath,
 #endif // _WIN32
 }
 
-Expected<std::unique_ptr<raw_socket_stream>>
-ListeningSocket::accept(std::chrono::milliseconds Timeout) {
-
-  struct pollfd FDs[2];
-  FDs[0].events = POLLIN;
+// If a file descriptor being monitored by ::poll is closed by another thread,
+// the result is unspecified. In the case ::poll does not unblock and return,
+// when ActiveFD is closed, you can provide another file descriptor via CancelFD
+// that when written to will cause poll to return. Typically CancelFD is the
+// read end of a unidirectional pipe.
+//
+// Timeout should be -1 to block indefinitly
+//
+// getActiveFD is a callback to handle ActiveFD's of std::atomic<int> and int
+static std::error_code
+manageTimeout(const std::chrono::milliseconds &Timeout,
+              const std::function<int()> &getActiveFD,
+              const std::optional<int> &CancelFD = std::nullopt) {
+  struct pollfd FD[2];
+  FD[0].events = POLLIN;
 #ifdef _WIN32
-  SOCKET WinServerSock = _get_osfhandle(FD);
-  FDs[0].fd = WinServerSock;
+  SOCKET WinServerSock = _get_osfhandle(getActiveFD());
+  FD[0].fd = WinServerSock;
 #else
-  FDs[0].fd = FD;
+  FD[0].fd = getActiveFD();
 #endif
-  FDs[1].events = POLLIN;
-  FDs[1].fd = PipeFD[0];
-
-  // Keep track of how much time has passed in case poll is interupted by a
-  // signal and needs to be recalled
-  int RemainingTime = Timeout.count();
-  std::chrono::milliseconds ElapsedTime = std::chrono::milliseconds(0);
-  int PollStatus = -1;
-
-  while (PollStatus == -1 && (Timeout.count() == -1 || ElapsedTime < Timeout)) {
-    if (Timeout.count() != -1)
-      RemainingTime -= ElapsedTime.count();
+  uint8_t FDCount = 1;
+  if (CancelFD.has_value()) {
+    FD[1].events = POLLIN;
+    FD[1].fd = CancelFD.value();
+    FDCount++;
+  }
 
-    auto Start = std::chrono::steady_clock::now();
+  // Keep track of how much time has passed in case ::poll or WSAPoll are
+  // interupted by a signal and need to be recalled
+  auto Start = std::chrono::steady_clock::now();
+  auto RemainingTimeout = Timeout;
+  int PollStatus = 0;
+  do {
+    // If Timeout is -1 then poll should block and RemainingTimeout does not
+    // need to be recalculated
+    if (PollStatus != 0 && Timeout != std::chrono::milliseconds(-1)) {
+      auto TotalElapsedTime =
+          std::chrono::duration_cast<std::chrono::milliseconds>(
+              std::chrono::steady_clock::now() - Start);
+
+      if (TotalElapsedTime >= Timeout)
+        return std::make_error_code(std::errc::operation_would_block);
+
+      RemainingTimeout = Timeout - TotalElapsedTime;
+    }
 #ifdef _WIN32
-    PollStatus = WSAPoll(FDs, 2, RemainingTime);
+    PollStatus = WSAPoll(FD, FDCount, RemainingTimeout.count());
+  } while (PollStatus == SOCKET_ERROR &&
+           getLastSocketErrorCode() == std::errc::interrupted);
 #else
-    PollStatus = ::poll(FDs, 2, RemainingTime);
+    PollStatus = ::poll(FD, FDCount, RemainingTimeout.count());
+  } while (PollStatus == -1 &&
+           getLastSocketErrorCode() == std::errc::interrupted);
 #endif
-    // If FD equals -1 then ListeningSocket::shutdown has been called and it is
-    // appropriate to return operation_canceled
-    if (FD.load() == -1)
-      return llvm::make_error<StringError>(
-          std::make_error_code(std::errc::operation_canceled),
-          "Accept canceled");
 
+  // If ActiveFD equals -1 or CancelFD has data to be read then the operation
+  // has been canceled by another thread
+  if (getActiveFD() == -1 || (CancelFD.has_value() && FD[1].revents & POLLIN))
+    return std::make_error_code(std::errc::operation_canceled);
 #if _WIN32
-    if (PollStatus == SOCKET_ERROR) {
+  if (PollStatus == SOCKET_ERROR)
 #else
-    if (PollStatus == -1) {
+  if (PollStatus == -1)
 #endif
-      std::error_code PollErrCode = getLastSocketErrorCode();
-      // Ignore EINTR (signal occured before any request event) and retry
-      if (PollErrCode != std::errc::interrupted)
-        return llvm::make_error<StringError>(PollErrCode, "FD poll failed");
-    }
-    if (PollStatus == 0)
-      return llvm::make_error<StringError>(
-          std::make_error_code(std::errc::timed_out),
-          "No client requests within timeout window");
-
-    if (FDs[0].revents & POLLNVAL)
-      return llvm::make_error<StringError>(
-          std::make_error_code(std::errc::bad_file_descriptor));
+    return getLastSocketErrorCode();
+  if (PollStatus == 0)
+    return std::make_error_code(std::errc::timed_out);
+  if (FD[0].revents & POLLNVAL)
+    return std::make_error_code(std::errc::bad_file_descriptor);
+  return std::error_code();
+}
 
-    auto Stop = std::chrono::steady_clock::now();
-    ElapsedTime +=
-        std::chrono::duration_cast<std::chrono::milliseconds>(Stop - Start);
-  }
+Expected<std::unique_ptr<raw_socket_stream>>
+ListeningSocket::accept(const std::chrono::milliseconds &Timeout) {
+  auto getActiveFD = [this]() -> int { return FD; };
+  std::error_code TimeoutErr = manageTimeout(Timeout, getActiveFD, PipeFD[0]);
+  if (TimeoutErr)
+    return llvm::make_error<StringError>(TimeoutErr, "Timeout error");
 
   int AcceptFD;
 #ifdef _WIN32
-  SOCKET WinAcceptSock = ::accept(WinServerSock, NULL, NULL);
+  SOCKET WinAcceptSock = ::accept(_get_osfhandle(FD), NULL, NULL);
   AcceptFD = _open_osfhandle(WinAcceptSock, 0);
 #else
   AcceptFD = ::accept(FD, NULL, NULL);
@@ -295,6 +315,8 @@ ListeningSocket::~ListeningSocket() {
 raw_socket_stream::raw_socket_stream(int SocketFD)
     : raw_fd_stream(SocketFD, true) {}
 
+raw_socket_stream::~raw_socket_stream() {}
+
 Expected<std::unique_ptr<raw_socket_stream>>
 raw_socket_stream::createConnectedUnix(StringRef SocketPath) {
 #ifdef _WIN32
@@ -306,4 +328,14 @@ raw_socket_stream::createConnectedUnix(StringRef SocketPath) {
   return std::make_unique<raw_socket_stream>(*FD);
 }
 
-raw_socket_stream::~raw_socket_stream() {}
+ssize_t raw_socket_stream::read(char *Ptr, size_t Size,
+                                const std::chrono::milliseconds &Timeout) {
+  auto getActiveFD = [this]() -> int { return this->get_fd(); };
+  std::error_code Err = manageTimeout(Timeout, getActiveFD);
+  // Mimic raw_fd_stream::read error handling behavior
+  if (Err) {
+    raw_fd_stream::error_detected(Err);
+    return -1;
+  }
+  return raw_fd_stream::read(Ptr, Size);
+}
diff --git a/llvm/unittests/Support/raw_socket_stream_test.cpp b/llvm/unittests/Support/raw_socket_stream_test.cpp
index c4e8cfbbe7e6a..348fb4bb3e089 100644
--- a/llvm/unittests/Support/raw_socket_stream_test.cpp
+++ b/llvm/unittests/Support/raw_socket_stream_test.cpp
@@ -62,17 +62,50 @@ TEST(raw_socket_streamTest, CLIENT_TO_SERVER_AND_SERVER_TO_CLIENT) {
   ssize_t BytesRead = Server.read(Bytes, 8);
 
   std::string string(Bytes, 8);
+  ASSERT_EQ(Server.has_error(), false);
 
   ASSERT_EQ(8, BytesRead);
   ASSERT_EQ("01234567", string);
 }
 
-TEST(raw_socket_streamTest, TIMEOUT_PROVIDED) {
+TEST(raw_socket_streamTest, READ_WITH_TIMEOUT) {
   if (!hasUnixSocketSupport())
     GTEST_SKIP();
 
   SmallString<100> SocketPath;
-  llvm::sys::fs::createUniquePath("timout_provided.sock", SocketPath, true);
+  llvm::sys::fs::createUniquePath("read_with_timeout.sock", SocketPath, true);
+
+  // Make sure socket file does not exist. May still be there from the last test
+  std::remove(SocketPath.c_str());
+
+  Expected<ListeningSocket> MaybeServerListener =
+      ListeningSocket::createUnix(SocketPath);
+  ASSERT_THAT_EXPECTED(MaybeServerListener, llvm::Succeeded());
+  ListeningSocket ServerListener = std::move(*MaybeServerListener);
+
+  Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
+      raw_socket_stream::createConnectedUnix(SocketPath);
+  ASSERT_THAT_EXPECTED(MaybeClient, llvm::Succeeded());
+
+  Expected<std::unique_ptr<raw_socket_stream>> MaybeServer =
+      ServerListener.accept();
+  ASSERT_THAT_EXPECTED(MaybeServer, llvm::Succeeded());
+  raw_socket_stream &Server = **MaybeServer;
+
+  char Bytes[8];
+  ssize_t BytesRead = Server.read(Bytes, 8, std::chrono::milliseconds(100));
+  ASSERT_EQ(BytesRead, -1);
+  ASSERT_EQ(Server.has_error(), true);
+  ASSERT_EQ(Server.error(), std::errc::timed_out);
+  Server.clear_error();
+}
+
+TEST(raw_socket_streamTest, ACCEPT_WITH_TIMEOUT) {
+  if (!hasUnixSocketSupport())
+    GTEST_SKIP();
+
+  SmallString<100> SocketPath;
+  llvm::sys::fs::createUniquePath("accept_with_timeout.sock", SocketPath, true);
 
   // Make sure socket file does not exist. May still be there from the last test
   std::remove(SocketPath.c_str());
@@ -82,19 +115,19 @@ TEST(raw_socket_streamTest, TIMEOUT_PROVIDED) {
   ASSERT_THAT_EXPECTED(MaybeServerListener, llvm::Succeeded());
   ListeningSocket ServerListener = std::move(*MaybeServerListener);
 
-  std::chrono::milliseconds Timeout = std::chrono::milliseconds(100);
   Expected<std::unique_ptr<raw_socket_stream>> MaybeServer =
-      ServerListener.accept(Timeout);
+      ServerListener.accept(std::chrono::milliseconds(100));
   ASSERT_EQ(llvm::errorToErrorCode(MaybeServer.takeError()),
             std::errc::timed_out);
 }
 
-TEST(raw_socket_streamTest, FILE_DESCRIPTOR_CLOSED) {
+TEST(raw_socket_streamTest, ACCEPT_WITH_SHUTDOWN) {
   if (!hasUnixSocketSupport())
     GTEST_SKIP();
 
   SmallString<100> SocketPath;
-  llvm::sys::fs::createUniquePath("fd_closed.sock", SocketPath, true);
+  llvm::sys::fs::createUniquePath("accept_with_shutdown.sock", SocketPath,
+                                  true);
 
   // Make sure socket file does not exist. May still be there from the last test
   std::remove(SocketPath.c_str());

From 248fcab2fc9b3fc1bde5cd5b1fe8615791225c9e Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Mon, 22 Jul 2024 11:59:54 +0800
Subject: [PATCH 366/486]  [InstCombine] Do not use operand info in
 `replaceInInstruction` (#99492)

Consider the following case:
```
%cmp = icmp eq ptr %p, null
%load = load i32, ptr %p, align 4
%sel = select i1 %cmp, i32 %load, i32 0
```
`foldSelectValueEquivalence` converts `load i32, ptr %p, align 4` into
`load i32, ptr null, align 4`, which causes immediate UB. `%load` is
speculatable, but it doesn't hold after operand substitution.

This patch introduces a new helper
`isSafeToSpeculativelyExecuteWithVariableReplaced`. It ignores operand
info in these instructions since their operands will be replaced later.

Fixes #99436.

---------

Co-authored-by: Nikita Popov <github@npopov.com>
---
 llvm/include/llvm/Analysis/ValueTracking.h    | 26 +++++++++++++------
 llvm/lib/Analysis/ValueTracking.cpp           | 12 ++++++---
 .../InstCombine/InstCombineSelect.cpp         |  7 +++--
 .../InstCombine/select-binop-cmp.ll           | 13 ++++++++++
 llvm/test/Transforms/InstCombine/select.ll    | 16 ++++++++++++
 5 files changed, 60 insertions(+), 14 deletions(-)

diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 2c2f965a3cd6f..44e27234aa90d 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -822,15 +822,25 @@ bool isSafeToSpeculativelyExecute(const Instruction *I,
                                   const Instruction *CtxI = nullptr,
                                   AssumptionCache *AC = nullptr,
                                   const DominatorTree *DT = nullptr,
-                                  const TargetLibraryInfo *TLI = nullptr);
+                                  const TargetLibraryInfo *TLI = nullptr,
+                                  bool UseVariableInfo = true);
+
+inline bool isSafeToSpeculativelyExecute(const Instruction *I,
+                                         BasicBlock::iterator CtxI,
+                                         AssumptionCache *AC = nullptr,
+                                         const DominatorTree *DT = nullptr,
+                                         const TargetLibraryInfo *TLI = nullptr,
+                                         bool UseVariableInfo = true) {
+  // Take an iterator, and unwrap it into an Instruction *.
+  return isSafeToSpeculativelyExecute(I, &*CtxI, AC, DT, TLI, UseVariableInfo);
+}
 
+/// Don't use information from its non-constant operands. This helper is used
+/// when its operands are going to be replaced.
 inline bool
-isSafeToSpeculativelyExecute(const Instruction *I, BasicBlock::iterator CtxI,
-                             AssumptionCache *AC = nullptr,
-                             const DominatorTree *DT = nullptr,
-                             const TargetLibraryInfo *TLI = nullptr) {
-  // Take an iterator, and unwrap it into an Instruction *.
-  return isSafeToSpeculativelyExecute(I, &*CtxI, AC, DT, TLI);
+isSafeToSpeculativelyExecuteWithVariableReplaced(const Instruction *I) {
+  return isSafeToSpeculativelyExecute(I, nullptr, nullptr, nullptr, nullptr,
+                                      /*UseVariableInfo=*/false);
 }
 
 /// This returns the same result as isSafeToSpeculativelyExecute if Opcode is
@@ -853,7 +863,7 @@ isSafeToSpeculativelyExecute(const Instruction *I, BasicBlock::iterator CtxI,
 bool isSafeToSpeculativelyExecuteWithOpcode(
     unsigned Opcode, const Instruction *Inst, const Instruction *CtxI = nullptr,
     AssumptionCache *AC = nullptr, const DominatorTree *DT = nullptr,
-    const TargetLibraryInfo *TLI = nullptr);
+    const TargetLibraryInfo *TLI = nullptr, bool UseVariableInfo = true);
 
 /// Returns true if the result or effects of the given instructions \p I
 /// depend values not reachable through the def use graph.
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 03eb6ef42b0ff..5c56654246ead 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -6771,15 +6771,16 @@ bool llvm::isSafeToSpeculativelyExecute(const Instruction *Inst,
                                         const Instruction *CtxI,
                                         AssumptionCache *AC,
                                         const DominatorTree *DT,
-                                        const TargetLibraryInfo *TLI) {
+                                        const TargetLibraryInfo *TLI,
+                                        bool UseVariableInfo) {
   return isSafeToSpeculativelyExecuteWithOpcode(Inst->getOpcode(), Inst, CtxI,
-                                                AC, DT, TLI);
+                                                AC, DT, TLI, UseVariableInfo);
 }
 
 bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
     unsigned Opcode, const Instruction *Inst, const Instruction *CtxI,
-    AssumptionCache *AC, const DominatorTree *DT,
-    const TargetLibraryInfo *TLI) {
+    AssumptionCache *AC, const DominatorTree *DT, const TargetLibraryInfo *TLI,
+    bool UseVariableInfo) {
 #ifndef NDEBUG
   if (Inst->getOpcode() != Opcode) {
     // Check that the operands are actually compatible with the Opcode override.
@@ -6831,6 +6832,9 @@ bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
     return false;
   }
   case Instruction::Load: {
+    if (!UseVariableInfo)
+      return false;
+
     const LoadInst *LI = dyn_cast<LoadInst>(Inst);
     if (!LI)
       return false;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index e387034110df9..aaf4ece3249a2 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1247,8 +1247,11 @@ bool InstCombinerImpl::replaceInInstruction(Value *V, Value *Old, Value *New,
   if (Depth == 2)
     return false;
 
+  assert(!isa<Constant>(Old) && "Only replace non-constant values");
+
   auto *I = dyn_cast<Instruction>(V);
-  if (!I || !I->hasOneUse() || !isSafeToSpeculativelyExecute(I))
+  if (!I || !I->hasOneUse() ||
+      !isSafeToSpeculativelyExecuteWithVariableReplaced(I))
     return false;
 
   bool Changed = false;
@@ -1331,7 +1334,7 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
     // profitability is not clear for other cases.
     // FIXME: Support vectors.
     if (OldOp == CmpLHS && match(NewOp, m_ImmConstant()) &&
-        !match(OldOp, m_ImmConstant()) && !Cmp.getType()->isVectorTy() &&
+        !match(OldOp, m_Constant()) && !Cmp.getType()->isVectorTy() &&
         isGuaranteedNotToBeUndef(NewOp, SQ.AC, &Sel, &DT))
       if (replaceInInstruction(TrueVal, OldOp, NewOp))
         return &Sel;
diff --git a/llvm/test/Transforms/InstCombine/select-binop-cmp.ll b/llvm/test/Transforms/InstCombine/select-binop-cmp.ll
index 1fa0c09a9e987..fb56764598e2d 100644
--- a/llvm/test/Transforms/InstCombine/select-binop-cmp.ll
+++ b/llvm/test/Transforms/InstCombine/select-binop-cmp.ll
@@ -1315,6 +1315,19 @@ define i32 @select_replace_call_speculatable(i32 %x, i32 %y) {
   ret i32 %s
 }
 
+define i32 @select_replace_call_speculatable_intrinsic(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_replace_call_speculatable_intrinsic(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @llvm.smax.i32(i32 [[Y:%.*]], i32 0)
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i32 [[CALL]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %c = icmp eq i32 %x, 0
+  %call = call i32 @llvm.smax.i32(i32 %x, i32 %y)
+  %s = select i1 %c, i32 %call, i32 %y
+  ret i32 %s
+}
+
 ; We can't replace the call arguments, as the call is not speculatable. We
 ; may end up changing side-effects or causing undefined behavior.
 define i32 @select_replace_call_non_speculatable(i32 %x, i32 %y) {
diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll
index d66ffb9a63ac1..1369be305ec13 100644
--- a/llvm/test/Transforms/InstCombine/select.ll
+++ b/llvm/test/Transforms/InstCombine/select.ll
@@ -4713,3 +4713,19 @@ define i8 @select_knownbits_simplify_missing_noundef(i8 %x)  {
   %res = select i1 %cmp, i8 %and, i8 0
   ret i8 %res
 }
+
+@g_ext = external global i8
+
+; Make sure we don't replace %ptr with @g_ext, which may cause the load to trigger UB.
+define i32 @pr99436(ptr align 4 dereferenceable(4) %ptr) {
+; CHECK-LABEL: @pr99436(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[PTR:%.*]], @g_ext
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[PTR]], align 4
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 [[VAL]], i32 0
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %cmp = icmp eq ptr %ptr, @g_ext
+  %val = load i32, ptr %ptr, align 4
+  %ret = select i1 %cmp, i32 %val, i32 0
+  ret i32 %ret
+}

From 57ccd42393b2e9e341d730ec8a6f39a5c89d126d Mon Sep 17 00:00:00 2001
From: Chen Zheng <czhengsz@cn.ibm.com>
Date: Mon, 22 Jul 2024 00:07:47 -0400
Subject: [PATCH 367/486] [PowerPC]fix XRAY failures in
 https://lab.llvm.org/buildbot/#/builders/145/builds/726

Regression caused by 43213002b99e32d618f2afbbaaeb2ff8dfc84e33.
---
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 753f0d586a0d9..0e2811f87c817 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -1854,7 +1854,7 @@ void PPCLinuxAsmPrinter::emitInstruction(const MachineInstr *MI) {
       IsConditional = false;
     } else {
       EmitToStreamer(*OutStreamer, RetInst);
-      break;
+      return;
     }
 
     MCSymbol *FallthroughLabel;
@@ -1919,7 +1919,7 @@ void PPCLinuxAsmPrinter::emitInstruction(const MachineInstr *MI) {
     if (IsConditional)
       OutStreamer->emitLabel(FallthroughLabel);
     recordSled(BeginOfSled, *MI, SledKind::FUNCTION_EXIT, 2);
-    break;
+    return;
   }
   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
     llvm_unreachable("PATCHABLE_FUNCTION_EXIT should never be emitted");

From 7b28cc0c59d2f8a4427e5f0677ff69c7b319c9bb Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Mon, 22 Jul 2024 08:55:39 +0400
Subject: [PATCH 368/486] AMDGPU: Query MachineModuleInfo from PM instead of
 MachineFunction (#99679)

---
 llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 452dac4b00993..bd4203ccd6fe4 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -217,7 +217,7 @@ class SIMemOpInfo final {
 
 class SIMemOpAccess final {
 private:
-  AMDGPUMachineModuleInfo *MMI = nullptr;
+  const AMDGPUMachineModuleInfo *MMI = nullptr;
 
   /// Reports unsupported message \p Msg for \p MI to LLVM context.
   void reportUnsupported(const MachineBasicBlock::iterator &MI,
@@ -241,7 +241,7 @@ class SIMemOpAccess final {
 public:
   /// Construct class to support accessing the machine memory operands
   /// of instructions in the machine function \p MF.
-  SIMemOpAccess(MachineFunction &MF);
+  SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI);
 
   /// \returns Load info if \p MI is a load operation, "std::nullopt" otherwise.
   std::optional<SIMemOpInfo>
@@ -806,9 +806,8 @@ SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(unsigned AS) const {
   return SIAtomicAddrSpace::OTHER;
 }
 
-SIMemOpAccess::SIMemOpAccess(MachineFunction &MF) {
-  MMI = &MF.getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>();
-}
+SIMemOpAccess::SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI_)
+    : MMI(&MMI_) {}
 
 std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
     const MachineBasicBlock::iterator &MI) const {
@@ -2802,7 +2801,10 @@ bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI,
 bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) {
   bool Changed = false;
 
-  SIMemOpAccess MOA(MF);
+  const MachineModuleInfo &MMI =
+      getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
+
+  SIMemOpAccess MOA(MMI.getObjFileInfo<AMDGPUMachineModuleInfo>());
   CC = SICacheControl::create(MF.getSubtarget<GCNSubtarget>());
 
   for (auto &MBB : MF) {

From c91e85278cb87a35924d86463d3c0a379c64e9d1 Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777@gmail.com>
Date: Mon, 22 Jul 2024 13:16:51 +0800
Subject: [PATCH 369/486] =?UTF-8?q?Revert=20"[Clang]=20Implement=20P3034R1?=
 =?UTF-8?q?=20Module=20Declarations=20Shouldn=E2=80=99t=20be=20Macros"=20(?=
 =?UTF-8?q?#99838)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Reverts llvm/llvm-project#90574
---
 clang/docs/ReleaseNotes.rst                   |   2 -
 .../include/clang/Basic/DiagnosticLexKinds.td |   5 -
 clang/include/clang/Basic/IdentifierTable.h   |  24 +-
 clang/include/clang/Basic/TokenKinds.def      |   3 -
 clang/include/clang/Lex/Preprocessor.h        |  83 +---
 clang/include/clang/Lex/Token.h               |   3 -
 clang/include/clang/Parse/Parser.h            |   2 +-
 clang/lib/Basic/IdentifierTable.cpp           |   3 +-
 .../lib/Frontend/PrintPreprocessedOutput.cpp  |  12 +-
 clang/lib/Lex/PPLexerChange.cpp               |   9 +-
 clang/lib/Lex/Preprocessor.cpp                | 444 ++++++------------
 clang/lib/Lex/TokenConcatenation.cpp          |  10 -
 clang/lib/Parse/ParseDecl.cpp                 |   8 +-
 clang/lib/Parse/Parser.cpp                    |  93 ++--
 clang/test/CXX/cpp/cpp.module/p2.cppm         |  88 ----
 .../basic/basic.link/module-declaration.cpp   |  61 ++-
 .../dcl.module/dcl.module.import/p1.cppm      |  39 +-
 clang/test/SemaCXX/modules.cppm               |  89 ++--
 clang/www/cxx_status.html                     |   2 +-
 19 files changed, 263 insertions(+), 717 deletions(-)
 delete mode 100644 clang/test/CXX/cpp/cpp.module/p2.cppm

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 4ab6bd9de8ea9..7ac6ed934290d 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -285,8 +285,6 @@ C++2c Feature Support
 
 - Implemented `P2963R3 Ordering of constraints involving fold expressions <https://wg21.link/P2963R3>`_.
 
-- Implemented `P3034R1 Module Declarations Shouldn’t be Macros <https://wg21.link/P3034R1>`_.
-
 
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td
index 08ece01009387..12d7b8c0205ee 100644
--- a/clang/include/clang/Basic/DiagnosticLexKinds.td
+++ b/clang/include/clang/Basic/DiagnosticLexKinds.td
@@ -952,11 +952,6 @@ def warn_module_conflict : Warning<
   InGroup<ModuleConflict>;
 
 // C++20 modules
-def err_module_decl_cannot_be_macros : Error<
-  "the module name in a module%select{| partition}0 declaration cannot contain "
-  "an object-like macro %1">;
-def err_unxepected_paren_in_module_decl : Error<
-  "unexpected '(' after the module name in a module%select{| partition}0 declaration">;
 def err_header_import_semi_in_macro : Error<
   "semicolon terminating header import declaration cannot be produced "
   "by a macro">;
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index f40f74d0355ad..ae9ebd9f59154 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -180,10 +180,6 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsModulesImport : 1;
 
-  // True if this is the 'module' contextual keyword.
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned IsModulesDecl : 1;
-
   // True if this is a mangled OpenMP variant name.
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsMangledOpenMPVariantName : 1;
@@ -200,7 +196,7 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsFinal : 1;
 
-  // 21 bits left in a 64-bit word.
+  // 22 bits left in a 64-bit word.
 
   // Managed by the language front-end.
   void *FETokenInfo = nullptr;
@@ -216,8 +212,8 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
         IsCPPOperatorKeyword(false), NeedsHandleIdentifier(false),
         IsFromAST(false), ChangedAfterLoad(false), FEChangedAfterLoad(false),
         RevertedTokenID(false), OutOfDate(false), IsModulesImport(false),
-        IsModulesDecl(false), IsMangledOpenMPVariantName(false),
-        IsDeprecatedMacro(false), IsRestrictExpansion(false), IsFinal(false) {}
+        IsMangledOpenMPVariantName(false), IsDeprecatedMacro(false),
+        IsRestrictExpansion(false), IsFinal(false) {}
 
 public:
   IdentifierInfo(const IdentifierInfo &) = delete;
@@ -524,18 +520,6 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
       RecomputeNeedsHandleIdentifier();
   }
 
-  /// Determine whether this is the contextual keyword \c module.
-  bool isModulesDeclaration() const { return IsModulesDecl; }
-
-  /// Set whether this identifier is the contextual keyword \c module.
-  void setModulesDeclaration(bool I) {
-    IsModulesDecl = I;
-    if (I)
-      NeedsHandleIdentifier = true;
-    else
-      RecomputeNeedsHandleIdentifier();
-  }
-
   /// Determine whether this is the mangled name of an OpenMP variant.
   bool isMangledOpenMPVariantName() const { return IsMangledOpenMPVariantName; }
 
@@ -756,8 +740,6 @@ class IdentifierTable {
     // If this is the 'import' contextual keyword, mark it as such.
     if (Name == "import")
       II->setModulesImport(true);
-    else if (Name == "module")
-      II->setModulesDeclaration(true);
 
     return *II;
   }
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 8db18c049b6d0..7f4912b9bcd96 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -1003,9 +1003,6 @@ ANNOTATION(module_include)
 ANNOTATION(module_begin)
 ANNOTATION(module_end)
 
-// Annotations for C++, Clang and Objective-C named modules.
-ANNOTATION(module_name)
-
 // Annotation for a header_name token that has been looked up and transformed
 // into the name of a header unit.
 ANNOTATION(header_unit)
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 56aef99a3f38a..fc7d0053f2323 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -615,6 +615,10 @@ class Preprocessor {
 
   ModuleDeclSeq ModuleDeclState;
 
+  /// Whether the module import expects an identifier next. Otherwise,
+  /// it expects a '.' or ';'.
+  bool ModuleImportExpectsIdentifier = false;
+
   /// The identifier and source location of the currently-active
   /// \#pragma clang arc_cf_code_audited begin.
   std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
@@ -1740,14 +1744,11 @@ class Preprocessor {
   /// Lex a token, forming a header-name token if possible.
   bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
 
-  /// Lex a module name or a partition name.
-  bool LexModuleName(Token &Result, bool IsImport);
-
   /// Lex the parameters for an #embed directive, returns nullopt on error.
   std::optional<LexEmbedParametersResult> LexEmbedParameters(Token &Current,
                                                              bool ForHasEmbed);
+
   bool LexAfterModuleImport(Token &Result);
-  bool LexAfterModuleDecl(Token &Result);
   void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
 
   void makeModuleVisible(Module *M, SourceLocation Loc);
@@ -3038,9 +3039,6 @@ class Preprocessor {
   static bool CLK_LexAfterModuleImport(Preprocessor &P, Token &Result) {
     return P.LexAfterModuleImport(Result);
   }
-  static bool CLK_LexAfterModuleDecl(Preprocessor &P, Token &Result) {
-    return P.LexAfterModuleDecl(Result);
-  }
 };
 
 /// Abstract base class that describes a handler that will receive
@@ -3073,77 +3071,6 @@ struct EmbedAnnotationData {
 /// Registry of pragma handlers added by plugins
 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
 
-/// Represents module or partition name token sequance.
-///
-///     module-name:
-///           module-name-qualifier[opt] identifier
-///
-///     partition-name: [C++20]
-///           : module-name-qualifier[opt] identifier
-///
-///     module-name-qualifier
-///           module-name-qualifier[opt] identifier .
-///
-/// This class can only be created by the preprocessor and guarantees that the
-/// two source array being contiguous in memory and only contains 3 kind of
-/// tokens (identifier, '.' and ':'). And only available when the preprocessor
-/// returns annot_module_name token.
-///
-/// For exmaple:
-///
-/// export module m.n:c.d
-///
-/// The module name array has 3 tokens ['m', '.', 'n'].
-/// The partition name array has 4 tokens [':', 'c', '.', 'd'].
-///
-/// When import a partition in a named module fragment (Eg. import :part1;),
-/// the module name array will be empty, and the partition name array has 2
-/// tokens.
-///
-/// When we meet a private-module-fragment (Eg. module :private;), preprocessor
-/// will not return a annot_module_name token, but will return 2 separate tokens
-/// [':', 'kw_private'].
-
-class ModuleNameInfo {
-  friend class Preprocessor;
-  ArrayRef<Token> ModuleName;
-  ArrayRef<Token> PartitionName;
-
-  ModuleNameInfo(ArrayRef<Token> AnnotToks, std::optional<unsigned> ColonIndex);
-
-public:
-  /// Return the contiguous token array.
-  ArrayRef<Token> getTokens() const {
-    if (ModuleName.empty())
-      return PartitionName;
-    if (PartitionName.empty())
-      return ModuleName;
-    return ArrayRef(ModuleName.begin(), PartitionName.end());
-  }
-  bool hasModuleName() const { return !ModuleName.empty(); }
-  bool hasPartitionName() const { return !PartitionName.empty(); }
-  ArrayRef<Token> getModuleName() const { return ModuleName; }
-  ArrayRef<Token> getPartitionName() const { return PartitionName; }
-  Token getColonToken() const {
-    assert(hasPartitionName() && "Do not have a partition name");
-    return getPartitionName().front();
-  }
-
-  /// Under the standard C++ Modules, the dot is just part of the module name,
-  /// and not a real hierarchy separator. Flatten such module names now.
-  std::string getFlatName() const;
-
-  /// Build a module id path from the contiguous token array, both include
-  /// module name and partition name.
-  void getModuleIdPath(
-      SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path) const;
-
-  /// Build a module id path from \param ModuleName.
-  static void getModuleIdPath(
-      ArrayRef<Token> ModuleName,
-      SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path);
-};
-
 } // namespace clang
 
 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index 2be3ad39529f0..4f29fb7d11415 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -235,9 +235,6 @@ class Token {
     assert(isAnnotation() && "Used AnnotVal on non-annotation token");
     return PtrData;
   }
-  template <class T> T getAnnotationValueAs() const {
-    return static_cast<T>(getAnnotationValue());
-  }
   void setAnnotationValue(void *val) {
     assert(isAnnotation() && "Used AnnotVal on non-annotation token");
     PtrData = val;
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index afcdacf02583a..93e60be512aae 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3876,7 +3876,7 @@ class Parser : public CodeCompletionHandler {
   }
 
   bool ParseModuleName(
-      SourceLocation UseLoc, ArrayRef<Token> ModuleName,
+      SourceLocation UseLoc,
       SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path,
       bool IsImport);
 
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index 97d830214f890..4f7ccaf4021d6 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -322,9 +322,8 @@ void IdentifierTable::AddKeywords(const LangOptions &LangOpts) {
   if (LangOpts.IEEE128)
     AddKeyword("__ieee128", tok::kw___float128, KEYALL, LangOpts, *this);
 
-  // Add the 'import' and 'module' contextual keyword.
+  // Add the 'import' contextual keyword.
   get("import").setModulesImport(true);
-  get("module").setModulesDeclaration(true);
 }
 
 /// Checks if the specified token kind represents a keyword in the
diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 1fff88ccf0405..0592423c12eca 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -758,10 +758,9 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok,
   // These tokens are not expanded to anything and don't need whitespace before
   // them.
   if (Tok.is(tok::eof) ||
-      (Tok.isAnnotation() && Tok.isNot(tok::annot_header_unit) &&
-       Tok.isNot(tok::annot_module_begin) && Tok.isNot(tok::annot_module_end) &&
-       Tok.isNot(tok::annot_module_name) &&
-       Tok.isNot(tok::annot_repl_input_end) && Tok.isNot(tok::annot_embed)))
+      (Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) &&
+       !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) &&
+       !Tok.is(tok::annot_repl_input_end) && !Tok.is(tok::annot_embed)))
     return;
 
   // EmittedDirectiveOnThisLine takes priority over RequireSameLine.
@@ -952,11 +951,6 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
       PP.Lex(Tok);
       IsStartOfLine = true;
       continue;
-    } else if (Tok.is(tok::annot_module_name)) {
-      auto *Info = static_cast<ModuleNameInfo *>(Tok.getAnnotationValue());
-      *Callbacks->OS << Info->getFlatName();
-      PP.Lex(Tok);
-      continue;
     } else if (Tok.is(tok::annot_header_unit)) {
       // This is a header-name that has been (effectively) converted into a
       // module-name.
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index c3a903917e9ce..8221db46e06ac 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -122,8 +122,7 @@ void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
   CurPPLexer = TheLexer;
   CurDirLookup = CurDir;
   CurLexerSubmodule = nullptr;
-  if (CurLexerCallback != CLK_LexAfterModuleImport &&
-      CurLexerCallback != CLK_LexAfterModuleDecl)
+  if (CurLexerCallback != CLK_LexAfterModuleImport)
     CurLexerCallback = TheLexer->isDependencyDirectivesLexer()
                            ? CLK_DependencyDirectivesLexer
                            : CLK_Lexer;
@@ -162,7 +161,8 @@ void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
   PushIncludeMacroStack();
   CurDirLookup = nullptr;
   CurTokenLexer = std::move(TokLexer);
-  CurLexerCallback = CLK_TokenLexer;
+  if (CurLexerCallback != CLK_LexAfterModuleImport)
+    CurLexerCallback = CLK_TokenLexer;
 }
 
 /// EnterTokenStream - Add a "macro" context to the top of the include stack,
@@ -216,8 +216,7 @@ void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
   PushIncludeMacroStack();
   CurDirLookup = nullptr;
   CurTokenLexer = std::move(TokLexer);
-  if (CurLexerCallback != CLK_LexAfterModuleImport &&
-      CurLexerCallback != CLK_LexAfterModuleDecl)
+  if (CurLexerCallback != CLK_LexAfterModuleImport)
     CurLexerCallback = CLK_TokenLexer;
 }
 
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 2726fae344337..63e27e62cffc8 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -860,15 +860,9 @@ bool Preprocessor::HandleIdentifier(Token &Identifier) {
     ModuleImportLoc = Identifier.getLocation();
     NamedModuleImportPath.clear();
     IsAtImport = true;
+    ModuleImportExpectsIdentifier = true;
     CurLexerCallback = CLK_LexAfterModuleImport;
   }
-
-  if ((II.isModulesDeclaration() || Identifier.is(tok::kw_module)) &&
-      !InMacroArgs && !DisableMacroExpansion &&
-      (getLangOpts().CPlusPlusModules || getLangOpts().DebuggerSupport) &&
-      CurLexerCallback != CLK_CachingLexer) {
-    CurLexerCallback = CLK_LexAfterModuleDecl;
-  }
   return true;
 }
 
@@ -911,7 +905,6 @@ void Preprocessor::Lex(Token &Result) {
     // This token is injected to represent the translation of '#include "a.h"'
     // into "import a.h;". Mimic the notional ';'.
     case tok::annot_module_include:
-    case tok::annot_repl_input_end:
     case tok::semi:
       TrackGMFState.handleSemi();
       StdCXXImportSeqState.handleSemi();
@@ -926,30 +919,12 @@ void Preprocessor::Lex(Token &Result) {
       StdCXXImportSeqState.handleExport();
       ModuleDeclState.handleExport();
       break;
-    case tok::annot_module_name: {
-      auto *Info = static_cast<ModuleNameInfo *>(Result.getAnnotationValue());
-      for (const auto &Tok : Info->getTokens()) {
-        switch (Tok.getKind()) {
-        case tok::identifier:
-          ModuleDeclState.handleIdentifier(Tok.getIdentifierInfo());
-          break;
-        case tok::period:
-          ModuleDeclState.handlePeriod();
-          break;
-        case tok::colon:
-          ModuleDeclState.handleColon();
-          break;
-        default:
-          llvm_unreachable("Unexpected token in module name");
-        }
-      }
-      if (ModuleDeclState.isModuleCandidate())
-        break;
-      TrackGMFState.handleMisc();
-      StdCXXImportSeqState.handleMisc();
-      ModuleDeclState.handleMisc();
+    case tok::colon:
+      ModuleDeclState.handleColon();
+      break;
+    case tok::period:
+      ModuleDeclState.handlePeriod();
       break;
-    }
     case tok::identifier:
       // Check "import" and "module" when there is no open bracket. The two
       // identifiers are not meaningful with open brackets.
@@ -961,17 +936,17 @@ void Preprocessor::Lex(Token &Result) {
             ModuleImportLoc = Result.getLocation();
             NamedModuleImportPath.clear();
             IsAtImport = false;
+            ModuleImportExpectsIdentifier = true;
             CurLexerCallback = CLK_LexAfterModuleImport;
           }
           break;
-        }
-        if (Result.getIdentifierInfo()->isModulesDeclaration()) {
+        } else if (Result.getIdentifierInfo() == getIdentifierInfo("module")) {
           TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq());
           ModuleDeclState.handleModule();
-          CurLexerCallback = CLK_LexAfterModuleDecl;
           break;
         }
       }
+      ModuleDeclState.handleIdentifier(Result.getIdentifierInfo());
       if (ModuleDeclState.isModuleCandidate())
         break;
       [[fallthrough]];
@@ -1146,151 +1121,6 @@ void Preprocessor::CollectPpImportSuffix(SmallVectorImpl<Token> &Toks) {
   }
 }
 
-ModuleNameInfo::ModuleNameInfo(ArrayRef<Token> AnnotToks,
-                               std::optional<unsigned> ColonIndex) {
-  assert(!AnnotToks.empty() && "Named module token cannot be empty.");
-  if (!ColonIndex.has_value())
-    ColonIndex = AnnotToks.size();
-  ModuleName = ArrayRef(AnnotToks.begin(), AnnotToks.begin() + *ColonIndex);
-  PartitionName = ArrayRef(AnnotToks.begin() + *ColonIndex, AnnotToks.end());
-  assert(ModuleName.end() == PartitionName.begin());
-}
-
-std::string ModuleNameInfo::getFlatName() const {
-  std::string FlatModuleName;
-  for (auto &Tok : getTokens()) {
-    switch (Tok.getKind()) {
-    case tok::identifier:
-      FlatModuleName += Tok.getIdentifierInfo()->getName();
-      break;
-    case tok::period:
-      FlatModuleName += '.';
-      break;
-    case tok::colon:
-      FlatModuleName += ':';
-      break;
-    default:
-      llvm_unreachable("Unexpected token in module name");
-    }
-  }
-  return FlatModuleName;
-}
-
-void ModuleNameInfo::getModuleIdPath(
-    SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path) const {
-  return getModuleIdPath(getTokens(), Path);
-}
-
-void ModuleNameInfo::getModuleIdPath(
-    ArrayRef<Token> ModuleName,
-    SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path) {
-  for (const auto &Tok : ModuleName) {
-    if (Tok.is(tok::identifier))
-      Path.push_back(
-          std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation()));
-  }
-}
-
-/// Lex a module name or a partition name.
-///
-///     module-name:
-///           module-name-qualifier[opt] identifier
-///
-///     partition-name: [C++20]
-///           : module-name-qualifier[opt] identifier
-///
-///     module-name-qualifier
-///           module-name-qualifier[opt] identifier .
-bool Preprocessor::LexModuleName(Token &Result, bool IsImport) {
-  bool ExpectsIdentifier = true, IsLexingPartition = false;
-  SmallVector<Token, 8> ModuleName;
-  std::optional<unsigned> ColonTokIndex;
-  auto LexNextToken = [&](Token &Tok) {
-    if (IsImport)
-      Lex(Tok);
-    else
-      LexUnexpandedToken(Tok);
-  };
-
-  while (true) {
-    LexNextToken(Result);
-    if (ExpectsIdentifier && Result.is(tok::identifier)) {
-      auto *MI = getMacroInfo(Result.getIdentifierInfo());
-      if (getLangOpts().CPlusPlusModules && !IsImport && MI &&
-          MI->isObjectLike()) {
-        Diag(Result, diag::err_module_decl_cannot_be_macros)
-            << Result.getLocation() << IsLexingPartition
-            << Result.getIdentifierInfo();
-      }
-      ModuleName.push_back(Result);
-      ExpectsIdentifier = false;
-      continue;
-    }
-
-    if (!ExpectsIdentifier && Result.is(tok::period)) {
-      ModuleName.push_back(Result);
-      ExpectsIdentifier = true;
-      continue;
-    }
-
-    // Module partition only allowed in C++20 Modules.
-    if (getLangOpts().CPlusPlusModules && Result.is(tok::colon)) {
-      // Handle the form like: import :P;
-      // If the token after ':' is not an identifier, this is a invalid module
-      // name.
-      if (ModuleName.empty()) {
-        Token Tmp;
-        LexNextToken(Tmp);
-        EnterToken(Tmp, /*IsReiject=*/false);
-        // A private-module-fragment:
-        // export module :private;
-        if (!IsImport && Tmp.is(tok::kw_private))
-          return true;
-        // import :N;
-        if (IsImport && Tmp.isNot(tok::identifier))
-          return false;
-      } else if (!ExpectsIdentifier) {
-        ExpectsIdentifier = true;
-      }
-      IsLexingPartition = true;
-      ColonTokIndex = ModuleName.size();
-      ModuleName.push_back(Result);
-      continue;
-    }
-
-    // [cpp.module]/p2: where the pp-tokens (if any) shall not begin with a (
-    // preprocessing token [...]
-    //
-    // We only emit diagnostic in the preprocessor, and in the parser we skip
-    // invalid tokens and recover from errors.
-    if (getLangOpts().CPlusPlusModules && !ExpectsIdentifier &&
-        Result.is(tok::l_paren))
-      Diag(Result, diag::err_unxepected_paren_in_module_decl)
-          << IsLexingPartition;
-    break;
-  }
-
-  // Put the last token back to stream, it's not a valid part of module name.
-  // We lexed it unexpanded but it might be a valid macro expansion
-  Result.clearFlag(Token::DisableExpand);
-  auto ToksCopy = std::make_unique<Token[]>(1);
-  *ToksCopy.get() = Result;
-  EnterTokenStream(std::move(ToksCopy), 1,
-                   /*DisableMacroExpansion=*/false,
-                   /*IsReinject=*/false);
-
-  if (ModuleName.empty())
-    return false;
-  Result.startToken();
-  Result.setKind(tok::annot_module_name);
-  Result.setLocation(ModuleName.front().getLocation());
-  Result.setAnnotationEndLoc(ModuleName.back().getLocation());
-  auto AnnotToks = ArrayRef(ModuleName).copy(getPreprocessorAllocator());
-  ModuleNameInfo *Info =
-      new (getPreprocessorAllocator()) ModuleNameInfo(AnnotToks, ColonTokIndex);
-  Result.setAnnotationValue(static_cast<void *>(Info));
-  return true;
-}
 
 /// Lex a token following the 'import' contextual keyword.
 ///
@@ -1315,17 +1145,6 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
   // Figure out what kind of lexer we actually have.
   recomputeCurLexerKind();
 
-  // Allocate a holding buffer for a sequence of tokens and introduce it into
-  // the token stream.
-  auto EnterTokens = [this](ArrayRef<Token> Toks) {
-    auto ToksCopy = std::make_unique<Token[]>(Toks.size());
-    std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
-    EnterTokenStream(std::move(ToksCopy), Toks.size(),
-                     /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
-  };
-
-  SmallVector<Token, 32> Suffix;
-
   // Lex the next token. The header-name lexing rules are used at the start of
   // a pp-import.
   //
@@ -1336,108 +1155,122 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
     if (LexHeaderName(Result))
       return true;
 
-    // Check for a header-name.
-    if (Result.is(tok::header_name)) {
-      // Enter the header-name token into the token stream; a Lex action cannot
-      // both return a token and cache tokens (doing so would corrupt the token
-      // cache if the call to Lex comes from CachingLex / PeekAhead).
-      Suffix.push_back(Result);
-
-      // Consume the pp-import-suffix and expand any macros in it now. We'll add
-      // it back into the token stream later.
-      CollectPpImportSuffix(Suffix);
-      if (Suffix.back().isNot(tok::semi)) {
-        // This is not a pp-import after all.
-        EnterTokens(Suffix);
-        return false;
-      }
+    if (Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
+      std::string Name = ModuleDeclState.getPrimaryName().str();
+      Name += ":";
+      NamedModuleImportPath.push_back(
+          {getIdentifierInfo(Name), Result.getLocation()});
+      CurLexerCallback = CLK_LexAfterModuleImport;
+      return true;
+    }
+  } else {
+    Lex(Result);
+  }
 
-      // C++2a [cpp.module]p1:
-      //   The ';' preprocessing-token terminating a pp-import shall not have
-      //   been produced by macro replacement.
-      SourceLocation SemiLoc = Suffix.back().getLocation();
-      if (SemiLoc.isMacroID())
-        Diag(SemiLoc, diag::err_header_import_semi_in_macro);
-
-      // Reconstitute the import token.
-      Token ImportTok;
-      ImportTok.startToken();
-      ImportTok.setKind(tok::kw_import);
-      ImportTok.setLocation(ModuleImportLoc);
-      ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
-      ImportTok.setLength(6);
-
-      auto Action = HandleHeaderIncludeOrImport(
-          /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
-      switch (Action.Kind) {
-      case ImportAction::None:
-        break;
+  // Allocate a holding buffer for a sequence of tokens and introduce it into
+  // the token stream.
+  auto EnterTokens = [this](ArrayRef<Token> Toks) {
+    auto ToksCopy = std::make_unique<Token[]>(Toks.size());
+    std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
+    EnterTokenStream(std::move(ToksCopy), Toks.size(),
+                     /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
+  };
 
-      case ImportAction::ModuleBegin:
-        // Let the parser know we're textually entering the module.
-        Suffix.emplace_back();
-        Suffix.back().startToken();
-        Suffix.back().setKind(tok::annot_module_begin);
-        Suffix.back().setLocation(SemiLoc);
-        Suffix.back().setAnnotationEndLoc(SemiLoc);
-        Suffix.back().setAnnotationValue(Action.ModuleForHeader);
-        [[fallthrough]];
-
-      case ImportAction::ModuleImport:
-      case ImportAction::HeaderUnitImport:
-      case ImportAction::SkippedModuleImport:
-        // We chose to import (or textually enter) the file. Convert the
-        // header-name token into a header unit annotation token.
-        Suffix[0].setKind(tok::annot_header_unit);
-        Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
-        Suffix[0].setAnnotationValue(Action.ModuleForHeader);
-        // FIXME: Call the moduleImport callback?
-        break;
-      case ImportAction::Failure:
-        assert(TheModuleLoader.HadFatalFailure &&
-               "This should be an early exit only to a fatal error");
-        Result.setKind(tok::eof);
-        CurLexer->cutOffLexing();
-        EnterTokens(Suffix);
-        return true;
-      }
+  bool ImportingHeader = Result.is(tok::header_name);
+  // Check for a header-name.
+  SmallVector<Token, 32> Suffix;
+  if (ImportingHeader) {
+    // Enter the header-name token into the token stream; a Lex action cannot
+    // both return a token and cache tokens (doing so would corrupt the token
+    // cache if the call to Lex comes from CachingLex / PeekAhead).
+    Suffix.push_back(Result);
 
+    // Consume the pp-import-suffix and expand any macros in it now. We'll add
+    // it back into the token stream later.
+    CollectPpImportSuffix(Suffix);
+    if (Suffix.back().isNot(tok::semi)) {
+      // This is not a pp-import after all.
       EnterTokens(Suffix);
       return false;
     }
-  } else {
-    Lex(Result);
-  }
 
-  if (Result.isOneOf(tok::identifier, tok::colon)) {
-    EnterToken(Result, /*IsReinject=*/false);
-    if (!LexModuleName(Result, /*IsImport=*/true))
+    // C++2a [cpp.module]p1:
+    //   The ';' preprocessing-token terminating a pp-import shall not have
+    //   been produced by macro replacement.
+    SourceLocation SemiLoc = Suffix.back().getLocation();
+    if (SemiLoc.isMacroID())
+      Diag(SemiLoc, diag::err_header_import_semi_in_macro);
+
+    // Reconstitute the import token.
+    Token ImportTok;
+    ImportTok.startToken();
+    ImportTok.setKind(tok::kw_import);
+    ImportTok.setLocation(ModuleImportLoc);
+    ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
+    ImportTok.setLength(6);
+
+    auto Action = HandleHeaderIncludeOrImport(
+        /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
+    switch (Action.Kind) {
+    case ImportAction::None:
+      break;
+
+    case ImportAction::ModuleBegin:
+      // Let the parser know we're textually entering the module.
+      Suffix.emplace_back();
+      Suffix.back().startToken();
+      Suffix.back().setKind(tok::annot_module_begin);
+      Suffix.back().setLocation(SemiLoc);
+      Suffix.back().setAnnotationEndLoc(SemiLoc);
+      Suffix.back().setAnnotationValue(Action.ModuleForHeader);
+      [[fallthrough]];
+
+    case ImportAction::ModuleImport:
+    case ImportAction::HeaderUnitImport:
+    case ImportAction::SkippedModuleImport:
+      // We chose to import (or textually enter) the file. Convert the
+      // header-name token into a header unit annotation token.
+      Suffix[0].setKind(tok::annot_header_unit);
+      Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
+      Suffix[0].setAnnotationValue(Action.ModuleForHeader);
+      // FIXME: Call the moduleImport callback?
+      break;
+    case ImportAction::Failure:
+      assert(TheModuleLoader.HadFatalFailure &&
+             "This should be an early exit only to a fatal error");
+      Result.setKind(tok::eof);
+      CurLexer->cutOffLexing();
+      EnterTokens(Suffix);
       return true;
-    auto *Info = Result.getAnnotationValueAs<ModuleNameInfo *>();
-    if (getLangOpts().CPlusPlusModules) {
-      // Under the standard C++ Modules, the dot is just part of the module
-      // name, and not a real hierarchy separator. Flatten such module names
-      // now.
-      //
-      // FIXME: Is this the right level to be performing this transformation?
-      std::string FlatModuleName;
-      if (Info->getTokens().front().is(tok::colon)) {
-        // Import a module partition allowed in C++20 Modules.
-        // We can import a partition in named module TU.
-        if (NamedModuleImportPath.empty() && ModuleDeclState.isNamedModule())
-          FlatModuleName = llvm::Twine(ModuleDeclState.getPrimaryName())
-                               .concat(Info->getFlatName())
-                               .str();
-        else
-          return true;
-      } else {
-        FlatModuleName = Info->getFlatName();
-      }
-      NamedModuleImportPath.emplace_back(getIdentifierInfo(FlatModuleName),
-                                         Result.getLocation());
-    } else {
-      Info->getModuleIdPath(NamedModuleImportPath);
     }
+
+    EnterTokens(Suffix);
+    return false;
+  }
+
+  // The token sequence
+  //
+  //   import identifier (. identifier)*
+  //
+  // indicates a module import directive. We already saw the 'import'
+  // contextual keyword, so now we're looking for the identifiers.
+  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
+    // We expected to see an identifier here, and we did; continue handling
+    // identifiers.
+    NamedModuleImportPath.push_back(
+        std::make_pair(Result.getIdentifierInfo(), Result.getLocation()));
+    ModuleImportExpectsIdentifier = false;
+    CurLexerCallback = CLK_LexAfterModuleImport;
+    return true;
+  }
+
+  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
+  // see the next identifier. (We can also see a '[[' that begins an
+  // attribute-specifier-seq here under the Standard C++ Modules.)
+  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
+    ModuleImportExpectsIdentifier = true;
+    CurLexerCallback = CLK_LexAfterModuleImport;
+    return true;
   }
 
   // If we didn't recognize a module name at all, this is not a (valid) import.
@@ -1458,6 +1291,24 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
     SemiLoc = Suffix.back().getLocation();
   }
 
+  // Under the standard C++ Modules, the dot is just part of the module name,
+  // and not a real hierarchy separator. Flatten such module names now.
+  //
+  // FIXME: Is this the right level to be performing this transformation?
+  std::string FlatModuleName;
+  if (getLangOpts().CPlusPlusModules) {
+    for (auto &Piece : NamedModuleImportPath) {
+      // If the FlatModuleName ends with colon, it implies it is a partition.
+      if (!FlatModuleName.empty() && FlatModuleName.back() != ':')
+        FlatModuleName += ".";
+      FlatModuleName += Piece.first->getName();
+    }
+    SourceLocation FirstPathLoc = NamedModuleImportPath[0].second;
+    NamedModuleImportPath.clear();
+    NamedModuleImportPath.push_back(
+        std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
+  }
+
   Module *Imported = nullptr;
   // We don't/shouldn't load the standard c++20 modules when preprocessing.
   if (getLangOpts().Modules && !isInImportingCXXNamedModules()) {
@@ -1479,33 +1330,6 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
   return true;
 }
 
-/// Lex a token following the 'module' contextual keyword.
-///
-/// [cpp.module]/p2:
-/// The pp-tokens, if any, of a pp-module shall be of the form:
-///       pp-module-name pp-module-partition[opt] pp-tokens[opt]
-///
-/// where the pp-tokens (if any) shall not begin with a ( preprocessing token
-/// and the grammar non-terminals are defined as:
-///       pp-module-name:
-///             pp-module-name-qualifierp[opt] identifier
-///       pp-module-partition:
-///             : pp-module-name-qualifier[opt] identifier
-///       pp-module-name-qualifier:
-///             identifier .
-///             pp-module-name-qualifier identifier .
-/// No identifier in the pp-module-name or pp-module-partition shall currently
-/// be defined as an object-like macro.
-///
-/// [cpp.module]/p3:
-/// Any preprocessing tokens after the module preprocessing token in the module
-/// directive are processed just as in normal text.
-bool Preprocessor::LexAfterModuleDecl(Token &Result) {
-  // Figure out what kind of lexer we actually have.
-  recomputeCurLexerKind();
-  return LexModuleName(Result, /*IsImport=*/false);
-}
-
 void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc) {
   CurSubmoduleState->VisibleModules.setVisible(
       M, Loc, [](Module *) {},
diff --git a/clang/lib/Lex/TokenConcatenation.cpp b/clang/lib/Lex/TokenConcatenation.cpp
index cdb636923b9e9..865879d180533 100644
--- a/clang/lib/Lex/TokenConcatenation.cpp
+++ b/clang/lib/Lex/TokenConcatenation.cpp
@@ -160,13 +160,6 @@ static char GetFirstChar(const Preprocessor &PP, const Token &Tok) {
 bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
                                      const Token &PrevTok,
                                      const Token &Tok) const {
-  // If previous token is a module name, we need avoid concat it with current
-  // token, otherwise, there will has an extra space between 'M' and ';' for the
-  // following code:
-  //
-  // import M;
-  if (PrevTok.is(tok::annot_module_name))
-    return false;
   // Conservatively assume that every annotation token that has a printable
   // form requires whitespace.
   if (PrevTok.isAnnotation())
@@ -197,9 +190,6 @@ bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
       return true;
     ConcatInfo &= ~aci_avoid_equal;
   }
-
-  if (Tok.is(tok::annot_module_name))
-    return true;
   if (Tok.isAnnotation()) {
     // Modules annotation can show up when generated automatically for includes.
     assert(Tok.isOneOf(tok::annot_module_include, tok::annot_module_begin,
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 577527d0318f2..7ce9a9cea1c7a 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -3958,13 +3958,7 @@ void Parser::ParseDeclarationSpecifiers(
 
       // We're done with the declaration-specifiers.
       goto DoneWithDeclSpec;
-    case tok::annot_module_name: {
-      PP.EnterTokenStream(
-          Tok.getAnnotationValueAs<ModuleNameInfo *>()->getTokens(),
-          /*DisableMacroExpansion=*/true, /*IsReinject=*/false);
-      ConsumeAnyToken();
-      [[fallthrough]];
-    }
+
       // typedef-name
     case tok::kw___super:
     case tok::kw_decltype:
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index afb2e1e416168..5ebe71e496a2e 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -2511,28 +2511,18 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) {
   }
 
   SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
-  if (Tok.isNot(tok::annot_module_name)) {
-    Diag(Tok, diag::err_module_expected_ident) << /*IsImport=*/false;
-    SkipUntil(tok::semi, StopBeforeMatch);
-    return nullptr;
-  }
-
-  auto *Info = Tok.getAnnotationValueAs<ModuleNameInfo *>();
-  ConsumeAnnotationToken();
-  if (ParseModuleName(ModuleLoc, Info->getModuleName(), Path,
-                      /*IsImport=*/false))
+  if (ParseModuleName(ModuleLoc, Path, /*IsImport*/ false))
     return nullptr;
 
   // Parse the optional module-partition.
   SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Partition;
-  if (Info->hasPartitionName()) {
-    SourceLocation ColonLoc = Info->getColonToken().getLocation();
+  if (Tok.is(tok::colon)) {
+    SourceLocation ColonLoc = ConsumeToken();
     if (!getLangOpts().CPlusPlusModules)
       Diag(ColonLoc, diag::err_unsupported_module_partition)
           << SourceRange(ColonLoc, Partition.back().second);
     // Recover by ignoring the partition name.
-    else if (ParseModuleName(ModuleLoc, Info->getPartitionName(), Partition,
-                             /*IsImport=*/false))
+    else if (ParseModuleName(ModuleLoc, Partition, /*IsImport*/ false))
       return nullptr;
   }
 
@@ -2591,32 +2581,18 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
     // This is a header import that the preprocessor mapped to a module import.
     HeaderUnit = reinterpret_cast<Module *>(Tok.getAnnotationValue());
     ConsumeAnnotationToken();
-  } else {
-    if (Tok.isNot(tok::annot_module_name)) {
-      if (Tok.is(tok::code_completion)) {
-        cutOffParsing();
-        Actions.CodeCompletion().CodeCompleteModuleImport(ImportLoc, Path);
-        return nullptr;
-      }
-      Diag(Tok, diag::err_module_expected_ident) << /*IsImport=*/true;
-      SkipUntil(tok::semi, StopBeforeMatch);
+  } else if (Tok.is(tok::colon)) {
+    SourceLocation ColonLoc = ConsumeToken();
+    if (!getLangOpts().CPlusPlusModules)
+      Diag(ColonLoc, diag::err_unsupported_module_partition)
+          << SourceRange(ColonLoc, Path.back().second);
+    // Recover by leaving partition empty.
+    else if (ParseModuleName(ColonLoc, Path, /*IsImport*/ true))
       return nullptr;
-    }
-    auto *Info = Tok.getAnnotationValueAs<ModuleNameInfo *>();
-    ConsumeAnnotationToken();
-    if (Info->hasPartitionName()) {
-      SourceLocation ColonLoc = Info->getColonToken().getLocation();
-      if (!getLangOpts().CPlusPlusModules)
-        Diag(ColonLoc, diag::err_unsupported_module_partition)
-            << SourceRange(ColonLoc, Path.back().second);
-      // Recover by leaving partition empty.
-      else if (ParseModuleName(ColonLoc, Info->getPartitionName(), Path,
-                               /*IsImport=*/true))
-        return nullptr;
-      else
-        IsPartition = true;
-    } else if (ParseModuleName(ImportLoc, Info->getModuleName(), Path,
-                               /*IsImport=*/true))
+    else
+      IsPartition = true;
+  } else {
+    if (ParseModuleName(ImportLoc, Path, /*IsImport*/ true))
       return nullptr;
   }
 
@@ -2713,31 +2689,32 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
 ///         module-name-qualifier:
 ///           module-name-qualifier[opt] identifier '.'
 bool Parser::ParseModuleName(
-    SourceLocation UseLoc, ArrayRef<Token> ModuleName,
+    SourceLocation UseLoc,
     SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path,
     bool IsImport) {
-  ModuleNameInfo::getModuleIdPath(ModuleName, Path);
-  // Eg. import A.B.
-  if (ModuleName.back().isNot(tok::identifier)) {
-    if (Tok.is(tok::code_completion)) {
-      cutOffParsing();
-      Actions.CodeCompletion().CodeCompleteModuleImport(UseLoc, Path);
+  // Parse the module path.
+  while (true) {
+    if (!Tok.is(tok::identifier)) {
+      if (Tok.is(tok::code_completion)) {
+        cutOffParsing();
+        Actions.CodeCompletion().CodeCompleteModuleImport(UseLoc, Path);
+        return true;
+      }
+
+      Diag(Tok, diag::err_module_expected_ident) << IsImport;
+      SkipUntil(tok::semi);
       return true;
     }
-    Diag(ModuleName.back(), diag::err_module_expected_ident) << IsImport;
-    SkipUntil(tok::semi, StopBeforeMatch);
-    return true;
-  }
 
-  // [cpp.module]/p2: where the pp-tokens (if any) shall not begin with a (
-  // preprocessing token [...]
-  //
-  // Skip unitl ';' to recovery.
-  if (getLangOpts().CPlusPlusModules && Tok.is(tok::l_paren)) {
-    SkipUntil(tok::semi, StopBeforeMatch);
-    return true;
+    // Record this part of the module path.
+    Path.push_back(std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation()));
+    ConsumeToken();
+
+    if (Tok.isNot(tok::period))
+      return false;
+
+    ConsumeToken();
   }
-  return false;
 }
 
 /// Try recover parser when module annotation appears where it must not
diff --git a/clang/test/CXX/cpp/cpp.module/p2.cppm b/clang/test/CXX/cpp/cpp.module/p2.cppm
deleted file mode 100644
index 966a88ccfa972..0000000000000
--- a/clang/test/CXX/cpp/cpp.module/p2.cppm
+++ /dev/null
@@ -1,88 +0,0 @@
-// RUN: rm -rf %t
-// RUN: mkdir -p %t
-// RUN: split-file %s %t
-
-// RUN: %clang_cc1 -std=c++20 %t/A.cppm -triple x86_64-linux-gnu -verify
-// RUN: %clang_cc1 -std=c++20 %t/B.cppm -triple x86_64-linux-gnu -verify
-// RUN: %clang_cc1 -std=c++20 %t/C.cppm -triple x86_64-linux-gnu -verify
-// RUN: %clang_cc1 -std=c++20 %t/D.cppm -triple x86_64-linux-gnu -verify
-// RUN: %clang_cc1 -std=c++20 %t/E.cppm -triple x86_64-linux-gnu -verify
-// RUN: %clang_cc1 -std=c++20 %t/F.cppm -triple x86_64-linux-gnu -verify
-// RUN: %clang_cc1 -std=c++20 %t/G.cppm -triple x86_64-linux-gnu -verify
-// RUN: %clang_cc1 -std=c++20 %t/H.cppm -triple x86_64-linux-gnu -verify
-// RUN: %clang_cc1 -std=c++20 %t/I.cppm -triple x86_64-linux-gnu -verify
-// RUN: %clang_cc1 -std=c++20 %t/J.cppm -triple x86_64-linux-gnu -verify
-
-//--- version.h
-#ifndef VERSION_H
-#define VERSION_H
-
-#define VERSION libv5
-#define A a
-#define B b
-#define C c
-#define FUNC_LIKE(X) function_like_##X
-#define ATTRS [[]]
-#define SEMICOLON ;
-
-#endif // VERSION_H
-
-//--- A.cppm
-module;
-#include "version.h"
-export module VERSION;  // expected-error {{the module name in a module declaration cannot contain an object-like macro 'VERSION'}}
-
-//--- B.cppm
-module;
-#include "version.h"
-export module A.B;      // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
-                        // expected-error {{the module name in a module declaration cannot contain an object-like macro 'B'}}
-
-//--- C.cppm
-module;                             // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
-#include "version.h"
-export module A.FUNC_LIKE(foo):C;   // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
-                                    // expected-error {{unexpected '(' after the module name in a module declaration}}
-
-//--- D.cppm
-module;                               // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
-#include "version.h"
-export module B.A.FUNC_LIKE(bar):C;   // expected-error {{the module name in a module declaration cannot contain an object-like macro 'B'}} \
-                                      // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
-                                      // expected-error {{unexpected '(' after the module name in a module declaration}}
-
-//--- E.cppm
-module;
-#include "version.h"
-export module a.FUNC_LIKE:c; // OK, FUNC_LIKE would not be treated as a macro name.
-// expected-no-diagnostics
-
-//--- F.cppm
-module;
-#include "version.h"
-export module a.FUNC_LIKE:c ATTRS; // OK, FUNC_LIKE would not be treated as a macro name.
-// expected-no-diagnostics
-
-//--- G.cppm
-module;                               // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
-#include "version.h"
-export module A.FUNC_LIKE(B c:C ATTRS // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
-                                      // expected-error {{unexpected '(' after the module name in a module declaration}}
-
-//--- H.cppm
-module;                                   // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
-#include "version.h"
-export module A.FUNC_LIKE(B,). c:C ATTRS  // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
-                                          // expected-error {{unexpected '(' after the module name in a module declaration}}
-
-//--- I.cppm
-module;                                   // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
-#include "version.h"
-export module A.FUNC_LIKE(B,) c:C ATTRS   // expected-error {{the module name in a module declaration cannot contain an object-like macro 'A'}} \
-                                          // expected-error {{unexpected '(' after the module name in a module declaration}}
-
-//--- J.cppm
-module;
-#include "version.h"
-export module unexpanded : unexpanded ATTRS SEMICOLON // OK, ATTRS and SEMICOLON can be expanded.
-// expected-no-diagnostics
diff --git a/clang/test/CXX/module/basic/basic.link/module-declaration.cpp b/clang/test/CXX/module/basic/basic.link/module-declaration.cpp
index 14bbc911febfc..d71358cc7a571 100644
--- a/clang/test/CXX/module/basic/basic.link/module-declaration.cpp
+++ b/clang/test/CXX/module/basic/basic.link/module-declaration.cpp
@@ -8,19 +8,27 @@
 // RUN: %clang_cc1 -std=c++20 -emit-module-interface -fmodule-file=x=%t/x.pcm %t/x.y.cppm -o %t/x.y.pcm
 //
 // Module implementation for unknown and known module. (The former is ill-formed.)
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify -x c++ %t/M1.cpp
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x=%t/x.pcm -fmodule-file=x.y=%t/x.y.pcm -verify -x c++ %t/M2.cpp
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify -x c++ %t/M.cpp \
+// RUN:            -DTEST=1 -DEXPORT= -DMODULE_NAME=z
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x=%t/x.pcm -fmodule-file=x.y=%t/x.y.pcm -verify -x c++ %t/M.cpp \
+// RUN:            -DTEST=2 -DEXPORT= -DMODULE_NAME=x
 //
 // Module interface for unknown and known module. (The latter is ill-formed due to
 // redefinition.)
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M3.cpp
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M4.cpp
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
+// RUN:            -DTEST=3 -DEXPORT=export -DMODULE_NAME=z
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
+// RUN:            -DTEST=4 -DEXPORT=export -DMODULE_NAME=x
 //
 // Miscellaneous syntax.
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M5.cpp
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M6.cpp
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M7.cpp
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M8.cpp
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
+// RUN:            -DTEST=7 -DEXPORT=export -DMODULE_NAME='z elderberry'
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
+// RUN:            -DTEST=8 -DEXPORT=export -DMODULE_NAME='z [[]]'
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
+// RUN:            -DTEST=9 -DEXPORT=export -DMODULE_NAME='z [[fancy]]'
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -verify %t/M.cpp \
+// RUN:            -DTEST=10 -DEXPORT=export -DMODULE_NAME='z [[maybe_unused]]'
 
 //--- x.cppm
 export module x;
@@ -30,26 +38,17 @@ int a, b;
 export module x.y;
 int c;
 
-//--- M1.cpp
-module z; // expected-error {{module 'z' not found}}
-
-//--- M2.cpp
-module x; // expected-no-diagnostics
-
-//--- M3.cpp
-export module z; // expected-no-diagnostics
-
-//--- M4.cpp
-export module x; // expected-no-diagnostics
-
-//--- M5.cpp
-export module z elderberry; // expected-error {{expected ';'}} expected-error {{a type specifier is required}}
-
-//--- M6.cpp
-export module z [[]]; // expected-no-diagnostics
-
-//--- M7.cpp
-export module z [[fancy]]; // expected-warning {{unknown attribute 'fancy' ignored}}
-
-//--- M8.cpp
-export module z [[maybe_unused]]; // expected-error-re {{'maybe_unused' attribute cannot be applied to a module{{$}}}}
+//--- M.cpp
+
+EXPORT module MODULE_NAME;
+#if TEST == 7
+// expected-error@-2 {{expected ';'}} expected-error@-2 {{a type specifier is required}}
+#elif TEST == 9
+// expected-warning@-4 {{unknown attribute 'fancy' ignored}}
+#elif TEST == 10
+// expected-error-re@-6 {{'maybe_unused' attribute cannot be applied to a module{{$}}}}
+#elif TEST == 1
+// expected-error@-8 {{module 'z' not found}}
+#else
+// expected-no-diagnostics
+#endif
diff --git a/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm b/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm
index ecad4db32a7e9..873e4c0edeac2 100644
--- a/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm
+++ b/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm
@@ -6,12 +6,10 @@
 // RUN: %clang_cc1 -std=c++20 -emit-module-interface -fmodule-file=x=%t/x.pcm %t/x.y.cppm -o %t/x.y.pcm
 // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/a.b.cppm -o %t/a.b.pcm
 //
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -fmodule-file=x=%t/x.pcm -verify %t/test-interface.cpp \
-// RUN:            -DINTERFACE
+// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -fmodule-file=x=%t/x.pcm -verify %t/test.cpp \
+// RUN:            -DMODULE_NAME=z -DINTERFACE
 // RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -fmodule-file=x=%t/x.pcm \
-// RUN:            -fmodule-file=a.b=%t/a.b.pcm -verify %t/test.cpp
-// RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -fmodule-file=x=%t/x.pcm \
-// RUN:            -verify %t/test-module-not-found.cpp
+// RUN:            -fmodule-file=a.b=%t/a.b.pcm -verify %t/test.cpp -DMODULE_NAME=a.b
 // RUN: %clang_cc1 -std=c++20 -I%t -fmodule-file=x.y=%t/x.y.pcm -fmodule-file=x=%t/x.pcm -verify %t/test.x.cpp
 
 //--- x.cppm
@@ -36,26 +34,11 @@ int use_2 = b; // ok
 int use_3 = c; // expected-error {{use of undeclared identifier 'c'}}
 
 //--- test.cpp
-module;
-module a.b;
-
-import x;
-
-import x [[]];
-import x [[foo]]; // expected-warning {{unknown attribute 'foo' ignored}}
-import x [[noreturn]]; // expected-error {{'noreturn' attribute cannot be applied to a module import}}
-import x [[blarg::noreturn]]; // expected-warning {{unknown attribute 'noreturn' ignored}}
-
-import x.y;
-import x.; // expected-error {{expected a module name after 'import'}}
-import .x; // expected-error {{expected a module name after 'import'}}
-
-int use_4 = c; // ok
-
-
-//--- test-interface.cpp
-module;
-export module z;
+#ifdef INTERFACE
+export module MODULE_NAME;
+#else
+module MODULE_NAME;
+#endif
 
 import x;
 
@@ -68,10 +51,6 @@ import x.y;
 import x.; // expected-error {{expected a module name after 'import'}}
 import .x; // expected-error {{expected a module name after 'import'}}
 
-int use_4 = c; // ok
-
-//--- test-module-not-found.cpp
-module;
-
 import blarg; // expected-error {{module 'blarg' not found}}
 
+int use_4 = c; // ok
diff --git a/clang/test/SemaCXX/modules.cppm b/clang/test/SemaCXX/modules.cppm
index 267417bf5da2c..41204be76eafa 100644
--- a/clang/test/SemaCXX/modules.cppm
+++ b/clang/test/SemaCXX/modules.cppm
@@ -1,17 +1,19 @@
-// RUN: rm -rf %t
-// RUN: mkdir -p %t
-// RUN: split-file %s %t
+// RUN:     %clang_cc1 -std=c++20 -emit-module-interface %s -o %t.0.pcm -verify -DTEST=0
+// RUN:     %clang_cc1 -std=c++20 -emit-module-interface %s -o %t.1.pcm -verify -DTEST=1
+// RUN:     %clang_cc1 -std=c++20 -emit-module-interface %s -fmodule-file=foo=%t.0.pcm -o %t.2.pcm -verify -DTEST=2
+// RUN:     %clang_cc1 -std=c++20 -emit-module-interface %s -fmodule-file=foo=%t.0.pcm -o %t.3.pcm -verify -Dfoo=bar -DTEST=3
 
-// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/A.cppm -o %t.0.pcm -verify
-// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/B.cppm -o %t.1.pcm -verify
-// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/C.cppm -fmodule-file=foo=%t.0.pcm -o %t.2.pcm -verify
-// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/D.cppm -fmodule-file=foo=%t.0.pcm -o %t.3.pcm -verify
-// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/E.cppm -fmodule-file=foo=%t.0.pcm -o %t.3.pcm -verify -Dfoo=bar
+#if TEST == 0 || TEST == 2
+// expected-no-diagnostics
+#endif
 
-//--- A.cppm
 export module foo;
+
 static int m;
+
 int n;
+
+#if TEST == 0
 export {
   int a;
   int b;
@@ -25,43 +27,7 @@ export void f() {}
 
 export struct T {
 } t;
-// expected-no-diagnostics
-
-//--- B.cppm
-export module foo;
-static int m;
-int n;
-struct S {
-  export int n;        // expected-error {{expected member name or ';'}}
-  export static int n; // expected-error {{expected member name or ';'}}
-};
-
-// FIXME: Exports of declarations without external linkage are disallowed.
-// Exports of declarations with non-external-linkage types are disallowed.
-
-// Cannot export within another export. This isn't precisely covered by the
-// language rules right now, but (per personal correspondence between zygoloid
-// and gdr) is the intent.
-export { // expected-note {{export block begins here}}
-  extern "C++" {
-    namespace NestedExport {
-      export { // expected-error {{export declaration appears within another export declaration}}
-        int q;
-      }
-    } // namespace NestedExport
-  }
-}
-
-//--- C.cppm
-export module foo;
-static int m;
-int n;
-// expected-no-diagnostics
-
-//--- D.cppm
-export module foo;
-static int m;
-int n;
+#elif TEST == 3
 int use_a = a; // expected-error {{use of undeclared identifier 'a'}}
 
 #undef foo
@@ -80,12 +46,29 @@ int use_n = n; // FIXME: this should not be visible, because it is not exported
 
 extern int n;
 static_assert(&n != p); // expected-error{{use of undeclared identifier 'p'}}
+#endif
 
-//--- E.cppm
-export module foo; // expected-error {{the module name in a module declaration cannot contain an object-like macro 'foo'}}
-static int m;
-int n;
-int use_a = a; // expected-error {{use of undeclared identifier 'a'}}
+#if TEST == 1
+struct S {
+  export int n;        // expected-error {{expected member name or ';'}}
+  export static int n; // expected-error {{expected member name or ';'}}
+};
+#endif
 
-#undef foo
-import foo; // expected-error {{imports must immediately follow the module declaration}}
+// FIXME: Exports of declarations without external linkage are disallowed.
+// Exports of declarations with non-external-linkage types are disallowed.
+
+// Cannot export within another export. This isn't precisely covered by the
+// language rules right now, but (per personal correspondence between zygoloid
+// and gdr) is the intent.
+#if TEST == 1
+export { // expected-note {{export block begins here}}
+  extern "C++" {
+  namespace NestedExport {
+  export { // expected-error {{export declaration appears within another export declaration}}
+    int q;
+  }
+  } // namespace NestedExport
+  }
+}
+#endif
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 1f69a4e8a5620..a6ded8be3ae9e 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -182,7 +182,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Module Declarations Shouldn’t be Macros</td>
   <td><a href="https://wg21.link/P3034R1">P3034R1</a> (<a href="#dr">DR</a>)</td>
-  <td class="unreleased" align="center">Clang 19</td>
+  <td class="none" align="center">No</td>
  </tr>
  <tr>
   <td>Trivial infinite loops are not Undefined Behavior</td>

From 9d2f81ea85e410da9d79b9dfad9b9769525e4387 Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Mon, 22 Jul 2024 06:26:08 +0100
Subject: [PATCH 370/486] [compiler-rt] fix couple of netbsd build warnings.
 (#99745)

---
 .../lib/sanitizer_common/sanitizer_common_interceptors.inc    | 2 +-
 compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cpp   | 3 ++-
 .../lib/sanitizer_common/sanitizer_stacktrace_printer.cpp     | 4 ++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 032b04a09ae76..49c9dcbef358f 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -6336,7 +6336,7 @@ INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
 
       const char *SelfFName = DladdrSelfFName();
       VPrintf(1, "dlopen interceptor: DladdrSelfFName: %p %s\n",
-              (void *)SelfFName, SelfFName);
+              (const void *)SelfFName, SelfFName);
 
       if (SelfFName && internal_strcmp(SelfFName, filename) == 0) {
         // It's possible they copied the string from dladdr, so
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cpp
index 7d7c009f64442..ee3fd5b0a817a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cpp
@@ -47,7 +47,8 @@ void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
   struct kinfo_proc2 *InfoProc;
   uptr Len = sizeof(*InfoProc);
   uptr Size = Len;
-  const int Mib[] = {CTL_KERN, KERN_PROC2, KERN_PROC_PID, getpid(), Size, 1};
+  const int Mib[] = {CTL_KERN, KERN_PROC2, KERN_PROC_PID,
+                     getpid(), (int)Size,  1};
   InfoProc = (struct kinfo_proc2 *)MmapOrDie(Size, "GetMemoryProfile()");
   CHECK_EQ(
       internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)InfoProc, &Len, 0),
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
index dddae441d5de9..5631d132da74a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
@@ -269,7 +269,7 @@ void FormattedStackTracePrinter::RenderFrame(InternalScopedString *buffer,
       break;
     default:
       Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
-             (void *)p);
+             (const void *)p);
       Die();
     }
   }
@@ -323,7 +323,7 @@ void FormattedStackTracePrinter::RenderData(InternalScopedString *buffer,
         break;
       default:
         Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
-               (void *)p);
+               (const void *)p);
         Die();
     }
   }

From 58854facb30f7c660c1fd05376f8a4722eeefb9e Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Mon, 22 Jul 2024 13:56:10 +0800
Subject: [PATCH 371/486] [RISCV] Don't cost vector arithmetic fp ops as
 cheaper than scalar (#99594)

I was comparing some SPEC CPU 2017 benchmarks across rva22u64 and
rva22u64_v, and noticed that in a few cases that rva22u64_v was
considerably slower.

One of them was 519.lbm_r, which has a large loop that was being
unprofitably vectorized. It has an if/else in the loop which requires
large amounts of predication when vectorized, but despite the loop
vectorizer taking this into account the vector cost came out as cheaper
than the scalar.

It looks like the reason for this is because we cost scalar floating
point ops as 2, but their vector equivalents as 1 (for LMUL 1). This
comes from how we use BasicTTIImpl for scalars which treats floats as
twice as expensive as integers.

This patch doubles the cost of vector floating point arithmetic ops so
that they're at least as expensive as their scalar counterparts, which
gives a 13% speedup on 519.lbm_r at -O3 on the spacemit-x60.

Fixes #62576 (the last point there about scalar fsub/fmul)
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  11 +-
 .../test/Analysis/CostModel/RISCV/arith-fp.ll | 300 +++++++++---------
 .../CostModel/RISCV/rvv-intrinsics.ll         | 130 ++++----
 .../RISCV/masked_gather_scatter.ll            |  84 ++---
 .../RISCV/riscv-vector-reverse.ll             | 206 ++++++++----
 .../RISCV/vpintrin-scalarization.ll           | 100 ++++--
 6 files changed, 484 insertions(+), 347 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index f9eef60f77b7a..5a92d6bab31a9 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1688,7 +1688,6 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
                                          Args, CxtI);
 
-
   auto getConstantMatCost =
     [&](unsigned Operand, TTI::OperandValueInfo OpInfo) -> InstructionCost {
     if (OpInfo.isUniform() && TLI->canSplatOperand(Opcode, Operand))
@@ -1760,8 +1759,14 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
                                                            Op1Info, Op2Info,
                                                            Args, CxtI);
   }
-  return ConstantMatCost +
-         LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);
+
+  InstructionCost InstrCost = getRISCVInstructionCost(Op, LT.second, CostKind);
+  // We use BasicTTIImpl to calculate scalar costs, which assumes floating point
+  // ops are twice as expensive as integer ops. Do the same for vectors so
+  // scalar floating point ops aren't cheaper than their vector equivalents.
+  if (Ty->isFPOrFPVectorTy())
+    InstrCost *= 2;
+  return ConstantMatCost + LT.first * InstrCost;
 }
 
 // TODO: Deduplicate from TargetTransformInfoImplCRTPBase.
diff --git a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
index d1e8bb015491e..5236f5a3bae95 100644
--- a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
@@ -8,36 +8,36 @@ define i32 @fadd() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fadd half undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fadd float undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fadd double undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = fadd <1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fadd <2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fadd <4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fadd <8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fadd <16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32F16 = fadd <32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = fadd <vscale x 1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = fadd <vscale x 2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = fadd <vscale x 4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F16 = fadd <vscale x 8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16F16 = fadd <vscale x 16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32F16 = fadd <vscale x 32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F32 = fadd <1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fadd <2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fadd <4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fadd <8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fadd <16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F32 = fadd <vscale x 1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F32 = fadd <vscale x 2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F32 = fadd <vscale x 4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F32 = fadd <vscale x 8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F32 = fadd <vscale x 16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = fadd <1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fadd <2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fadd <4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fadd <8 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F64 = fadd <vscale x 1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F64 = fadd <vscale x 2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F64 = fadd <vscale x 4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F64 = fadd <vscale x 8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F16 = fadd <1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16 = fadd <2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16 = fadd <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16 = fadd <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fadd <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32F16 = fadd <32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F16 = fadd <vscale x 1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F16 = fadd <vscale x 2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F16 = fadd <vscale x 4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F16 = fadd <vscale x 8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F16 = fadd <vscale x 16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32F16 = fadd <vscale x 32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F32 = fadd <1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fadd <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fadd <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fadd <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fadd <16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F32 = fadd <vscale x 1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F32 = fadd <vscale x 2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F32 = fadd <vscale x 4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F32 = fadd <vscale x 8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16F32 = fadd <vscale x 16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = fadd <1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fadd <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fadd <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fadd <8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F64 = fadd <vscale x 1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2F64 = fadd <vscale x 2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4F64 = fadd <vscale x 4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8F64 = fadd <vscale x 8 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %F16 = fadd half undef, undef
@@ -88,36 +88,36 @@ define i32 @fsub() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fsub half undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fsub float undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fsub double undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = fsub <1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fsub <2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fsub <4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fsub <8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fsub <16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32F16 = fsub <32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = fsub <vscale x 1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = fsub <vscale x 2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = fsub <vscale x 4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F16 = fsub <vscale x 8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16F16 = fsub <vscale x 16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32F16 = fsub <vscale x 32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F32 = fsub <1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fsub <2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fsub <4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fsub <8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fsub <16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F32 = fsub <vscale x 1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F32 = fsub <vscale x 2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F32 = fsub <vscale x 4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F32 = fsub <vscale x 8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F32 = fsub <vscale x 16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = fsub <1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fsub <2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fsub <4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fsub <8 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F64 = fsub <vscale x 1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F64 = fsub <vscale x 2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F64 = fsub <vscale x 4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F64 = fsub <vscale x 8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F16 = fsub <1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16 = fsub <2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16 = fsub <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16 = fsub <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fsub <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32F16 = fsub <32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F16 = fsub <vscale x 1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F16 = fsub <vscale x 2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F16 = fsub <vscale x 4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F16 = fsub <vscale x 8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F16 = fsub <vscale x 16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32F16 = fsub <vscale x 32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F32 = fsub <1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fsub <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fsub <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fsub <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fsub <16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F32 = fsub <vscale x 1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F32 = fsub <vscale x 2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F32 = fsub <vscale x 4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F32 = fsub <vscale x 8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16F32 = fsub <vscale x 16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = fsub <1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fsub <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fsub <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fsub <8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F64 = fsub <vscale x 1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2F64 = fsub <vscale x 2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4F64 = fsub <vscale x 4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8F64 = fsub <vscale x 8 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %F16 = fsub half undef, undef
@@ -168,36 +168,36 @@ define i32 @fmul() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fmul half undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fmul float undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fmul double undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = fmul <1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fmul <2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fmul <4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fmul <8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fmul <16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32F16 = fmul <32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = fmul <vscale x 1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = fmul <vscale x 2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = fmul <vscale x 4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F16 = fmul <vscale x 8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16F16 = fmul <vscale x 16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32F16 = fmul <vscale x 32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F32 = fmul <1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fmul <2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fmul <4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fmul <8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fmul <16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F32 = fmul <vscale x 1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F32 = fmul <vscale x 2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F32 = fmul <vscale x 4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F32 = fmul <vscale x 8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F32 = fmul <vscale x 16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = fmul <1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fmul <2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fmul <4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fmul <8 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F64 = fmul <vscale x 1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F64 = fmul <vscale x 2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F64 = fmul <vscale x 4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F64 = fmul <vscale x 8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F16 = fmul <1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16 = fmul <2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16 = fmul <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16 = fmul <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fmul <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32F16 = fmul <32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F16 = fmul <vscale x 1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F16 = fmul <vscale x 2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F16 = fmul <vscale x 4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F16 = fmul <vscale x 8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F16 = fmul <vscale x 16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32F16 = fmul <vscale x 32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F32 = fmul <1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fmul <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fmul <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fmul <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fmul <16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F32 = fmul <vscale x 1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F32 = fmul <vscale x 2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F32 = fmul <vscale x 4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F32 = fmul <vscale x 8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16F32 = fmul <vscale x 16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = fmul <1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fmul <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fmul <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fmul <8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F64 = fmul <vscale x 1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2F64 = fmul <vscale x 2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4F64 = fmul <vscale x 4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8F64 = fmul <vscale x 8 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %F16 = fmul half undef, undef
@@ -248,36 +248,36 @@ define i32 @fdiv() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fdiv half undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fdiv float undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fdiv double undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = fdiv <1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fdiv <2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fdiv <4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fdiv <8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fdiv <16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32F16 = fdiv <32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = fdiv <vscale x 1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = fdiv <vscale x 2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = fdiv <vscale x 4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F16 = fdiv <vscale x 8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16F16 = fdiv <vscale x 16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32F16 = fdiv <vscale x 32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F32 = fdiv <1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fdiv <2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fdiv <4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fdiv <8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fdiv <16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F32 = fdiv <vscale x 1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F32 = fdiv <vscale x 2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F32 = fdiv <vscale x 4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F32 = fdiv <vscale x 8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F32 = fdiv <vscale x 16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = fdiv <1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fdiv <2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fdiv <4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fdiv <8 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F64 = fdiv <vscale x 1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F64 = fdiv <vscale x 2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F64 = fdiv <vscale x 4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F64 = fdiv <vscale x 8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F16 = fdiv <1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16 = fdiv <2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16 = fdiv <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16 = fdiv <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fdiv <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32F16 = fdiv <32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F16 = fdiv <vscale x 1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F16 = fdiv <vscale x 2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F16 = fdiv <vscale x 4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F16 = fdiv <vscale x 8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F16 = fdiv <vscale x 16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32F16 = fdiv <vscale x 32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F32 = fdiv <1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fdiv <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fdiv <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fdiv <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fdiv <16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F32 = fdiv <vscale x 1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F32 = fdiv <vscale x 2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F32 = fdiv <vscale x 4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F32 = fdiv <vscale x 8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16F32 = fdiv <vscale x 16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = fdiv <1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fdiv <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fdiv <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fdiv <8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F64 = fdiv <vscale x 1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2F64 = fdiv <vscale x 2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4F64 = fdiv <vscale x 4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8F64 = fdiv <vscale x 8 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %F16 = fdiv half undef, undef
@@ -408,36 +408,36 @@ define i32 @fneg() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fneg half undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fneg float undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fneg double undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = fneg <1 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fneg <16 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32F16 = fneg <32 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = fneg <vscale x 1 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = fneg <vscale x 2 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = fneg <vscale x 4 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F16 = fneg <vscale x 8 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16F16 = fneg <vscale x 16 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32F16 = fneg <vscale x 32 x half> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F32 = fneg <1 x float> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fneg <8 x float> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fneg <16 x float> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F32 = fneg <vscale x 1 x float> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F32 = fneg <vscale x 2 x float> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F32 = fneg <vscale x 4 x float> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F32 = fneg <vscale x 8 x float> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F32 = fneg <vscale x 16 x float> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = fneg <1 x double> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fneg <4 x double> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fneg <8 x double> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F64 = fneg <vscale x 1 x double> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F64 = fneg <vscale x 2 x double> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F64 = fneg <vscale x 4 x double> undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F64 = fneg <vscale x 8 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F16 = fneg <1 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16 = fneg <2 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16 = fneg <4 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16 = fneg <8 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fneg <16 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32F16 = fneg <32 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F16 = fneg <vscale x 1 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F16 = fneg <vscale x 2 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F16 = fneg <vscale x 4 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F16 = fneg <vscale x 8 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F16 = fneg <vscale x 16 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32F16 = fneg <vscale x 32 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F32 = fneg <1 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fneg <2 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fneg <4 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fneg <8 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fneg <16 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F32 = fneg <vscale x 1 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F32 = fneg <vscale x 2 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F32 = fneg <vscale x 4 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F32 = fneg <vscale x 8 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16F32 = fneg <vscale x 16 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = fneg <1 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fneg <2 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fneg <4 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fneg <8 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F64 = fneg <vscale x 1 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2F64 = fneg <vscale x 2 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4F64 = fneg <vscale x 4 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8F64 = fneg <vscale x 8 x double> undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %F16 = fneg half undef
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-intrinsics.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-intrinsics.ll
index 87ffb23dcb88e..67c081ba5d3c6 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-intrinsics.ll
@@ -21,7 +21,7 @@ define void @unsupported_fp_ops(<vscale x 4 x float> %vec, i32 %extraarg) {
 
 define void @powi(<vscale x 4 x float> %vec) {
 ; CHECK-LABEL: 'powi'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPEBASED-LABEL: 'powi'
@@ -1383,73 +1383,73 @@ define void @reduce_fadd() {
 
 define void @vp_fadd(){
 ; CHECK-LABEL: 'vp_fadd'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x float> @llvm.vp.fadd.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t1 = fadd <2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x float> @llvm.vp.fadd.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t3 = fadd <4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t4 = call <8 x float> @llvm.vp.fadd.v8f32(<8 x float> undef, <8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t5 = fadd <8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t6 = call <16 x float> @llvm.vp.fadd.v16f32(<16 x float> undef, <16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t7 = fadd <16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x double> @llvm.vp.fadd.v2f64(<2 x double> undef, <2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t9 = fadd <2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t10 = call <4 x double> @llvm.vp.fadd.v4f64(<4 x double> undef, <4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t11 = fadd <4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t12 = call <8 x double> @llvm.vp.fadd.v8f64(<8 x double> undef, <8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t13 = fadd <8 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t14 = call <16 x double> @llvm.vp.fadd.v16f64(<16 x double> undef, <16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t15 = fadd <16 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t17 = call <vscale x 2 x float> @llvm.vp.fadd.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = fadd <vscale x 2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t19 = call <vscale x 4 x float> @llvm.vp.fadd.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t20 = fadd <vscale x 4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t21 = call <vscale x 8 x float> @llvm.vp.fadd.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t22 = fadd <vscale x 8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t23 = call <vscale x 16 x float> @llvm.vp.fadd.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t24 = fadd <vscale x 16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t25 = call <vscale x 2 x double> @llvm.vp.fadd.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = fadd <vscale x 2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t27 = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = fadd <vscale x 4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fadd.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = fadd <vscale x 8 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fadd.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t32 = fadd <vscale x 16 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t0 = call <2 x float> @llvm.vp.fadd.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = fadd <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = call <4 x float> @llvm.vp.fadd.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t3 = fadd <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = call <8 x float> @llvm.vp.fadd.v8f32(<8 x float> undef, <8 x float> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t5 = fadd <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t6 = call <16 x float> @llvm.vp.fadd.v16f32(<16 x float> undef, <16 x float> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t7 = fadd <16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = call <2 x double> @llvm.vp.fadd.v2f64(<2 x double> undef, <2 x double> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t9 = fadd <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10 = call <4 x double> @llvm.vp.fadd.v4f64(<4 x double> undef, <4 x double> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t11 = fadd <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t12 = call <8 x double> @llvm.vp.fadd.v8f64(<8 x double> undef, <8 x double> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t13 = fadd <8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t14 = call <16 x double> @llvm.vp.fadd.v16f64(<16 x double> undef, <16 x double> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t15 = fadd <16 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t17 = call <vscale x 2 x float> @llvm.vp.fadd.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t18 = fadd <vscale x 2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t19 = call <vscale x 4 x float> @llvm.vp.fadd.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t20 = fadd <vscale x 4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t21 = call <vscale x 8 x float> @llvm.vp.fadd.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t22 = fadd <vscale x 8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t23 = call <vscale x 16 x float> @llvm.vp.fadd.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t24 = fadd <vscale x 16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x double> @llvm.vp.fadd.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t26 = fadd <vscale x 2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t28 = fadd <vscale x 4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fadd.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t30 = fadd <vscale x 8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fadd.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t32 = fadd <vscale x 16 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPEBASED-LABEL: 'vp_fadd'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x float> @llvm.vp.fadd.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t1 = fadd <2 x float> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x float> @llvm.vp.fadd.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t3 = fadd <4 x float> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t4 = call <8 x float> @llvm.vp.fadd.v8f32(<8 x float> undef, <8 x float> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t5 = fadd <8 x float> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t6 = call <16 x float> @llvm.vp.fadd.v16f32(<16 x float> undef, <16 x float> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t7 = fadd <16 x float> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x double> @llvm.vp.fadd.v2f64(<2 x double> undef, <2 x double> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t9 = fadd <2 x double> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t10 = call <4 x double> @llvm.vp.fadd.v4f64(<4 x double> undef, <4 x double> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t11 = fadd <4 x double> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t12 = call <8 x double> @llvm.vp.fadd.v8f64(<8 x double> undef, <8 x double> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t13 = fadd <8 x double> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t14 = call <16 x double> @llvm.vp.fadd.v16f64(<16 x double> undef, <16 x double> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t15 = fadd <16 x double> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t17 = call <vscale x 2 x float> @llvm.vp.fadd.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = fadd <vscale x 2 x float> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t19 = call <vscale x 4 x float> @llvm.vp.fadd.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t20 = fadd <vscale x 4 x float> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t21 = call <vscale x 8 x float> @llvm.vp.fadd.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t22 = fadd <vscale x 8 x float> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t23 = call <vscale x 16 x float> @llvm.vp.fadd.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t24 = fadd <vscale x 16 x float> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t25 = call <vscale x 2 x double> @llvm.vp.fadd.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = fadd <vscale x 2 x double> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t27 = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = fadd <vscale x 4 x double> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fadd.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = fadd <vscale x 8 x double> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fadd.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t32 = fadd <vscale x 16 x double> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t0 = call <2 x float> @llvm.vp.fadd.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = fadd <2 x float> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = call <4 x float> @llvm.vp.fadd.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t3 = fadd <4 x float> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = call <8 x float> @llvm.vp.fadd.v8f32(<8 x float> undef, <8 x float> undef, <8 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t5 = fadd <8 x float> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t6 = call <16 x float> @llvm.vp.fadd.v16f32(<16 x float> undef, <16 x float> undef, <16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t7 = fadd <16 x float> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = call <2 x double> @llvm.vp.fadd.v2f64(<2 x double> undef, <2 x double> undef, <2 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t9 = fadd <2 x double> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10 = call <4 x double> @llvm.vp.fadd.v4f64(<4 x double> undef, <4 x double> undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t11 = fadd <4 x double> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t12 = call <8 x double> @llvm.vp.fadd.v8f64(<8 x double> undef, <8 x double> undef, <8 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t13 = fadd <8 x double> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t14 = call <16 x double> @llvm.vp.fadd.v16f64(<16 x double> undef, <16 x double> undef, <16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t15 = fadd <16 x double> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t17 = call <vscale x 2 x float> @llvm.vp.fadd.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t18 = fadd <vscale x 2 x float> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t19 = call <vscale x 4 x float> @llvm.vp.fadd.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t20 = fadd <vscale x 4 x float> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t21 = call <vscale x 8 x float> @llvm.vp.fadd.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t22 = fadd <vscale x 8 x float> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t23 = call <vscale x 16 x float> @llvm.vp.fadd.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t24 = fadd <vscale x 16 x float> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x double> @llvm.vp.fadd.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t26 = fadd <vscale x 2 x double> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t28 = fadd <vscale x 4 x double> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fadd.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t30 = fadd <vscale x 8 x double> undef, undef
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fadd.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t32 = fadd <vscale x 16 x double> undef, undef
 ; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x float> @llvm.vp.fadd.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
index e50d7362365b8..a151232df0cd5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -39,32 +39,32 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV32-NEXT:    [[N_MOD_VF:%.*]] = urem i64 625, [[TMP4]]
 ; RV32-NEXT:    [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]]
 ; RV32-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 16
-; RV32-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 2
-; RV32-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
-; RV32-NEXT:    [[TMP6:%.*]] = add <vscale x 2 x i64> [[TMP5]], zeroinitializer
-; RV32-NEXT:    [[TMP7:%.*]] = mul <vscale x 2 x i64> [[TMP6]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 16, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; RV32-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP7]]
-; RV32-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
-; RV32-NEXT:    [[TMP10:%.*]] = mul i64 16, [[TMP9]]
-; RV32-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP10]], i64 0
+; RV32-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; RV32-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; RV32-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; RV32-NEXT:    [[TMP8:%.*]] = add <vscale x 2 x i64> [[TMP7]], zeroinitializer
+; RV32-NEXT:    [[TMP9:%.*]] = mul <vscale x 2 x i64> [[TMP8]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 16, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; RV32-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP9]]
+; RV32-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; RV32-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; RV32-NEXT:    [[TMP12:%.*]] = mul i64 16, [[TMP11]]
+; RV32-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i64 0
 ; RV32-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; RV32-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; RV32:       vector.body:
 ; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; RV32-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; RV32-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], <vscale x 2 x i64> [[VEC_IND]]
-; RV32-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> [[TMP11]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i32> poison), !alias.scope !0
-; RV32-NEXT:    [[TMP12:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 100, i64 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
-; RV32-NEXT:    [[TMP13:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; RV32-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, ptr [[B]], <vscale x 2 x i64> [[TMP13]]
-; RV32-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> [[TMP14]], i32 8, <vscale x 2 x i1> [[TMP12]], <vscale x 2 x double> poison), !alias.scope !3
-; RV32-NEXT:    [[TMP15:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
-; RV32-NEXT:    [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP15]]
-; RV32-NEXT:    [[TMP17:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
-; RV32-NEXT:    call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP16]], <vscale x 2 x ptr> [[TMP17]], i32 8, <vscale x 2 x i1> [[TMP12]]), !alias.scope !5, !noalias !7
-; RV32-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; RV32-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], <vscale x 2 x i64> [[VEC_IND]]
+; RV32-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> [[TMP13]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i32> poison), !alias.scope [[META0:![0-9]+]]
+; RV32-NEXT:    [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 100, i64 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
+; RV32-NEXT:    [[TMP15:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; RV32-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, ptr [[B]], <vscale x 2 x i64> [[TMP15]]
+; RV32-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> [[TMP14]], <vscale x 2 x double> poison), !alias.scope [[META3:![0-9]+]]
+; RV32-NEXT:    [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
+; RV32-NEXT:    [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
+; RV32-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
+; RV32-NEXT:    call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP18]], <vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> [[TMP14]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
+; RV32-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; RV32-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; RV32-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; RV32-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -121,32 +121,32 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 625, [[TMP4]]
 ; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]]
 ; RV64-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 16
-; RV64-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 2
-; RV64-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
-; RV64-NEXT:    [[TMP6:%.*]] = add <vscale x 2 x i64> [[TMP5]], zeroinitializer
-; RV64-NEXT:    [[TMP7:%.*]] = mul <vscale x 2 x i64> [[TMP6]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 16, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; RV64-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP7]]
-; RV64-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
-; RV64-NEXT:    [[TMP10:%.*]] = mul i64 16, [[TMP9]]
-; RV64-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP10]], i64 0
+; RV64-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; RV64-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; RV64-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; RV64-NEXT:    [[TMP8:%.*]] = add <vscale x 2 x i64> [[TMP7]], zeroinitializer
+; RV64-NEXT:    [[TMP9:%.*]] = mul <vscale x 2 x i64> [[TMP8]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 16, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; RV64-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP9]]
+; RV64-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; RV64-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; RV64-NEXT:    [[TMP12:%.*]] = mul i64 16, [[TMP11]]
+; RV64-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i64 0
 ; RV64-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; RV64-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; RV64:       vector.body:
 ; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; RV64-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; RV64-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], <vscale x 2 x i64> [[VEC_IND]]
-; RV64-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> [[TMP11]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i32> poison), !alias.scope !0
-; RV64-NEXT:    [[TMP12:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 100, i64 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
-; RV64-NEXT:    [[TMP13:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; RV64-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, ptr [[B]], <vscale x 2 x i64> [[TMP13]]
-; RV64-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> [[TMP14]], i32 8, <vscale x 2 x i1> [[TMP12]], <vscale x 2 x double> poison), !alias.scope !3
-; RV64-NEXT:    [[TMP15:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
-; RV64-NEXT:    [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP15]]
-; RV64-NEXT:    [[TMP17:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
-; RV64-NEXT:    call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP16]], <vscale x 2 x ptr> [[TMP17]], i32 8, <vscale x 2 x i1> [[TMP12]]), !alias.scope !5, !noalias !7
-; RV64-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; RV64-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], <vscale x 2 x i64> [[VEC_IND]]
+; RV64-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> [[TMP13]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i32> poison), !alias.scope [[META0:![0-9]+]]
+; RV64-NEXT:    [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 100, i64 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
+; RV64-NEXT:    [[TMP15:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; RV64-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, ptr [[B]], <vscale x 2 x i64> [[TMP15]]
+; RV64-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> [[TMP14]], <vscale x 2 x double> poison), !alias.scope [[META3:![0-9]+]]
+; RV64-NEXT:    [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
+; RV64-NEXT:    [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
+; RV64-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
+; RV64-NEXT:    call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP18]], <vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> [[TMP14]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
+; RV64-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; RV64-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; RV64-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; RV64-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 8e9713fecf29d..fc310f4163082 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -54,43 +54,46 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
-; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
-; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
-; CHECK:       ir-bb<for.body.preheader>:
-; CHECK-NEXT:    EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
+; CHECK-NEXT:  Live-in vp<%0> = VF * UF
+; CHECK-NEXT:  Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT:  vp<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.body.preheader>:
+; CHECK-NEXT:    EMIT vp<%2> = EXPAND SCEV (zext i32 %n to i64)
 ; CHECK-NEXT:  No successors
-; CHECK:       vector.ph:
+; CHECK-EMPTY:
+; CHECK-NEXT:  vector.ph:
 ; CHECK-NEXT:  Successor(s): vector loop
-; CHECK:       <x1> vector loop: {
+; CHECK-EMPTY:
+; CHECK-NEXT:  <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:    EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT:    vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
-; CHECK-NEXT:    vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT:    CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
-; CHECK-NEXT:    CLONE ir<%idxprom> = zext ir<%i.0>
-; CHECK-NEXT:    CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:    vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
-; CHECK-NEXT:    WIDEN ir<%1> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT:    WIDEN ir<%add9> = add ir<%1>, ir<1>
-; CHECK-NEXT:    CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:    vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
-; CHECK-NEXT:    WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
-; CHECK-NEXT:    EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT:    EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:      EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%8>
+; CHECK-NEXT:      vp<%4> = DERIVED-IV ir<%n> + vp<%3> * ir<-1>
+; CHECK-NEXT:      vp<%5> = SCALAR-STEPS vp<%4>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<%5>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
+; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
+; CHECK-NEXT:      vp<%6> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      WIDEN ir<%1> = load vp<%6>
+; CHECK-NEXT:      WIDEN ir<%add9> = add ir<%1>, ir<1>
+; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
+; CHECK-NEXT:      vp<%7> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      WIDEN store vp<%7>, ir<%add9>
+; CHECK-NEXT:      EMIT vp<%8> = add nuw vp<%3>, vp<%0>
+; CHECK-NEXT:      EMIT branch-on-count vp<%8>, vp<%1>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
-; CHECK:       middle.block:
-; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
-; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
+; CHECK-NEXT:  middle.block:
+; CHECK-NEXT:    EMIT vp<%10> = icmp eq vp<%2>, vp<%1>
+; CHECK-NEXT:    EMIT branch-on-cond vp<%10>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>
+; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
-; CHECK-NEXT:  scalar.ph
+; CHECK-NEXT:  scalar.ph:
 ; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
@@ -134,7 +137,52 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LEV: Epilogue vectorization is not profitable for this loop
 ; CHECK-NEXT:  VF picked by VPlan cost model: vscale x 4
 ; CHECK-NEXT:  Executing best plan with VF=vscale x 4, UF=1
-; CHECK:       LV: Interleaving disabled by the pass manager
+; CHECK-NEXT:  VPlan 'Final VPlan for VF={vscale x 4},UF>=1' {
+; CHECK-NEXT:  Live-in vp<%0> = VF * UF
+; CHECK-NEXT:  Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT:  vp<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.body.preheader>:
+; CHECK-NEXT:    EMIT vp<%2> = EXPAND SCEV (zext i32 %n to i64)
+; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  vector.ph:
+; CHECK-NEXT:  Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT:  <x1> vector loop: {
+; CHECK-NEXT:    vector.body:
+; CHECK-NEXT:      EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%8>
+; CHECK-NEXT:      vp<%4> = DERIVED-IV ir<%n> + vp<%3> * ir<-1>
+; CHECK-NEXT:      vp<%5> = SCALAR-STEPS vp<%4>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<%5>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
+; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
+; CHECK-NEXT:      vp<%6> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      WIDEN ir<%13> = load vp<%6>
+; CHECK-NEXT:      WIDEN ir<%add9> = add ir<%13>, ir<1>
+; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
+; CHECK-NEXT:      vp<%7> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      WIDEN store vp<%7>, ir<%add9>
+; CHECK-NEXT:      EMIT vp<%8> = add nuw vp<%3>, vp<%0>
+; CHECK-NEXT:      EMIT branch-on-count vp<%8>, vp<%1>
+; CHECK-NEXT:    No successors
+; CHECK-NEXT:  }
+; CHECK-NEXT:  Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT:  middle.block:
+; CHECK-NEXT:    EMIT vp<%10> = icmp eq vp<%2>, vp<%1>
+; CHECK-NEXT:    EMIT branch-on-cond vp<%10>
+; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
+; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  scalar.ph:
+; CHECK-NEXT:  No successors
+; CHECK-NEXT:  }
+; CHECK-NEXT:  LV: Loop does not require scalar epilogue
+; CHECK-NEXT:  LV: Loop does not require scalar epilogue
+; CHECK-NEXT:  LV: Interleaving disabled by the pass manager
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
 ; CHECK-NEXT:  LV: Vectorizing: innermost loop.
 ; CHECK-EMPTY:
@@ -193,7 +241,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %idxprom = zext i32 %i.0 to i64
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom
 ; CHECK-NEXT:  LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: %1 = load float, ptr %arrayidx, align 4
-; CHECK-NEXT:  LV: Found an estimated cost of 2 for VF vscale x 4 For instruction: %conv1 = fadd float %1, 1.000000e+00
+; CHECK-NEXT:  LV: Found an estimated cost of 4 for VF vscale x 4 For instruction: %conv1 = fadd float %1, 1.000000e+00
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom
 ; CHECK-NEXT:  LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: store float %conv1, ptr %arrayidx3, align 4
 ; CHECK-NEXT:  LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %cmp = icmp ugt i64 %indvars.iv, 1
@@ -210,37 +258,40 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
-; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
-; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
-; CHECK:       ir-bb<for.body.preheader>:
-; CHECK-NEXT:    EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
+; CHECK-NEXT:  Live-in vp<%0> = VF * UF
+; CHECK-NEXT:  Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT:  vp<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.body.preheader>:
+; CHECK-NEXT:    EMIT vp<%2> = EXPAND SCEV (zext i32 %n to i64)
 ; CHECK-NEXT:  No successors
-; CHECK:       vector.ph:
+; CHECK-EMPTY:
+; CHECK-NEXT:  vector.ph:
 ; CHECK-NEXT:  Successor(s): vector loop
-; CHECK:       <x1> vector loop: {
+; CHECK-EMPTY:
+; CHECK-NEXT:  <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:    EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT:    vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
-; CHECK-NEXT:    vp<[[STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT:    CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
-; CHECK-NEXT:    CLONE ir<%idxprom> = zext ir<%i.0>
-; CHECK-NEXT:    CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:    vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
-; CHECK-NEXT:    WIDEN ir<%1> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT:    WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00>
-; CHECK-NEXT:    CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:    vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
-; CHECK-NEXT:    WIDEN store vp<[[VEC_PTR2]]>, ir<%conv1>
-; CHECK-NEXT:    EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT:    EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:      EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%8>
+; CHECK-NEXT:      vp<%4> = DERIVED-IV ir<%n> + vp<%3> * ir<-1>
+; CHECK-NEXT:      vp<%5> = SCALAR-STEPS vp<%4>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<%5>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
+; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
+; CHECK-NEXT:      vp<%6> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      WIDEN ir<%1> = load vp<%6>
+; CHECK-NEXT:      WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00>
+; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
+; CHECK-NEXT:      vp<%7> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      WIDEN store vp<%7>, ir<%conv1>
+; CHECK-NEXT:      EMIT vp<%8> = add nuw vp<%3>, vp<%0>
+; CHECK-NEXT:      EMIT branch-on-count vp<%8>, vp<%1>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
-; CHECK:       middle.block:
-; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
-; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
+; CHECK-NEXT:  middle.block:
+; CHECK-NEXT:    EMIT vp<%10> = icmp eq vp<%2>, vp<%1>
+; CHECK-NEXT:    EMIT branch-on-cond vp<%10>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
@@ -255,7 +306,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %idxprom = zext i32 %i.0 to i64
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom
 ; CHECK-NEXT:  LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: %1 = load float, ptr %arrayidx, align 4
-; CHECK-NEXT:  LV: Found an estimated cost of 2 for VF vscale x 4 For instruction: %conv1 = fadd float %1, 1.000000e+00
+; CHECK-NEXT:  LV: Found an estimated cost of 4 for VF vscale x 4 For instruction: %conv1 = fadd float %1, 1.000000e+00
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom
 ; CHECK-NEXT:  LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: store float %conv1, ptr %arrayidx3, align 4
 ; CHECK-NEXT:  LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %cmp = icmp ugt i64 %indvars.iv, 1
@@ -281,7 +332,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: The target has 31 registers of RISCV::GPRRC register class
 ; CHECK-NEXT:  LV: The target has 32 registers of RISCV::VRRC register class
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
-; CHECK-NEXT:  LV: Loop cost is 32
+; CHECK-NEXT:  LV: Loop cost is 34
 ; CHECK-NEXT:  LV: IC is 1
 ; CHECK-NEXT:  LV: VF is vscale x 4
 ; CHECK-NEXT:  LV: Not Interleaving.
@@ -290,7 +341,52 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LEV: Epilogue vectorization is not profitable for this loop
 ; CHECK-NEXT:  VF picked by VPlan cost model: vscale x 4
 ; CHECK-NEXT:  Executing best plan with VF=vscale x 4, UF=1
-; CHECK:       LV: Interleaving disabled by the pass manager
+; CHECK-NEXT:  VPlan 'Final VPlan for VF={vscale x 4},UF>=1' {
+; CHECK-NEXT:  Live-in vp<%0> = VF * UF
+; CHECK-NEXT:  Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT:  vp<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.body.preheader>:
+; CHECK-NEXT:    EMIT vp<%2> = EXPAND SCEV (zext i32 %n to i64)
+; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  vector.ph:
+; CHECK-NEXT:  Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT:  <x1> vector loop: {
+; CHECK-NEXT:    vector.body:
+; CHECK-NEXT:      EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%8>
+; CHECK-NEXT:      vp<%4> = DERIVED-IV ir<%n> + vp<%3> * ir<-1>
+; CHECK-NEXT:      vp<%5> = SCALAR-STEPS vp<%4>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<%5>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
+; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
+; CHECK-NEXT:      vp<%6> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      WIDEN ir<%13> = load vp<%6>
+; CHECK-NEXT:      WIDEN ir<%conv1> = fadd ir<%13>, ir<1.000000e+00>
+; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
+; CHECK-NEXT:      vp<%7> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      WIDEN store vp<%7>, ir<%conv1>
+; CHECK-NEXT:      EMIT vp<%8> = add nuw vp<%3>, vp<%0>
+; CHECK-NEXT:      EMIT branch-on-count vp<%8>, vp<%1>
+; CHECK-NEXT:    No successors
+; CHECK-NEXT:  }
+; CHECK-NEXT:  Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT:  middle.block:
+; CHECK-NEXT:    EMIT vp<%10> = icmp eq vp<%2>, vp<%1>
+; CHECK-NEXT:    EMIT branch-on-cond vp<%10>
+; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
+; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  scalar.ph:
+; CHECK-NEXT:  No successors
+; CHECK-NEXT:  }
+; CHECK-NEXT:  LV: Loop does not require scalar epilogue
+; CHECK-NEXT:  LV: Loop does not require scalar epilogue
+; CHECK-NEXT:  LV: Interleaving disabled by the pass manager
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
 ; CHECK-NEXT:  LV: Vectorizing: innermost loop.
 ;
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll b/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
index 03fbb5e5a4674..97608174b524d 100644
--- a/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
@@ -1130,14 +1130,23 @@ define <vscale x 1 x i64> @umax_nxv1i64_anymask(<vscale x 1 x i64> %x, i64 %y, <
 }
 
 define <vscale x 1 x float> @fadd_nxv1f32_allonesmask(<vscale x 1 x float> %x, float %y, i32 zeroext %evl) {
-; ALL-LABEL: @fadd_nxv1f32_allonesmask(
-; ALL-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x float> poison, float [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[TMP1]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[TMP2]], <vscale x 1 x float> shufflevector (<vscale x 1 x float> insertelement (<vscale x 1 x float> poison, float 4.200000e+01, i64 0), <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
-; ALL-NEXT:    ret <vscale x 1 x float> [[TMP4]]
+; VEC-COMBINE-LABEL: @fadd_nxv1f32_allonesmask(
+; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = fadd float [[Y:%.*]], 4.200000e+01
+; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x float> poison, float [[TMP1]], i64 0
+; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[DOTSPLATINSERT]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x float> [[TMP3]]
+;
+; NO-VEC-COMBINE-LABEL: @fadd_nxv1f32_allonesmask(
+; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x float> poison, float [[Y:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[TMP1]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[TMP2]], <vscale x 1 x float> shufflevector (<vscale x 1 x float> insertelement (<vscale x 1 x float> poison, float 4.200000e+01, i64 0), <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x float> [[TMP4]]
 ;
   %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
   %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
@@ -1164,14 +1173,23 @@ define <vscale x 1 x float> @fadd_nxv1f32_anymask(<vscale x 1 x float> %x, float
 }
 
 define <vscale x 1 x float> @fsub_nxv1f32_allonesmask(<vscale x 1 x float> %x, float %y, i32 zeroext %evl) {
-; ALL-LABEL: @fsub_nxv1f32_allonesmask(
-; ALL-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x float> poison, float [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[TMP1]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fsub.nxv1f32(<vscale x 1 x float> [[TMP2]], <vscale x 1 x float> shufflevector (<vscale x 1 x float> insertelement (<vscale x 1 x float> poison, float 4.200000e+01, i64 0), <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
-; ALL-NEXT:    ret <vscale x 1 x float> [[TMP4]]
+; VEC-COMBINE-LABEL: @fsub_nxv1f32_allonesmask(
+; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = fsub float [[Y:%.*]], 4.200000e+01
+; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x float> poison, float [[TMP1]], i64 0
+; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[DOTSPLATINSERT]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x float> [[TMP3]]
+;
+; NO-VEC-COMBINE-LABEL: @fsub_nxv1f32_allonesmask(
+; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x float> poison, float [[Y:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[TMP1]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fsub.nxv1f32(<vscale x 1 x float> [[TMP2]], <vscale x 1 x float> shufflevector (<vscale x 1 x float> insertelement (<vscale x 1 x float> poison, float 4.200000e+01, i64 0), <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x float> [[TMP4]]
 ;
   %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
   %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
@@ -1198,14 +1216,23 @@ define <vscale x 1 x float> @fsub_nxv1f32_anymask(<vscale x 1 x float> %x, float
 }
 
 define <vscale x 1 x float> @fdiv_nxv1f32_allonesmask(<vscale x 1 x float> %x, float %y, i32 zeroext %evl) {
-; ALL-LABEL: @fdiv_nxv1f32_allonesmask(
-; ALL-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x float> poison, float [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[TMP1]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fdiv.nxv1f32(<vscale x 1 x float> [[TMP2]], <vscale x 1 x float> shufflevector (<vscale x 1 x float> insertelement (<vscale x 1 x float> poison, float 4.200000e+01, i64 0), <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
-; ALL-NEXT:    ret <vscale x 1 x float> [[TMP4]]
+; VEC-COMBINE-LABEL: @fdiv_nxv1f32_allonesmask(
+; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = fdiv float [[Y:%.*]], 4.200000e+01
+; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x float> poison, float [[TMP1]], i64 0
+; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[DOTSPLATINSERT]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x float> [[TMP3]]
+;
+; NO-VEC-COMBINE-LABEL: @fdiv_nxv1f32_allonesmask(
+; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x float> poison, float [[Y:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[TMP1]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fdiv.nxv1f32(<vscale x 1 x float> [[TMP2]], <vscale x 1 x float> shufflevector (<vscale x 1 x float> insertelement (<vscale x 1 x float> poison, float 4.200000e+01, i64 0), <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x float> [[TMP4]]
 ;
   %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
   %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
@@ -1275,14 +1302,23 @@ define <vscale x 1 x float> @frem_nxv1f32_allonesmask(<vscale x 1 x float> %x, f
 }
 
 define <vscale x 1 x float> @fdiv_nxv1f32_allonesmask_knownvl(<vscale x 1 x float> %x, float %y) {
-; ALL-LABEL: @fdiv_nxv1f32_allonesmask_knownvl(
-; ALL-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x float> poison, float [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[TMP1]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fdiv.nxv1f32(<vscale x 1 x float> [[TMP2]], <vscale x 1 x float> shufflevector (<vscale x 1 x float> insertelement (<vscale x 1 x float> poison, float 4.200000e+01, i64 0), <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 4)
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 4)
-; ALL-NEXT:    ret <vscale x 1 x float> [[TMP4]]
+; VEC-COMBINE-LABEL: @fdiv_nxv1f32_allonesmask_knownvl(
+; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = fdiv float [[Y:%.*]], 4.200000e+01
+; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x float> poison, float [[TMP1]], i64 0
+; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[DOTSPLATINSERT]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 4)
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x float> [[TMP3]]
+;
+; NO-VEC-COMBINE-LABEL: @fdiv_nxv1f32_allonesmask_knownvl(
+; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x float> poison, float [[Y:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x float> [[TMP1]], <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x float> @llvm.vp.fdiv.nxv1f32(<vscale x 1 x float> [[TMP2]], <vscale x 1 x float> shufflevector (<vscale x 1 x float> insertelement (<vscale x 1 x float> poison, float 4.200000e+01, i64 0), <vscale x 1 x float> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 4)
+; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x float> @llvm.vp.fadd.nxv1f32(<vscale x 1 x float> [[X:%.*]], <vscale x 1 x float> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 4)
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x float> [[TMP4]]
 ;
   %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
   %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer

From f18fd6e3f9c38560bc26253cae77ac3935149602 Mon Sep 17 00:00:00 2001
From: Daniel Bertalan <dani@danielbertalan.dev>
Date: Mon, 22 Jul 2024 08:13:07 +0200
Subject: [PATCH 372/486] [lld-macho] Use parallel algorithms in favor of
 `ThreadPool` (#99471)

In https://reviews.llvm.org/D115416, it was decided that an explicit
thread pool should be used instead of the simpler fork-join model of the
`parallelFor*` family of functions. Since then, more parallelism has
been added to LLD, but these changes always used the latter strategy,
similarly to other ports of LLD.

This meant that we ended up spawning twice the requested amount of
threads; one set for the `llvm/Support/Parallel.h` executor, and one for
the thread pool.

Since that decision, 3b4d800911 has landed, which allows us to
explicitly enqueue jobs on the executor pool of the parallel algorithms,
which should be enough to achieve sharded output writing and
parallelized input file parsing. Now only the construction of the map
file is left that should be done *concurrently* with different linking
steps, this commit proposes explicitly spawning a dedicated worker
thread for it.
---
 lld/MachO/Writer.cpp | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index e6b80c1d42d9e..0eb809282af28 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -28,8 +28,8 @@
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/Parallel.h"
 #include "llvm/Support/Path.h"
-#include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/TimeProfiler.h"
+#include "llvm/Support/thread.h"
 #include "llvm/Support/xxhash.h"
 
 #include <algorithm>
@@ -66,7 +66,6 @@ class Writer {
 
   template <class LP> void run();
 
-  DefaultThreadPool threadPool;
   std::unique_ptr<FileOutputBuffer> &buffer;
   uint64_t addr = 0;
   uint64_t fileOff = 0;
@@ -1121,14 +1120,12 @@ void Writer::finalizeLinkEditSegment() {
       symtabSection,     indirectSymtabSection,
       dataInCodeSection, functionStartsSection,
   };
-  SmallVector<std::shared_future<void>> threadFutures;
-  threadFutures.reserve(linkEditSections.size());
-  for (LinkEditSection *osec : linkEditSections)
-    if (osec)
-      threadFutures.emplace_back(threadPool.async(
-          [](LinkEditSection *osec) { osec->finalizeContents(); }, osec));
-  for (std::shared_future<void> &future : threadFutures)
-    future.wait();
+
+  parallelForEach(linkEditSections.begin(), linkEditSections.end(),
+                  [](LinkEditSection *osec) {
+                    if (osec)
+                      osec->finalizeContents();
+                  });
 
   // Now that __LINKEDIT is filled out, do a proper calculation of its
   // addresses and offsets.
@@ -1170,6 +1167,8 @@ void Writer::openFile() {
 }
 
 void Writer::writeSections() {
+  TimeTraceScope timeScope("Write output sections");
+
   uint8_t *buf = buffer->getBufferStart();
   std::vector<const OutputSection *> osecs;
   for (const OutputSegment *seg : outputSegments)
@@ -1200,18 +1199,15 @@ void Writer::writeUuid() {
 
   ArrayRef<uint8_t> data{buffer->getBufferStart(), buffer->getBufferEnd()};
   std::vector<ArrayRef<uint8_t>> chunks = split(data, 1024 * 1024);
+
   // Leave one slot for filename
   std::vector<uint64_t> hashes(chunks.size() + 1);
-  SmallVector<std::shared_future<void>> threadFutures;
-  threadFutures.reserve(chunks.size());
-  for (size_t i = 0; i < chunks.size(); ++i)
-    threadFutures.emplace_back(threadPool.async(
-        [&](size_t j) { hashes[j] = xxh3_64bits(chunks[j]); }, i));
-  for (std::shared_future<void> &future : threadFutures)
-    future.wait();
+  parallelFor(0, chunks.size(),
+              [&](size_t i) { hashes[i] = xxh3_64bits(chunks[i]); });
   // Append the output filename so that identical binaries with different names
   // don't get the same UUID.
   hashes[chunks.size()] = xxh3_64bits(sys::path::filename(config->finalOutput));
+
   uint64_t digest = xxh3_64bits({reinterpret_cast<uint8_t *>(hashes.data()),
                                  hashes.size() * sizeof(uint64_t)});
   uuidCommand->writeUuid(digest);
@@ -1330,15 +1326,18 @@ template <class LP> void Writer::run() {
   sortSegmentsAndSections();
   createLoadCommands<LP>();
   finalizeAddresses();
-  threadPool.async([&] {
+
+  llvm::thread mapFileWriter([&] {
     if (LLVM_ENABLE_THREADS && config->timeTraceEnabled)
       timeTraceProfilerInitialize(config->timeTraceGranularity, "writeMapFile");
     writeMapFile();
     if (LLVM_ENABLE_THREADS && config->timeTraceEnabled)
       timeTraceProfilerFinishThread();
   });
+
   finalizeLinkEditSegment();
   writeOutputFile();
+  mapFileWriter.join();
 }
 
 template <class LP> void macho::writeResult() { Writer().run<LP>(); }

From f4d4ce1a3189730ef09ee04207004f500a71070d Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen@sifive.com>
Date: Mon, 22 Jul 2024 14:18:05 +0800
Subject: [PATCH 373/486] [RISCV] Add groupid/bitmask for RISC-V extension
 (#94440)

Base on https://github.com/riscv-non-isa/riscv-c-api-doc/pull/74.

This patch defines the groupid/bitmask in RISCVFeatures.td and generates
the corresponding table in RISCVTargetParserDef.inc.

The groupid/bitmask of extensions provides an abstraction layer between
the compiler and runtime functions.
---
 .../llvm/TargetParser/RISCVTargetParser.h     |   8 ++
 llvm/lib/Target/RISCV/RISCVFeatures.td        | 134 ++++++++++++------
 llvm/lib/TargetParser/RISCVTargetParser.cpp   |  27 ++++
 llvm/test/TableGen/riscv-target-def.td        |  18 ++-
 llvm/utils/TableGen/RISCVTargetDefEmitter.cpp |  37 +++++
 5 files changed, 180 insertions(+), 44 deletions(-)

diff --git a/llvm/include/llvm/TargetParser/RISCVTargetParser.h b/llvm/include/llvm/TargetParser/RISCVTargetParser.h
index 7421dac2744b6..c75778952e0f5 100644
--- a/llvm/include/llvm/TargetParser/RISCVTargetParser.h
+++ b/llvm/include/llvm/TargetParser/RISCVTargetParser.h
@@ -24,6 +24,14 @@ class Triple;
 
 namespace RISCV {
 
+namespace RISCVExtensionBitmaskTable {
+struct RISCVExtensionBitmask {
+  const char *Name;
+  unsigned GroupID;
+  unsigned BitPosition;
+};
+} // namespace RISCVExtensionBitmaskTable
+
 // We use 64 bits as the known part in the scalable vector types.
 static constexpr unsigned RVVBitsPerBlock = 64;
 
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index c9979b2b36fc3..c96d2dd83816b 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -37,6 +37,15 @@ class RISCVExtension<string name, int major, int minor, string desc,
   bit Experimental = false;
 }
 
+// The groupID/bitmask of RISCVExtension is used to retrieve a specific bit value 
+// from __riscv_feature_bits based on the groupID and bitmask.
+// groupID - groupID of extension
+// bitPos  - bit position of extension bitmask
+class RISCVExtensionBitmask<bits<3> groupID, int bitPos> {
+    int GroupID = groupID;
+    int BitPos = bitPos;
+}
+
 // Version of RISCVExtension to be used for Experimental extensions. This
 // sets the Experimental flag and prepends experimental- to the -mattr name.
 class RISCVExperimentalExtension<string name, int major, int minor, string desc,
@@ -52,7 +61,8 @@ class RISCVExperimentalExtension<string name, int major, int minor, string desc,
 
 def FeatureStdExtI
     : RISCVExtension<"i", 2, 1,
-                     "'I' (Base Integer Instruction Set)">;
+                     "'I' (Base Integer Instruction Set)">,
+      RISCVExtensionBitmask<0, 8>;
 
 def FeatureStdExtE
     : RISCVExtension<"e", 2, 0,
@@ -78,7 +88,8 @@ def HasStdExtZicbop : Predicate<"Subtarget->hasStdExtZicbop()">,
 
 def FeatureStdExtZicboz
     : RISCVExtension<"zicboz", 1, 0,
-                     "'Zicboz' (Cache-Block Zero Instructions)">;
+                     "'Zicboz' (Cache-Block Zero Instructions)">,
+      RISCVExtensionBitmask<0, 37>;
 def HasStdExtZicboz : Predicate<"Subtarget->hasStdExtZicboz()">,
                       AssemblerPredicate<(all_of FeatureStdExtZicboz),
                           "'Zicboz' (Cache-Block Zero Instructions)">;
@@ -113,7 +124,8 @@ def FeatureStdExtZicntr
 
 def FeatureStdExtZicond
     : RISCVExtension<"zicond", 1, 0,
-                     "'Zicond' (Integer Conditional Operations)">;
+                     "'Zicond' (Integer Conditional Operations)">,
+      RISCVExtensionBitmask<0, 38>;
 def HasStdExtZicond : Predicate<"Subtarget->hasStdExtZicond()">,
                       AssemblerPredicate<(all_of FeatureStdExtZicond),
                           "'Zicond' (Integer Conditional Operations)">;
@@ -127,14 +139,16 @@ def HasStdExtZifencei : Predicate<"Subtarget->hasStdExtZifencei()">,
 
 def FeatureStdExtZihintpause
     : RISCVExtension<"zihintpause", 2, 0,
-                     "'Zihintpause' (Pause Hint)">;
+                     "'Zihintpause' (Pause Hint)">,
+      RISCVExtensionBitmask<0, 40>;
 def HasStdExtZihintpause : Predicate<"Subtarget->hasStdExtZihintpause()">,
                            AssemblerPredicate<(all_of FeatureStdExtZihintpause),
                                               "'Zihintpause' (Pause Hint)">;
 
 def FeatureStdExtZihintntl
     : RISCVExtension<"zihintntl", 1, 0,
-                     "'Zihintntl' (Non-Temporal Locality Hints)">;
+                     "'Zihintntl' (Non-Temporal Locality Hints)">,
+      RISCVExtensionBitmask<0, 39>;
 def HasStdExtZihintntl : Predicate<"Subtarget->hasStdExtZihintntl()">,
                          AssemblerPredicate<(all_of FeatureStdExtZihintntl),
                              "'Zihintntl' (Non-Temporal Locality Hints)">;
@@ -181,7 +195,8 @@ def HasStdExtZmmul : Predicate<"Subtarget->hasStdExtZmmul()">,
 def FeatureStdExtM
     : RISCVExtension<"m", 2, 0,
                      "'M' (Integer Multiplication and Division)",
-                     [FeatureStdExtZmmul]>;
+                     [FeatureStdExtZmmul]>,
+      RISCVExtensionBitmask<0, 12>;
 def HasStdExtM : Predicate<"Subtarget->hasStdExtM()">,
                  AssemblerPredicate<(all_of FeatureStdExtM),
                      "'M' (Integer Multiplication and Division)">;
@@ -190,14 +205,16 @@ def HasStdExtM : Predicate<"Subtarget->hasStdExtM()">,
 
 def FeatureStdExtA
     : RISCVExtension<"a", 2, 1,
-                     "'A' (Atomic Instructions)">;
+                     "'A' (Atomic Instructions)">,
+      RISCVExtensionBitmask<0, 0>;
 def HasStdExtA : Predicate<"Subtarget->hasStdExtA()">,
                  AssemblerPredicate<(all_of FeatureStdExtA),
                                     "'A' (Atomic Instructions)">;
 
 def FeatureStdExtZtso
     : RISCVExtension<"ztso", 1, 0,
-                     "'Ztso' (Memory Model - Total Store Order)">;
+                     "'Ztso' (Memory Model - Total Store Order)">,
+      RISCVExtensionBitmask<0, 47>;
 def HasStdExtZtso : Predicate<"Subtarget->hasStdExtZtso()">,
                     AssemblerPredicate<(all_of FeatureStdExtZtso),
                         "'Ztso' (Memory Model - Total Store Order)">;
@@ -227,7 +244,8 @@ def HasStdExtZabha : Predicate<"Subtarget->hasStdExtZabha()">,
 
 def FeatureStdExtZacas
     : RISCVExtension<"zacas", 1, 0,
-                     "'Zacas' (Atomic Compare-And-Swap Instructions)">;
+                     "'Zacas' (Atomic Compare-And-Swap Instructions)">,
+      RISCVExtensionBitmask<0, 26>;
 def HasStdExtZacas : Predicate<"Subtarget->hasStdExtZacas()">,
                      AssemblerPredicate<(all_of FeatureStdExtZacas),
                          "'Zacas' (Atomic Compare-And-Swap Instructions)">;
@@ -264,7 +282,8 @@ def HasStdExtZawrs : Predicate<"Subtarget->hasStdExtZawrs()">,
 def FeatureStdExtF
     : RISCVExtension<"f", 2, 2,
                      "'F' (Single-Precision Floating-Point)",
-                     [FeatureStdExtZicsr]>;
+                     [FeatureStdExtZicsr]>,
+      RISCVExtensionBitmask<0, 5>;
 def HasStdExtF : Predicate<"Subtarget->hasStdExtF()">,
                  AssemblerPredicate<(all_of FeatureStdExtF),
                                     "'F' (Single-Precision Floating-Point)">;
@@ -272,7 +291,8 @@ def HasStdExtF : Predicate<"Subtarget->hasStdExtF()">,
 def FeatureStdExtD
     : RISCVExtension<"d", 2, 2,
                      "'D' (Double-Precision Floating-Point)",
-                     [FeatureStdExtF]>;
+                     [FeatureStdExtF]>,
+      RISCVExtensionBitmask<0, 3>;
 def HasStdExtD : Predicate<"Subtarget->hasStdExtD()">,
                  AssemblerPredicate<(all_of FeatureStdExtD),
                                     "'D' (Double-Precision Floating-Point)">;
@@ -280,7 +300,8 @@ def HasStdExtD : Predicate<"Subtarget->hasStdExtD()">,
 def FeatureStdExtZfhmin
     : RISCVExtension<"zfhmin", 1, 0,
                      "'Zfhmin' (Half-Precision Floating-Point Minimal)",
-                     [FeatureStdExtF]>;
+                     [FeatureStdExtF]>,
+      RISCVExtensionBitmask<0, 36>;
 def HasStdExtZfhmin : Predicate<"Subtarget->hasStdExtZfhmin()">,
                       AssemblerPredicate<(all_of FeatureStdExtZfhmin),
                           "'Zfh' (Half-Precision Floating-Point) or "
@@ -289,7 +310,8 @@ def HasStdExtZfhmin : Predicate<"Subtarget->hasStdExtZfhmin()">,
 def FeatureStdExtZfh
     : RISCVExtension<"zfh", 1, 0,
                      "'Zfh' (Half-Precision Floating-Point)",
-                     [FeatureStdExtZfhmin]>;
+                     [FeatureStdExtZfhmin]>,
+      RISCVExtensionBitmask<0, 35>;
 def HasStdExtZfh : Predicate<"Subtarget->hasStdExtZfh()">,
                    AssemblerPredicate<(all_of FeatureStdExtZfh),
                        "'Zfh' (Half-Precision Floating-Point)">;
@@ -313,7 +335,8 @@ def HasHalfFPLoadStoreMove
 def FeatureStdExtZfa
     : RISCVExtension<"zfa", 1, 0,
                      "'Zfa' (Additional Floating-Point)",
-                     [FeatureStdExtF]>;
+                     [FeatureStdExtF]>,
+      RISCVExtensionBitmask<0, 34>;
 def HasStdExtZfa : Predicate<"Subtarget->hasStdExtZfa()">,
                    AssemblerPredicate<(all_of FeatureStdExtZfa),
                                       "'Zfa' (Additional Floating-Point)">;
@@ -356,7 +379,8 @@ def NoStdExtZhinx : Predicate<"!Subtarget->hasStdExtZhinx()">;
 
 def FeatureStdExtC
     : RISCVExtension<"c", 2, 0,
-                     "'C' (Compressed Instructions)">;
+                     "'C' (Compressed Instructions)">,
+      RISCVExtensionBitmask<0, 2>;
 def HasStdExtC : Predicate<"Subtarget->hasStdExtC()">,
                  AssemblerPredicate<(all_of FeatureStdExtC),
                                     "'C' (Compressed Instructions)">;
@@ -445,7 +469,8 @@ def HasStdExtZcmop : Predicate<"Subtarget->hasStdExtZcmop()">,
 
 def FeatureStdExtZba
     : RISCVExtension<"zba", 1, 0,
-                     "'Zba' (Address Generation Instructions)">;
+                     "'Zba' (Address Generation Instructions)">,
+      RISCVExtensionBitmask<0, 27>;
 def HasStdExtZba : Predicate<"Subtarget->hasStdExtZba()">,
                    AssemblerPredicate<(all_of FeatureStdExtZba),
                                       "'Zba' (Address Generation Instructions)">;
@@ -453,7 +478,8 @@ def NotHasStdExtZba : Predicate<"!Subtarget->hasStdExtZba()">;
 
 def FeatureStdExtZbb
     : RISCVExtension<"zbb", 1, 0,
-                     "'Zbb' (Basic Bit-Manipulation)">;
+                     "'Zbb' (Basic Bit-Manipulation)">,
+      RISCVExtensionBitmask<0, 28>;
 def HasStdExtZbb : Predicate<"Subtarget->hasStdExtZbb()">,
                    AssemblerPredicate<(all_of FeatureStdExtZbb),
                                       "'Zbb' (Basic Bit-Manipulation)">;
@@ -462,14 +488,16 @@ def NoStdExtZbb : Predicate<"!Subtarget->hasStdExtZbb()">,
 
 def FeatureStdExtZbc
     : RISCVExtension<"zbc", 1, 0,
-                     "'Zbc' (Carry-Less Multiplication)">;
+                     "'Zbc' (Carry-Less Multiplication)">,
+      RISCVExtensionBitmask<0, 29>;
 def HasStdExtZbc : Predicate<"Subtarget->hasStdExtZbc()">,
                    AssemblerPredicate<(all_of FeatureStdExtZbc),
                                       "'Zbc' (Carry-Less Multiplication)">;
 
 def FeatureStdExtZbs
     : RISCVExtension<"zbs", 1, 0,
-                     "'Zbs' (Single-Bit Instructions)">;
+                     "'Zbs' (Single-Bit Instructions)">,
+      RISCVExtensionBitmask<0, 33>;
 def HasStdExtZbs : Predicate<"Subtarget->hasStdExtZbs()">,
                    AssemblerPredicate<(all_of FeatureStdExtZbs),
                                       "'Zbs' (Single-Bit Instructions)">;
@@ -486,14 +514,16 @@ def HasStdExtB : Predicate<"Subtarget->hasStdExtB()">,
 
 def FeatureStdExtZbkb
     : RISCVExtension<"zbkb", 1, 0,
-                     "'Zbkb' (Bitmanip instructions for Cryptography)">;
+                     "'Zbkb' (Bitmanip instructions for Cryptography)">,
+      RISCVExtensionBitmask<0, 30>;
 def HasStdExtZbkb : Predicate<"Subtarget->hasStdExtZbkb()">,
                     AssemblerPredicate<(all_of FeatureStdExtZbkb),
                         "'Zbkb' (Bitmanip instructions for Cryptography)">;
 
 def FeatureStdExtZbkx
     : RISCVExtension<"zbkx", 1, 0,
-                     "'Zbkx' (Crossbar permutation instructions)">;
+                     "'Zbkx' (Crossbar permutation instructions)">,
+      RISCVExtensionBitmask<0, 32>;
 def HasStdExtZbkx : Predicate<"Subtarget->hasStdExtZbkx()">,
                     AssemblerPredicate<(all_of FeatureStdExtZbkx),
                         "'Zbkx' (Crossbar permutation instructions)">;
@@ -510,7 +540,8 @@ def HasStdExtZbbOrZbkb
 def FeatureStdExtZbkc
     : RISCVExtension<"zbkc", 1, 0,
                      "'Zbkc' (Carry-less multiply instructions for "
-                     "Cryptography)">;
+                     "Cryptography)">,
+      RISCVExtensionBitmask<0, 31>;
 def HasStdExtZbkc
     : Predicate<"Subtarget->hasStdExtZbkc()">,
       AssemblerPredicate<(all_of FeatureStdExtZbkc),
@@ -527,14 +558,16 @@ def HasStdExtZbcOrZbkc
 
 def FeatureStdExtZknd
     : RISCVExtension<"zknd", 1, 0,
-                     "'Zknd' (NIST Suite: AES Decryption)">;
+                     "'Zknd' (NIST Suite: AES Decryption)">,
+      RISCVExtensionBitmask<0, 41>;
 def HasStdExtZknd : Predicate<"Subtarget->hasStdExtZknd()">,
                     AssemblerPredicate<(all_of FeatureStdExtZknd),
                                        "'Zknd' (NIST Suite: AES Decryption)">;
 
 def FeatureStdExtZkne
     : RISCVExtension<"zkne", 1, 0,
-                     "'Zkne' (NIST Suite: AES Encryption)">;
+                     "'Zkne' (NIST Suite: AES Encryption)">,
+      RISCVExtensionBitmask<0, 42>;
 def HasStdExtZkne : Predicate<"Subtarget->hasStdExtZkne()">,
                     AssemblerPredicate<(all_of FeatureStdExtZkne),
                                        "'Zkne' (NIST Suite: AES Encryption)">;
@@ -549,21 +582,24 @@ def HasStdExtZkndOrZkne
 
 def FeatureStdExtZknh
     : RISCVExtension<"zknh", 1, 0,
-                     "'Zknh' (NIST Suite: Hash Function Instructions)">;
+                     "'Zknh' (NIST Suite: Hash Function Instructions)">,
+      RISCVExtensionBitmask<0, 43>;
 def HasStdExtZknh : Predicate<"Subtarget->hasStdExtZknh()">,
                     AssemblerPredicate<(all_of FeatureStdExtZknh),
                         "'Zknh' (NIST Suite: Hash Function Instructions)">;
 
 def FeatureStdExtZksed
     : RISCVExtension<"zksed", 1, 0,
-                     "'Zksed' (ShangMi Suite: SM4 Block Cipher Instructions)">;
+                     "'Zksed' (ShangMi Suite: SM4 Block Cipher Instructions)">,
+      RISCVExtensionBitmask<0, 44>;
 def HasStdExtZksed : Predicate<"Subtarget->hasStdExtZksed()">,
                      AssemblerPredicate<(all_of FeatureStdExtZksed),
                          "'Zksed' (ShangMi Suite: SM4 Block Cipher Instructions)">;
 
 def FeatureStdExtZksh
     : RISCVExtension<"zksh", 1, 0,
-                     "'Zksh' (ShangMi Suite: SM3 Hash Function Instructions)">;
+                     "'Zksh' (ShangMi Suite: SM3 Hash Function Instructions)">,
+      RISCVExtensionBitmask<0, 45>;
 def HasStdExtZksh : Predicate<"Subtarget->hasStdExtZksh()">,
                     AssemblerPredicate<(all_of FeatureStdExtZksh),
                         "'Zksh' (ShangMi Suite: SM3 Hash Function Instructions)">;
@@ -596,7 +632,8 @@ def FeatureStdExtZks
 
 def FeatureStdExtZkt
     : RISCVExtension<"zkt", 1, 0,
-                     "'Zkt' (Data Independent Execution Latency)">;
+                     "'Zkt' (Data Independent Execution Latency)">,
+      RISCVExtensionBitmask<0, 46>;
 
 def FeatureStdExtZk
     : RISCVExtension<"zk", 1, 0,
@@ -626,6 +663,7 @@ def FeatureStdExtZve32x
                      "with maximal 32 EEW)",
                      [FeatureStdExtZicsr, FeatureStdExtZvl32b]>;
 
+
 def FeatureStdExtZve32f
     : RISCVExtension<"zve32f", 1, 0,
                      "'Zve32f' (Vector Extensions for Embedded Processors "
@@ -653,7 +691,8 @@ def FeatureStdExtZve64d
 def FeatureStdExtV
     : RISCVExtension<"v", 1, 0,
                      "'V' (Vector Extension for Application Processors)",
-                     [FeatureStdExtZvl128b, FeatureStdExtZve64d]>;
+                     [FeatureStdExtZvl128b, FeatureStdExtZve64d]>,
+      RISCVExtensionBitmask<0, 21>;
 
 def FeatureStdExtZvfbfmin
     : RISCVExtension<"zvfbfmin", 1, 0, "'Zvbfmin' (Vector BF16 Converts)",
@@ -673,12 +712,14 @@ def HasStdExtZvfbfwma : Predicate<"Subtarget->hasStdExtZvfbfwma()">,
 def FeatureStdExtZvfhmin
     : RISCVExtension<"zvfhmin", 1, 0,
                      "'Zvfhmin' (Vector Half-Precision Floating-Point Minimal)",
-                     [FeatureStdExtZve32f]>;
+                     [FeatureStdExtZve32f]>,
+      RISCVExtensionBitmask<0, 51>;
 
 def FeatureStdExtZvfh
     : RISCVExtension<"zvfh", 1, 0,
                      "'Zvfh' (Vector Half-Precision Floating-Point)",
-                     [FeatureStdExtZvfhmin, FeatureStdExtZfhmin]>;
+                     [FeatureStdExtZvfhmin, FeatureStdExtZfhmin]>,
+      RISCVExtensionBitmask<0, 50>;
 
 def HasStdExtZfhOrZvfh
     : Predicate<"Subtarget->hasStdExtZfh() || Subtarget->hasStdExtZvfh()">,
@@ -690,7 +731,8 @@ def HasStdExtZfhOrZvfh
 
 def FeatureStdExtZvkb
     : RISCVExtension<"zvkb", 1, 0,
-                     "'Zvkb' (Vector Bit-manipulation used in Cryptography)">;
+                     "'Zvkb' (Vector Bit-manipulation used in Cryptography)">,
+      RISCVExtensionBitmask<0, 52>;
 def HasStdExtZvkb : Predicate<"Subtarget->hasStdExtZvkb()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvkb),
                         "'Zvkb' (Vector Bit-manipulation used in Cryptography)">;
@@ -698,35 +740,40 @@ def HasStdExtZvkb : Predicate<"Subtarget->hasStdExtZvkb()">,
 def FeatureStdExtZvbb
     : RISCVExtension<"zvbb", 1, 0,
                      "'Zvbb' (Vector basic bit-manipulation instructions)",
-                     [FeatureStdExtZvkb]>;
+                     [FeatureStdExtZvkb]>,
+      RISCVExtensionBitmask<0, 48>;
 def HasStdExtZvbb : Predicate<"Subtarget->hasStdExtZvbb()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvbb),
                         "'Zvbb' (Vector basic bit-manipulation instructions)">;
 
 def FeatureStdExtZvbc
     : RISCVExtension<"zvbc", 1, 0,
-                     "'Zvbc' (Vector Carryless Multiplication)">;
+                     "'Zvbc' (Vector Carryless Multiplication)">,
+      RISCVExtensionBitmask<0, 49>;
 def HasStdExtZvbc : Predicate<"Subtarget->hasStdExtZvbc()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvbc),
                         "'Zvbc' (Vector Carryless Multiplication)">;
 
 def FeatureStdExtZvkg
     : RISCVExtension<"zvkg", 1, 0,
-                     "'Zvkg' (Vector GCM instructions for Cryptography)">;
+                     "'Zvkg' (Vector GCM instructions for Cryptography)">,
+      RISCVExtensionBitmask<0, 53>;
 def HasStdExtZvkg : Predicate<"Subtarget->hasStdExtZvkg()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvkg),
                         "'Zvkg' (Vector GCM instructions for Cryptography)">;
 
 def FeatureStdExtZvkned
     : RISCVExtension<"zvkned", 1, 0,
-                     "'Zvkned' (Vector AES Encryption & Decryption (Single Round))">;
+                     "'Zvkned' (Vector AES Encryption & Decryption (Single Round))">,
+      RISCVExtensionBitmask<0, 54>;
 def HasStdExtZvkned : Predicate<"Subtarget->hasStdExtZvkned()">,
                       AssemblerPredicate<(all_of FeatureStdExtZvkned),
                           "'Zvkned' (Vector AES Encryption & Decryption (Single Round))">;
 
 def FeatureStdExtZvknha
     : RISCVExtension<"zvknha", 1, 0,
-                     "'Zvknha' (Vector SHA-2 (SHA-256 only))">;
+                     "'Zvknha' (Vector SHA-2 (SHA-256 only))">,
+      RISCVExtensionBitmask<0, 55>;
 def HasStdExtZvknha : Predicate<"Subtarget->hasStdExtZvknha()">,
                       AssemblerPredicate<(all_of FeatureStdExtZvknha),
                           "'Zvknha' (Vector SHA-2 (SHA-256 only))">;
@@ -734,7 +781,8 @@ def HasStdExtZvknha : Predicate<"Subtarget->hasStdExtZvknha()">,
 def FeatureStdExtZvknhb
     : RISCVExtension<"zvknhb", 1, 0,
                      "'Zvknhb' (Vector SHA-2 (SHA-256 and SHA-512))",
-                     [FeatureStdExtZve64x]>;
+                     [FeatureStdExtZve64x]>,
+      RISCVExtensionBitmask<0, 56>;
 def HasStdExtZvknhb : Predicate<"Subtarget->hasStdExtZvknhb()">,
                       AssemblerPredicate<(all_of FeatureStdExtZvknhb),
                           "'Zvknhb' (Vector SHA-2 (SHA-256 and SHA-512))">;
@@ -745,21 +793,24 @@ def HasStdExtZvknhaOrZvknhb : Predicate<"Subtarget->hasStdExtZvknha() || Subtarg
 
 def FeatureStdExtZvksed
     : RISCVExtension<"zvksed", 1, 0,
-                     "'Zvksed' (SM4 Block Cipher Instructions)">;
+                     "'Zvksed' (SM4 Block Cipher Instructions)">,
+      RISCVExtensionBitmask<0, 57>;
 def HasStdExtZvksed : Predicate<"Subtarget->hasStdExtZvksed()">,
                       AssemblerPredicate<(all_of FeatureStdExtZvksed),
                           "'Zvksed' (SM4 Block Cipher Instructions)">;
 
 def FeatureStdExtZvksh
     : RISCVExtension<"zvksh", 1, 0,
-                     "'Zvksh' (SM3 Hash Function Instructions)">;
+                     "'Zvksh' (SM3 Hash Function Instructions)">,
+      RISCVExtensionBitmask<0, 58>;
 def HasStdExtZvksh : Predicate<"Subtarget->hasStdExtZvksh()">,
                      AssemblerPredicate<(all_of FeatureStdExtZvksh),
                          "'Zvksh' (SM3 Hash Function Instructions)">;
 
 def FeatureStdExtZvkt
     : RISCVExtension<"zvkt", 1, 0,
-                     "'Zvkt' (Vector Data-Independent Execution Latency)">;
+                     "'Zvkt' (Vector Data-Independent Execution Latency)">,
+      RISCVExtensionBitmask<0, 59>;
 
 // Zvk short-hand extensions
 
@@ -796,7 +847,6 @@ def FeatureStdExtZvksg
     : RISCVExtension<"zvksg", 1, 0,
                      "'Zvksg' (shorthand for 'Zvks' and 'Zvkg')",
                      [FeatureStdExtZvks, FeatureStdExtZvkg]>;
-
 // Vector instruction predicates
 
 def HasVInstructions    : Predicate<"Subtarget->hasVInstructions()">,
diff --git a/llvm/lib/TargetParser/RISCVTargetParser.cpp b/llvm/lib/TargetParser/RISCVTargetParser.cpp
index db1b5f689d7da..e2d12293e1cd2 100644
--- a/llvm/lib/TargetParser/RISCVTargetParser.cpp
+++ b/llvm/lib/TargetParser/RISCVTargetParser.cpp
@@ -128,6 +128,33 @@ void getFeaturesForCPU(StringRef CPU,
     else
       EnabledFeatures.push_back(F.substr(1));
 }
+
+namespace RISCVExtensionBitmaskTable {
+#define GET_RISCVExtensionBitmaskTable_IMPL
+#include "llvm/TargetParser/RISCVTargetParserDef.inc"
+
+} // namespace RISCVExtensionBitmaskTable
+
+namespace {
+struct LessExtName {
+  bool operator()(const RISCVExtensionBitmaskTable::RISCVExtensionBitmask &LHS,
+                  StringRef RHS) {
+    return StringRef(LHS.Name) < RHS;
+  }
+};
+} // namespace
+
+static RISCVExtensionBitmaskTable::RISCVExtensionBitmask
+getExtensionBitmask(StringRef ExtName) {
+  ArrayRef<RISCVExtensionBitmaskTable::RISCVExtensionBitmask> ExtBitmasks =
+      RISCVExtensionBitmaskTable::ExtensionBitmask;
+  auto *I = llvm::lower_bound(ExtBitmasks, ExtName, LessExtName());
+
+  if (I != ExtBitmasks.end() && ExtName.equals_insensitive(I->Name))
+    return *I;
+
+  return RISCVExtensionBitmaskTable::RISCVExtensionBitmask();
+}
 } // namespace RISCV
 
 namespace RISCVVType {
diff --git a/llvm/test/TableGen/riscv-target-def.td b/llvm/test/TableGen/riscv-target-def.td
index 7137cf96fd3d4..c071cfd731cb5 100644
--- a/llvm/test/TableGen/riscv-target-def.td
+++ b/llvm/test/TableGen/riscv-target-def.td
@@ -12,6 +12,11 @@ class RISCVExtension<string name, int major, int minor, string desc,
   bit Experimental = false;
 }
 
+class RISCVExtensionBitmask<bits<3> groupID, int bitPos> {
+  int GroupID = groupID;
+  int BitPos = bitPos;
+}
+
 class RISCVExperimentalExtension<string name, int major, int minor, string desc,
                                  list<RISCVExtension> implies = [],
                                  string fieldname = !subst("Feature", "Has", NAME),
@@ -23,7 +28,8 @@ class RISCVExperimentalExtension<string name, int major, int minor, string desc,
 
 def FeatureStdExtI
     : RISCVExtension<"i", 2, 1,
-                     "'I' (Base Integer Instruction Set)">;
+                     "'I' (Base Integer Instruction Set)">,
+      RISCVExtensionBitmask<0, 8>;
 
 def FeatureStdExtZicsr
     : RISCVExtension<"zicsr", 2, 0,
@@ -36,7 +42,8 @@ def FeatureStdExtZifencei
 def FeatureStdExtF
     : RISCVExtension<"f", 2, 2,
                      "'F' (Single-Precision Floating-Point)",
-                     [FeatureStdExtZicsr]>;
+                     [FeatureStdExtZicsr]>,
+      RISCVExtensionBitmask<0, 5>;
 
 def FeatureStdExtZidummy
     : RISCVExperimentalExtension<"zidummy", 0, 1,
@@ -171,3 +178,10 @@ def ROCKET : RISCVTuneProcessorModel<"rocket",
 // CHECK-NEXT: TUNE_PROC(ROCKET, "rocket")
 
 // CHECK: #undef TUNE_PROC
+
+// CHECK: #ifdef GET_RISCVExtensionBitmaskTable_IMPL
+// CHECK-NEXT: static const RISCVExtensionBitmask ExtensionBitmask[]={
+// CHECK-NEXT:     {"f", 0, 5ULL},
+// CHECK-NEXT:     {"i", 0, 8ULL},
+// CHECK-NEXT: };
+// CHECK-NEXT: #endif
diff --git a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
index 04e9e0fa48db0..910488a14b985 100644
--- a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/Support/RISCVISAUtils.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
@@ -209,10 +210,46 @@ static void emitRISCVProcs(RecordKeeper &RK, raw_ostream &OS) {
   OS << "\n#undef TUNE_PROC\n";
 }
 
+static void emitRISCVExtensionBitmask(RecordKeeper &RK, raw_ostream &OS) {
+
+  std::vector<Record *> Extensions =
+      RK.getAllDerivedDefinitionsIfDefined("RISCVExtensionBitmask");
+  llvm::sort(Extensions, [](const Record *Rec1, const Record *Rec2) {
+    return getExtensionName(Rec1) < getExtensionName(Rec2);
+  });
+
+#ifndef NDEBUG
+  llvm::DenseSet<std::pair<uint64_t, uint64_t>> Seen;
+#endif
+
+  OS << "#ifdef GET_RISCVExtensionBitmaskTable_IMPL\n";
+  OS << "static const RISCVExtensionBitmask ExtensionBitmask[]={\n";
+  for (const Record *Rec : Extensions) {
+    unsigned GroupIDVal = Rec->getValueAsInt("GroupID");
+    unsigned BitPosVal = Rec->getValueAsInt("BitPos");
+
+    StringRef ExtName = Rec->getValueAsString("Name");
+    ExtName.consume_front("experimental-");
+
+#ifndef NDEBUG
+    assert(Seen.insert(std::make_pair(GroupIDVal, BitPosVal)).second &&
+           "duplicated bitmask");
+#endif
+
+    OS << "    {"
+       << "\"" << ExtName << "\""
+       << ", " << GroupIDVal << ", " << BitPosVal << "ULL"
+       << "},\n";
+  }
+  OS << "};\n";
+  OS << "#endif\n";
+}
+
 static void EmitRISCVTargetDef(RecordKeeper &RK, raw_ostream &OS) {
   emitRISCVExtensions(RK, OS);
   emitRISCVProfiles(RK, OS);
   emitRISCVProcs(RK, OS);
+  emitRISCVExtensionBitmask(RK, OS);
 }
 
 static TableGen::Emitter::Opt X("gen-riscv-target-def", EmitRISCVTargetDef,

From 2873edd2867c151871f7ec3865819ff0f874c95a Mon Sep 17 00:00:00 2001
From: Brandon Wu <brandon.wu@sifive.com>
Date: Mon, 22 Jul 2024 14:31:28 +0800
Subject: [PATCH 374/486] [lld][RISCV] Add break to nested switch in
 `mergeAtomic` (#99762)

This prevent the warnings from compiler.
---
 lld/ELF/Arch/RISCV.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lld/ELF/Arch/RISCV.cpp b/lld/ELF/Arch/RISCV.cpp
index 6af89ce3517b7..56759c28dcf41 100644
--- a/lld/ELF/Arch/RISCV.cpp
+++ b/lld/ELF/Arch/RISCV.cpp
@@ -1131,6 +1131,7 @@ static void mergeAtomic(DenseMap<unsigned, unsigned>::iterator it,
     case RISCVAttrs::RISCVAtomicAbiTag::A6C:
       return;
     };
+    break;
 
   case RISCVAtomicAbiTag::A6S:
     switch (newTag) {
@@ -1144,6 +1145,7 @@ static void mergeAtomic(DenseMap<unsigned, unsigned>::iterator it,
     case RISCVAttrs::RISCVAtomicAbiTag::A6S:
       return;
     };
+    break;
 
   case RISCVAtomicAbiTag::A7:
     switch (newTag) {
@@ -1157,6 +1159,7 @@ static void mergeAtomic(DenseMap<unsigned, unsigned>::iterator it,
     case RISCVAttrs::RISCVAtomicAbiTag::A7:
       return;
     };
+    break;
   };
 
   // If we get here, then we have an invalid tag, so report it.

From 8dafbb5fdd4099df72d24af2ceb7ecade8216820 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Mon, 22 Jul 2024 14:33:45 +0800
Subject: [PATCH 375/486] [RISCV] Remove unused function (NFC)

/llvm-project/llvm/lib/TargetParser/RISCVTargetParser.cpp:148:1:
error: unused function 'getExtensionBitmask' [-Werror,-Wunused-function]
getExtensionBitmask(StringRef ExtName) {
^
1 error generated.
---
 llvm/lib/TargetParser/RISCVTargetParser.cpp | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVTargetParser.cpp b/llvm/lib/TargetParser/RISCVTargetParser.cpp
index e2d12293e1cd2..49a35bfcf4b9b 100644
--- a/llvm/lib/TargetParser/RISCVTargetParser.cpp
+++ b/llvm/lib/TargetParser/RISCVTargetParser.cpp
@@ -144,17 +144,6 @@ struct LessExtName {
 };
 } // namespace
 
-static RISCVExtensionBitmaskTable::RISCVExtensionBitmask
-getExtensionBitmask(StringRef ExtName) {
-  ArrayRef<RISCVExtensionBitmaskTable::RISCVExtensionBitmask> ExtBitmasks =
-      RISCVExtensionBitmaskTable::ExtensionBitmask;
-  auto *I = llvm::lower_bound(ExtBitmasks, ExtName, LessExtName());
-
-  if (I != ExtBitmasks.end() && ExtName.equals_insensitive(I->Name))
-    return *I;
-
-  return RISCVExtensionBitmaskTable::RISCVExtensionBitmask();
-}
 } // namespace RISCV
 
 namespace RISCVVType {

From 6b9ac2a4951453fa61fbda285a23be1b32bbff49 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp@bytedance.com>
Date: Mon, 22 Jul 2024 14:41:07 +0800
Subject: [PATCH 376/486] [BranchFolding] Add a hook to override tail merge
 size (#99025)

A new hook `TargetInstrInfo::getTailMergeSize()` is added so that
targets can override it.

This removes an existing TODO.
---
 llvm/include/llvm/CodeGen/TargetInstrInfo.h | 6 ++++++
 llvm/lib/CodeGen/BranchFolding.cpp          | 9 ++++++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 5c7f6ddc94840..649711d8faf65 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -2226,6 +2226,12 @@ class TargetInstrInfo : public MCInstrInfo {
     return OptLevel >= CodeGenOptLevel::Aggressive ? 4 : 2;
   }
 
+  /// Returns the target-specific default value for tail merging.
+  /// This value will be used if the tail-merge-size argument is not provided.
+  virtual unsigned getTailMergeSize(const MachineFunction &MF) const {
+    return 3;
+  }
+
   /// Returns the callee operand from the given \p MI.
   virtual const MachineOperand &getCalleeOperand(const MachineInstr &MI) const {
     return MI.getOperand(0);
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index c0fc7a2b35ea3..92a03eb52e35d 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -80,7 +80,6 @@ TailMergeThreshold("tail-merge-threshold",
           cl::init(150), cl::Hidden);
 
 // Heuristic for tail merging (and, inversely, tail duplication).
-// TODO: This should be replaced with a target query.
 static cl::opt<unsigned>
 TailMergeSize("tail-merge-size",
               cl::desc("Min number of instructions to consider tail merging"),
@@ -145,8 +144,6 @@ BranchFolder::BranchFolder(bool DefaultEnableTailMerge, bool CommonHoist,
                            ProfileSummaryInfo *PSI, unsigned MinTailLength)
     : EnableHoistCommonCode(CommonHoist), MinCommonTailLength(MinTailLength),
       MBBFreqInfo(FreqInfo), MBPI(ProbInfo), PSI(PSI) {
-  if (MinCommonTailLength == 0)
-    MinCommonTailLength = TailMergeSize;
   switch (FlagEnableTailMerge) {
   case cl::BOU_UNSET:
     EnableTailMerge = DefaultEnableTailMerge;
@@ -195,6 +192,12 @@ bool BranchFolder::OptimizeFunction(MachineFunction &MF,
   MLI = mli;
   this->MRI = &MRI;
 
+  if (MinCommonTailLength == 0) {
+    MinCommonTailLength = TailMergeSize.getNumOccurrences() > 0
+                              ? TailMergeSize
+                              : TII->getTailMergeSize(MF);
+  }
+
   UpdateLiveIns = MRI.tracksLiveness() && TRI->trackLivenessAfterRegAlloc(MF);
   if (!UpdateLiveIns)
     MRI.invalidateLiveness();

From 10d720b5b4383c9900caaed7edf38ae31e8b9ac4 Mon Sep 17 00:00:00 2001
From: Antonio Frighetto <me@antoniofrighetto.com>
Date: Mon, 22 Jul 2024 09:03:54 +0200
Subject: [PATCH 377/486] [DeadStoreElimination] Add test for recent worklist
 revision (NFC)

As d5c89cc proved not to be NFC, prior to this change, duplicate
`MemoryAccess` entries were being added to the worklist in
`isWriteAtEndOfFunction`, prematurely reaching the exploration
limit. When `MemorySSAScanLimit` cutoff is set to 4, the store
was previously not eliminated. Introduce a regression test for
additional validation. The test is a simplified variant of function
`ntlmssp_create_session_key`, coming from @dtcxzyw/llvm-opt-benchmark,
bench/wireshark/original/packet-ntlmssp.c.ll.
---
 .../memoryssa-scan-limit.ll                   | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/llvm/test/Transforms/DeadStoreElimination/memoryssa-scan-limit.ll b/llvm/test/Transforms/DeadStoreElimination/memoryssa-scan-limit.ll
index bd225def3addc..231aba5cc2f0c 100644
--- a/llvm/test/Transforms/DeadStoreElimination/memoryssa-scan-limit.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/memoryssa-scan-limit.ll
@@ -4,6 +4,7 @@
 ; RUN: opt < %s -passes=dse -dse-memoryssa-scanlimit=0 -S | FileCheck --check-prefix=LIMIT-0 %s
 ; RUN: opt < %s -passes=dse -dse-memoryssa-scanlimit=2 -S | FileCheck --check-prefix=LIMIT-2 %s
 ; RUN: opt < %s -passes=dse -dse-memoryssa-scanlimit=3 -S | FileCheck --check-prefix=LIMIT-3 %s
+; RUN: opt < %s -passes=dse -dse-memoryssa-scanlimit=4 -S | FileCheck --check-prefix=LIMIT-4 %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
@@ -71,3 +72,47 @@ bb3:
   store i32 0, ptr %P
   ret void
 }
+
+define void @duplicate_worklist_endoffunction(ptr %ptr.0, ptr %ptr.1) {
+; LIMIT-4-LABEL: @duplicate_worklist_endoffunction(
+; LIMIT-4-NEXT:  entry:
+; LIMIT-4-NEXT:    [[STACK_0:%.*]] = alloca [768 x i8], align 16
+; LIMIT-4-NEXT:    [[VAL_0:%.*]] = load i16, ptr [[PTR_1:%.*]], align 8
+; LIMIT-4-NEXT:    [[COND:%.*]] = icmp ugt i16 [[VAL_0]], 24
+; LIMIT-4-NEXT:    br i1 [[COND]], label [[BB_1:%.*]], label [[EXIT:%.*]]
+; LIMIT-4:       bb.1:
+; LIMIT-4-NEXT:    br label [[LOOP:%.*]]
+; LIMIT-4:       loop:
+; LIMIT-4-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[BB_1]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; LIMIT-4-NEXT:    [[PTR_3:%.*]] = getelementptr i8, ptr [[STACK_0]], i64 [[IV]]
+; LIMIT-4-NEXT:    store ptr [[PTR_0:%.*]], ptr [[PTR_3]], align 2
+; LIMIT-4-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; LIMIT-4-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 10
+; LIMIT-4-NEXT:    br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]]
+; LIMIT-4:       exit:
+; LIMIT-4-NEXT:    ret void
+;
+entry:
+  %stack.0 = alloca [768 x i8], align 16
+  %stack.1 = alloca [20 x i8], align 8
+  %val.0 = load i16, ptr %ptr.1, align 8
+  %cond = icmp ugt i16 %val.0, 24
+  br i1 %cond, label %bb.1, label %exit
+
+bb.1:                                             ; preds = %entry
+  %ptr.2 = getelementptr inbounds i8, ptr %ptr.1, i64 8
+  %val.1 = load i64, ptr %ptr.2, align 8
+  store i64 %val.1, ptr %stack.1, align 8
+  br label %loop
+
+loop:                                             ; preds = %loop, %bb.1
+  %iv = phi i64 [ 0, %bb.1 ], [ %iv.next, %loop ]
+  %ptr.3 = getelementptr i8, ptr %stack.0, i64 %iv
+  store ptr %ptr.0, ptr %ptr.3, align 2
+  %iv.next = add nuw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 10
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                             ; preds = %loop, %entry
+  ret void
+}

From 36d384b4ddb54d40866614df5128be60f36d92a2 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Mon, 22 Jul 2024 09:09:02 +0200
Subject: [PATCH 378/486] [mlir][Transforms][NFC] Dialect conversion: Simplify
 `EraseBlockRewrite` constructor (#99805)

---
 mlir/lib/Transforms/Utils/DialectConversion.cpp | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 0b552a7e1ca3b..a045868b66031 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -293,10 +293,9 @@ class CreateBlockRewrite : public BlockRewrite {
 /// original location.
 class EraseBlockRewrite : public BlockRewrite {
 public:
-  EraseBlockRewrite(ConversionPatternRewriterImpl &rewriterImpl, Block *block,
-                    Region *region, Block *insertBeforeBlock)
-      : BlockRewrite(Kind::EraseBlock, rewriterImpl, block), region(region),
-        insertBeforeBlock(insertBeforeBlock) {}
+  EraseBlockRewrite(ConversionPatternRewriterImpl &rewriterImpl, Block *block)
+      : BlockRewrite(Kind::EraseBlock, rewriterImpl, block),
+        region(block->getParent()), insertBeforeBlock(block->getNextNode()) {}
 
   static bool classof(const IRRewrite *rewrite) {
     return rewrite->getKind() == Kind::EraseBlock;
@@ -1440,9 +1439,7 @@ void ConversionPatternRewriterImpl::notifyOpReplaced(Operation *op,
 }
 
 void ConversionPatternRewriterImpl::notifyBlockIsBeingErased(Block *block) {
-  Region *region = block->getParent();
-  Block *origNextBlock = block->getNextNode();
-  appendRewrite<EraseBlockRewrite>(block, region, origNextBlock);
+  appendRewrite<EraseBlockRewrite>(block);
 }
 
 void ConversionPatternRewriterImpl::notifyBlockInserted(

From 4c73b1a986bb4f3af6c5d5e9e705934dbcc36262 Mon Sep 17 00:00:00 2001
From: hev <wangrui@loongson.cn>
Date: Mon, 22 Jul 2024 15:10:21 +0800
Subject: [PATCH 379/486] [LoongArch] Recommit "Remove spurious mask operations
 from andn->icmp on 16 and 8 bit values" (#99798)

recommit of #99272
---
 .../LoongArch/LoongArchISelLowering.cpp       | 166 ++++++++++++++++++
 llvm/test/CodeGen/LoongArch/andn-icmp.ll      |  57 ++----
 2 files changed, 182 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index ba6be85c7f2e8..496f126b7173d 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -335,6 +335,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
   setTargetDAGCombine(ISD::AND);
   setTargetDAGCombine(ISD::OR);
   setTargetDAGCombine(ISD::SRL);
+  setTargetDAGCombine(ISD::SETCC);
 
   // Set DAG combine for 'LSX' feature.
 
@@ -2528,6 +2529,169 @@ static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+static bool checkValueWidth(SDValue V, ISD::LoadExtType &ExtType) {
+  ExtType = ISD::NON_EXTLOAD;
+
+  switch (V.getNode()->getOpcode()) {
+  case ISD::LOAD: {
+    LoadSDNode *LoadNode = cast<LoadSDNode>(V.getNode());
+    if ((LoadNode->getMemoryVT() == MVT::i8) ||
+        (LoadNode->getMemoryVT() == MVT::i16)) {
+      ExtType = LoadNode->getExtensionType();
+      return true;
+    }
+    return false;
+  }
+  case ISD::AssertSext: {
+    VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
+    if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
+      ExtType = ISD::SEXTLOAD;
+      return true;
+    }
+    return false;
+  }
+  case ISD::AssertZext: {
+    VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
+    if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
+      ExtType = ISD::ZEXTLOAD;
+      return true;
+    }
+    return false;
+  }
+  default:
+    return false;
+  }
+
+  return false;
+}
+
+// Eliminate redundant truncation and zero-extension nodes.
+// * Case 1:
+//  +------------+ +------------+ +------------+
+//  |   Input1   | |   Input2   | |     CC     |
+//  +------------+ +------------+ +------------+
+//         |              |              |
+//         V              V              +----+
+//  +------------+ +------------+             |
+//  |  TRUNCATE  | |  TRUNCATE  |             |
+//  +------------+ +------------+             |
+//         |              |                   |
+//         V              V                   |
+//  +------------+ +------------+             |
+//  |  ZERO_EXT  | |  ZERO_EXT  |             |
+//  +------------+ +------------+             |
+//         |              |                   |
+//         |              +-------------+     |
+//         V              V             |     |
+//        +----------------+            |     |
+//        |      AND       |            |     |
+//        +----------------+            |     |
+//                |                     |     |
+//                +---------------+     |     |
+//                                |     |     |
+//                                V     V     V
+//                               +-------------+
+//                               |     CMP     |
+//                               +-------------+
+// * Case 2:
+//  +------------+ +------------+ +-------------+ +------------+ +------------+
+//  |   Input1   | |   Input2   | | Constant -1 | | Constant 0 | |     CC     |
+//  +------------+ +------------+ +-------------+ +------------+ +------------+
+//         |              |             |               |               |
+//         V              |             |               |               |
+//  +------------+        |             |               |               |
+//  |     XOR    |<---------------------+               |               |
+//  +------------+        |                             |               |
+//         |              |                             |               |
+//         V              V             +---------------+               |
+//  +------------+ +------------+       |                               |
+//  |  TRUNCATE  | |  TRUNCATE  |       |     +-------------------------+
+//  +------------+ +------------+       |     |
+//         |              |             |     |
+//         V              V             |     |
+//  +------------+ +------------+       |     |
+//  |  ZERO_EXT  | |  ZERO_EXT  |       |     |
+//  +------------+ +------------+       |     |
+//         |              |             |     |
+//         V              V             |     |
+//        +----------------+            |     |
+//        |      AND       |            |     |
+//        +----------------+            |     |
+//                |                     |     |
+//                +---------------+     |     |
+//                                |     |     |
+//                                V     V     V
+//                               +-------------+
+//                               |     CMP     |
+//                               +-------------+
+static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
+                                   TargetLowering::DAGCombinerInfo &DCI,
+                                   const LoongArchSubtarget &Subtarget) {
+  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
+
+  SDNode *AndNode = N->getOperand(0).getNode();
+  if (AndNode->getOpcode() != ISD::AND)
+    return SDValue();
+
+  SDValue AndInputValue2 = AndNode->getOperand(1);
+  if (AndInputValue2.getOpcode() != ISD::ZERO_EXTEND)
+    return SDValue();
+
+  SDValue CmpInputValue = N->getOperand(1);
+  SDValue AndInputValue1 = AndNode->getOperand(0);
+  if (AndInputValue1.getOpcode() == ISD::XOR) {
+    if (CC != ISD::SETEQ && CC != ISD::SETNE)
+      return SDValue();
+    ConstantSDNode *CN = dyn_cast<ConstantSDNode>(AndInputValue1.getOperand(1));
+    if (!CN || CN->getSExtValue() != -1)
+      return SDValue();
+    CN = dyn_cast<ConstantSDNode>(CmpInputValue);
+    if (!CN || CN->getSExtValue() != 0)
+      return SDValue();
+    AndInputValue1 = AndInputValue1.getOperand(0);
+    if (AndInputValue1.getOpcode() != ISD::ZERO_EXTEND)
+      return SDValue();
+  } else if (AndInputValue1.getOpcode() == ISD::ZERO_EXTEND) {
+    if (AndInputValue2 != CmpInputValue)
+      return SDValue();
+  } else {
+    return SDValue();
+  }
+
+  SDValue TruncValue1 = AndInputValue1.getNode()->getOperand(0);
+  if (TruncValue1.getOpcode() != ISD::TRUNCATE)
+    return SDValue();
+
+  SDValue TruncValue2 = AndInputValue2.getNode()->getOperand(0);
+  if (TruncValue2.getOpcode() != ISD::TRUNCATE)
+    return SDValue();
+
+  SDValue TruncInputValue1 = TruncValue1.getNode()->getOperand(0);
+  SDValue TruncInputValue2 = TruncValue2.getNode()->getOperand(0);
+  ISD::LoadExtType ExtType1;
+  ISD::LoadExtType ExtType2;
+
+  if (!checkValueWidth(TruncInputValue1, ExtType1) ||
+      !checkValueWidth(TruncInputValue2, ExtType2))
+    return SDValue();
+
+  if (TruncInputValue1->getValueType(0) != TruncInputValue2->getValueType(0) ||
+      AndNode->getValueType(0) != TruncInputValue1->getValueType(0))
+    return SDValue();
+
+  if ((ExtType2 != ISD::ZEXTLOAD) &&
+      ((ExtType2 != ISD::SEXTLOAD) && (ExtType1 != ISD::SEXTLOAD)))
+    return SDValue();
+
+  // These truncation and zero-extension nodes are not necessary, remove them.
+  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N), AndNode->getValueType(0),
+                               TruncInputValue1, TruncInputValue2);
+  SDValue NewSetCC =
+      DAG.getSetCC(SDLoc(N), N->getValueType(0), NewAnd, TruncInputValue2, CC);
+  DAG.ReplaceAllUsesWith(N, NewSetCC.getNode());
+  return SDValue(N, 0);
+}
+
 // Combine (loongarch_bitrev_w (loongarch_revb_2w X)) to loongarch_bitrev_4b.
 static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG,
                                       TargetLowering::DAGCombinerInfo &DCI,
@@ -3155,6 +3319,8 @@ SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N,
     return performANDCombine(N, DAG, DCI, Subtarget);
   case ISD::OR:
     return performORCombine(N, DAG, DCI, Subtarget);
+  case ISD::SETCC:
+    return performSETCCCombine(N, DAG, DCI, Subtarget);
   case ISD::SRL:
     return performSRLCombine(N, DAG, DCI, Subtarget);
   case LoongArchISD::BITREV_W:
diff --git a/llvm/test/CodeGen/LoongArch/andn-icmp.ll b/llvm/test/CodeGen/LoongArch/andn-icmp.ll
index 447f3ac5c34fd..67d62eca914e9 100644
--- a/llvm/test/CodeGen/LoongArch/andn-icmp.ll
+++ b/llvm/test/CodeGen/LoongArch/andn-icmp.ll
@@ -6,14 +6,12 @@ define i1 @andn_icmp_eq_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_eq_i8:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    andn $a0, $a1, $a0
-; LA32-NEXT:    andi $a0, $a0, 255
 ; LA32-NEXT:    sltui $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_eq_i8:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    andn $a0, $a1, $a0
-; LA64-NEXT:    andi $a0, $a0, 255
 ; LA64-NEXT:    sltui $a0, $a0, 1
 ; LA64-NEXT:    ret
   %and = and i8 %a, %b
@@ -25,14 +23,12 @@ define i1 @andn_icmp_eq_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_eq_i16:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    andn $a0, $a1, $a0
-; LA32-NEXT:    bstrpick.w $a0, $a0, 15, 0
 ; LA32-NEXT:    sltui $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_eq_i16:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    andn $a0, $a1, $a0
-; LA64-NEXT:    bstrpick.d $a0, $a0, 15, 0
 ; LA64-NEXT:    sltui $a0, $a0, 1
 ; LA64-NEXT:    ret
   %and = and i16 %a, %b
@@ -80,14 +76,12 @@ define i1 @andn_icmp_ne_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ne_i8:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    andn $a0, $a1, $a0
-; LA32-NEXT:    andi $a0, $a0, 255
 ; LA32-NEXT:    sltu $a0, $zero, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ne_i8:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    andn $a0, $a1, $a0
-; LA64-NEXT:    andi $a0, $a0, 255
 ; LA64-NEXT:    sltu $a0, $zero, $a0
 ; LA64-NEXT:    ret
   %and = and i8 %a, %b
@@ -99,14 +93,12 @@ define i1 @andn_icmp_ne_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ne_i16:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    andn $a0, $a1, $a0
-; LA32-NEXT:    bstrpick.w $a0, $a0, 15, 0
 ; LA32-NEXT:    sltu $a0, $zero, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ne_i16:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    andn $a0, $a1, $a0
-; LA64-NEXT:    bstrpick.d $a0, $a0, 15, 0
 ; LA64-NEXT:    sltu $a0, $zero, $a0
 ; LA64-NEXT:    ret
   %and = and i16 %a, %b
@@ -153,15 +145,13 @@ define i1 @andn_icmp_ne_i64(i64 %a, i64 %b) nounwind {
 define i1 @andn_icmp_ult_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ult_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    andi $a1, $a1, 255
-; LA32-NEXT:    and $a0, $a1, $a0
+; LA32-NEXT:    and $a0, $a0, $a1
 ; LA32-NEXT:    sltu $a0, $a0, $a1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ult_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    andi $a1, $a1, 255
-; LA64-NEXT:    and $a0, $a1, $a0
+; LA64-NEXT:    and $a0, $a0, $a1
 ; LA64-NEXT:    sltu $a0, $a0, $a1
 ; LA64-NEXT:    ret
   %and = and i8 %a, %b
@@ -172,15 +162,13 @@ define i1 @andn_icmp_ult_i8(i8 signext %a, i8 signext %b) nounwind {
 define i1 @andn_icmp_ult_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ult_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
-; LA32-NEXT:    and $a0, $a1, $a0
+; LA32-NEXT:    and $a0, $a0, $a1
 ; LA32-NEXT:    sltu $a0, $a0, $a1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ult_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
-; LA64-NEXT:    and $a0, $a1, $a0
+; LA64-NEXT:    and $a0, $a0, $a1
 ; LA64-NEXT:    sltu $a0, $a0, $a1
 ; LA64-NEXT:    ret
   %and = and i16 %a, %b
@@ -191,16 +179,14 @@ define i1 @andn_icmp_ult_i16(i16 signext %a, i16 signext %b) nounwind {
 define i1 @andn_icmp_uge_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_uge_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    andi $a1, $a1, 255
-; LA32-NEXT:    and $a0, $a1, $a0
+; LA32-NEXT:    and $a0, $a0, $a1
 ; LA32-NEXT:    sltu $a0, $a0, $a1
 ; LA32-NEXT:    xori $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_uge_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    andi $a1, $a1, 255
-; LA64-NEXT:    and $a0, $a1, $a0
+; LA64-NEXT:    and $a0, $a0, $a1
 ; LA64-NEXT:    sltu $a0, $a0, $a1
 ; LA64-NEXT:    xori $a0, $a0, 1
 ; LA64-NEXT:    ret
@@ -212,16 +198,14 @@ define i1 @andn_icmp_uge_i8(i8 signext %a, i8 signext %b) nounwind {
 define i1 @andn_icmp_uge_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_uge_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
-; LA32-NEXT:    and $a0, $a1, $a0
+; LA32-NEXT:    and $a0, $a0, $a1
 ; LA32-NEXT:    sltu $a0, $a0, $a1
 ; LA32-NEXT:    xori $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_uge_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
-; LA64-NEXT:    and $a0, $a1, $a0
+; LA64-NEXT:    and $a0, $a0, $a1
 ; LA64-NEXT:    sltu $a0, $a0, $a1
 ; LA64-NEXT:    xori $a0, $a0, 1
 ; LA64-NEXT:    ret
@@ -233,15 +217,13 @@ define i1 @andn_icmp_uge_i16(i16 signext %a, i16 signext %b) nounwind {
 define i1 @andn_icmp_ugt_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ugt_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    andi $a1, $a1, 255
-; LA32-NEXT:    and $a0, $a1, $a0
+; LA32-NEXT:    and $a0, $a0, $a1
 ; LA32-NEXT:    sltu $a0, $a1, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ugt_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    andi $a1, $a1, 255
-; LA64-NEXT:    and $a0, $a1, $a0
+; LA64-NEXT:    and $a0, $a0, $a1
 ; LA64-NEXT:    sltu $a0, $a1, $a0
 ; LA64-NEXT:    ret
   %and = and i8 %a, %b
@@ -252,15 +234,13 @@ define i1 @andn_icmp_ugt_i8(i8 signext %a, i8 signext %b) nounwind {
 define i1 @andn_icmp_ugt_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ugt_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
-; LA32-NEXT:    and $a0, $a1, $a0
+; LA32-NEXT:    and $a0, $a0, $a1
 ; LA32-NEXT:    sltu $a0, $a1, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ugt_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
-; LA64-NEXT:    and $a0, $a1, $a0
+; LA64-NEXT:    and $a0, $a0, $a1
 ; LA64-NEXT:    sltu $a0, $a1, $a0
 ; LA64-NEXT:    ret
   %and = and i16 %a, %b
@@ -271,16 +251,14 @@ define i1 @andn_icmp_ugt_i16(i16 signext %a, i16 signext %b) nounwind {
 define i1 @andn_icmp_ule_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ule_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    andi $a1, $a1, 255
-; LA32-NEXT:    and $a0, $a1, $a0
+; LA32-NEXT:    and $a0, $a0, $a1
 ; LA32-NEXT:    sltu $a0, $a1, $a0
 ; LA32-NEXT:    xori $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ule_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    andi $a1, $a1, 255
-; LA64-NEXT:    and $a0, $a1, $a0
+; LA64-NEXT:    and $a0, $a0, $a1
 ; LA64-NEXT:    sltu $a0, $a1, $a0
 ; LA64-NEXT:    xori $a0, $a0, 1
 ; LA64-NEXT:    ret
@@ -292,16 +270,14 @@ define i1 @andn_icmp_ule_i8(i8 signext %a, i8 signext %b) nounwind {
 define i1 @andn_icmp_ule_i16(i16 signext %a, i16 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_ule_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
-; LA32-NEXT:    and $a0, $a1, $a0
+; LA32-NEXT:    and $a0, $a0, $a1
 ; LA32-NEXT:    sltu $a0, $a1, $a0
 ; LA32-NEXT:    xori $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: andn_icmp_ule_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
-; LA64-NEXT:    and $a0, $a1, $a0
+; LA64-NEXT:    and $a0, $a0, $a1
 ; LA64-NEXT:    sltu $a0, $a1, $a0
 ; LA64-NEXT:    xori $a0, $a0, 1
 ; LA64-NEXT:    ret
@@ -606,7 +582,6 @@ define i1 @andn_icmp_eq_i8_i32(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_eq_i8_i32:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    andn $a0, $a1, $a0
-; LA32-NEXT:    andi $a0, $a0, 255
 ; LA32-NEXT:    sltui $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;

From b15aa7f88c21d126bac4cf77eb05c522e0f1870d Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Mon, 22 Jul 2024 16:47:19 +1000
Subject: [PATCH 380/486] [ORC] Add unit test for MemoryFlags APIs, don't
 dereference end() iterator.

In AllocGroupSmallMap::find(AllocGroup) we were calling lower_bound(...) and
then unconditionally dereferencing the resulting iterator, however
lower_bound(...) may return end() if the value being searched for is higher
than any value present in the map. This patch adds a check for end() before
the dereference to guard against dereference of end().

This commit also adds some basic unit tests for MemProt and AllocGroupSmallMap.

rdar://129662981
---
 .../ExecutionEngine/Orc/Shared/MemoryFlags.h  |  2 +-
 .../ExecutionEngine/Orc/CMakeLists.txt        |  1 +
 .../ExecutionEngine/Orc/MemoryFlagsTest.cpp   | 55 +++++++++++++++++++
 3 files changed, 57 insertions(+), 1 deletion(-)
 create mode 100644 llvm/unittests/ExecutionEngine/Orc/MemoryFlagsTest.cpp

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h
index b8b5f90b6b0fb..fa522ff88e1db 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h
@@ -184,7 +184,7 @@ template <typename T> class AllocGroupSmallMap {
   iterator end() { return Elems.end(); }
   iterator find(AllocGroup G) {
     auto I = lower_bound(Elems, G, compareKey);
-    return (I->first == G) ? I : end();
+    return (I == end() || I->first == G) ? I : end();
   }
 
   bool empty() const { return Elems.empty(); }
diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
index af431658c9b7d..cc868e7587dc6 100644
--- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
@@ -28,6 +28,7 @@ add_llvm_unittest(OrcJITTests
   LookupAndRecordAddrsTest.cpp
   MachOPlatformTest.cpp
   MapperJITLinkMemoryManagerTest.cpp
+  MemoryFlagsTest.cpp
   MemoryMapperTest.cpp
   ObjectFormatsTest.cpp
   ObjectLinkingLayerTest.cpp
diff --git a/llvm/unittests/ExecutionEngine/Orc/MemoryFlagsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/MemoryFlagsTest.cpp
new file mode 100644
index 0000000000000..270ea2f442b9e
--- /dev/null
+++ b/llvm/unittests/ExecutionEngine/Orc/MemoryFlagsTest.cpp
@@ -0,0 +1,55 @@
+//===------- MemoryFlagsTest.cpp - Test MemoryFlags and related APIs ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
+#include "gtest/gtest.h"
+
+#include <future>
+
+using namespace llvm;
+using namespace llvm::orc;
+
+TEST(MemProtTest, Basics) {
+  MemProt MPNone = MemProt::None;
+
+  EXPECT_EQ(MPNone & MemProt::Read, MemProt::None);
+  EXPECT_EQ(MPNone & MemProt::Write, MemProt::None);
+  EXPECT_EQ(MPNone & MemProt::Exec, MemProt::None);
+
+  EXPECT_EQ(MPNone | MemProt::Read, MemProt::Read);
+  EXPECT_EQ(MPNone | MemProt::Write, MemProt::Write);
+  EXPECT_EQ(MPNone | MemProt::Exec, MemProt::Exec);
+
+  MemProt MPAll = MemProt::Read | MemProt::Write | MemProt::Exec;
+  EXPECT_EQ(MPAll & MemProt::Read, MemProt::Read);
+  EXPECT_EQ(MPAll & MemProt::Write, MemProt::Write);
+  EXPECT_EQ(MPAll & MemProt::Exec, MemProt::Exec);
+}
+
+TEST(AllocGroupSmallMap, EmptyMap) {
+  AllocGroupSmallMap<bool> EM;
+  EXPECT_TRUE(EM.empty());
+  EXPECT_EQ(EM.size(), 0u);
+}
+
+TEST(AllocGroupSmallMap, NonEmptyMap) {
+  AllocGroupSmallMap<unsigned> NEM;
+  NEM[MemProt::Read] = 42;
+
+  EXPECT_FALSE(NEM.empty());
+  EXPECT_EQ(NEM.size(), 1U);
+  EXPECT_EQ(NEM[MemProt::Read], 42U);
+  EXPECT_EQ(NEM.find(MemProt::Read), NEM.begin());
+  EXPECT_EQ(NEM.find(MemProt::Read | MemProt::Write), NEM.end());
+
+  NEM[MemProt::Read | MemProt::Write] = 7;
+  EXPECT_EQ(NEM.size(), 2U);
+  EXPECT_EQ(NEM.begin()->second, 42U);
+  EXPECT_EQ((NEM.begin() + 1)->second, 7U);
+}
+

From 12762b8648d88e6823aedc82f663a596fa2eef76 Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777@gmail.com>
Date: Mon, 22 Jul 2024 15:37:17 +0800
Subject: [PATCH 381/486] [Clang][Interp] Diagnose use sizeless vector type as
 the argument of `__builtin_vectorelements` (#99794)

This PR implement an opcode to diagnose use sizeless vector type as the
argument of `__builtin_vectorelements`.

---------

Signed-off-by: yronglin <yronglin777@gmail.com>
---
 clang/lib/AST/Interp/Compiler.cpp             | 4 +---
 clang/lib/AST/Interp/Interp.h                 | 9 +++++++++
 clang/lib/AST/Interp/Opcodes.td               | 2 ++
 clang/test/SemaCXX/builtin_vectorelements.cpp | 2 ++
 4 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/clang/lib/AST/Interp/Compiler.cpp b/clang/lib/AST/Interp/Compiler.cpp
index ef579bc5d8972..0fc93c14131e6 100644
--- a/clang/lib/AST/Interp/Compiler.cpp
+++ b/clang/lib/AST/Interp/Compiler.cpp
@@ -1698,10 +1698,8 @@ bool Compiler<Emitter>::VisitUnaryExprOrTypeTraitExpr(
   if (Kind == UETT_VectorElements) {
     if (const auto *VT = E->getTypeOfArgument()->getAs<VectorType>())
       return this->emitConst(VT->getNumElements(), E);
-
-    // FIXME: Apparently we need to catch the fact that a sizeless vector type
-    // has been passed and diagnose that (at run time).
     assert(E->getTypeOfArgument()->isSizelessVectorType());
+    return this->emitSizelessVectorElementSize(E);
   }
 
   if (Kind == UETT_VecStep) {
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index b2581b5f7b5d0..b8e4432004e99 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -2736,6 +2736,15 @@ inline bool InvalidDeclRef(InterpState &S, CodePtr OpPC,
   return CheckDeclRef(S, OpPC, DR);
 }
 
+inline bool SizelessVectorElementSize(InterpState &S, CodePtr OpPC) {
+  if (S.inConstantContext()) {
+    const SourceRange &ArgRange = S.Current->getRange(OpPC);
+    const Expr *E = S.Current->getExpr(OpPC);
+    S.CCEDiag(E, diag::note_constexpr_non_const_vectorelements) << ArgRange;
+  }
+  return false;
+}
+
 inline bool Assume(InterpState &S, CodePtr OpPC) {
   const auto Val = S.Stk.pop<Boolean>();
 
diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/Interp/Opcodes.td
index 49ebb156ab2fb..9f29fa9272711 100644
--- a/clang/lib/AST/Interp/Opcodes.td
+++ b/clang/lib/AST/Interp/Opcodes.td
@@ -733,6 +733,8 @@ def InvalidDeclRef : Opcode {
   let Args = [ArgDeclRef];
 }
 
+def SizelessVectorElementSize : Opcode;
+
 def Assume : Opcode;
 
 def ArrayDecay : Opcode;
diff --git a/clang/test/SemaCXX/builtin_vectorelements.cpp b/clang/test/SemaCXX/builtin_vectorelements.cpp
index 59ff09ac72e42..b23675ea0ac6a 100644
--- a/clang/test/SemaCXX/builtin_vectorelements.cpp
+++ b/clang/test/SemaCXX/builtin_vectorelements.cpp
@@ -1,7 +1,9 @@
 // RUN: %clang_cc1 -triple x86_64                       -std=c++20 -fsyntax-only -verify -disable-llvm-passes %s
+// RUN: %clang_cc1 -triple x86_64                       -std=c++20 -fsyntax-only -verify -disable-llvm-passes -fexperimental-new-constant-interpreter %s
 
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sve -std=c++20 -fsyntax-only -verify -disable-llvm-passes %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sve -std=c++20 -fsyntax-only -verify -disable-llvm-passes -fexperimental-new-constant-interpreter %s
 
 template <typename T>
 using VecT __attribute__((vector_size(16))) = T;

From 56ad7cc0126f9899fd884391cfa10b6359206c01 Mon Sep 17 00:00:00 2001
From: AtariDreams <gfunni234@gmail.com>
Date: Mon, 22 Jul 2024 03:47:37 -0400
Subject: [PATCH 382/486] [IR] Remove non-canonical matchings (#96763)

---
 llvm/lib/Analysis/ValueTracking.cpp                       | 2 +-
 llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp   | 2 +-
 llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp | 6 ++----
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 5c56654246ead..8f1c42b7ede44 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -934,7 +934,7 @@ getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
     // Demanded) == (xor(x, x-1) & Demanded). Extend the xor pattern
     // to use arbitrary C if xor(x, x-C) as the same as xor(x, x-1).
     if (HasKnownOne &&
-        match(I, m_c_Xor(m_Value(X), m_c_Add(m_Deferred(X), m_AllOnes())))) {
+        match(I, m_c_Xor(m_Value(X), m_Add(m_Deferred(X), m_AllOnes())))) {
       const KnownBits &XBits = I->getOperand(0) == X ? KnownLHS : KnownRHS;
       KnownOut = XBits.blsmsk();
     }
diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
index b75cd1beadc58..4ef009c87a1e6 100644
--- a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
@@ -829,7 +829,7 @@ bool PolynomialMultiplyRecognize::matchRightShift(SelectInst *SelI,
     return false;
 
   Value *X = nullptr;
-  if (!match(C, m_c_And(m_Value(X), m_One())))
+  if (!match(C, m_And(m_Value(X), m_One())))
     return false;
   // Matched: select (X & 1) == +++ ? ... : ...
   //          select (X & 1) != +++ ? ... : ...
diff --git a/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
index 9be60721bebb5..75585fcc80266 100644
--- a/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
@@ -425,14 +425,12 @@ void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForAdd(
 
 // Returns true if A matches B + C where C is constant.
 static bool matchesAdd(Value *A, Value *&B, ConstantInt *&C) {
-  return (match(A, m_Add(m_Value(B), m_ConstantInt(C))) ||
-          match(A, m_Add(m_ConstantInt(C), m_Value(B))));
+  return match(A, m_c_Add(m_Value(B), m_ConstantInt(C)));
 }
 
 // Returns true if A matches B | C where C is constant.
 static bool matchesOr(Value *A, Value *&B, ConstantInt *&C) {
-  return (match(A, m_Or(m_Value(B), m_ConstantInt(C))) ||
-          match(A, m_Or(m_ConstantInt(C), m_Value(B))));
+  return match(A, m_c_Or(m_Value(B), m_ConstantInt(C)));
 }
 
 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForMul(

From c7bfc41860a6abe5c92dc5afb47348b0c9e69963 Mon Sep 17 00:00:00 2001
From: Younan Zhang <zyn7109@gmail.com>
Date: Mon, 22 Jul 2024 16:37:05 +0800
Subject: [PATCH 383/486] [Clang][NFCI] Prefer non-canonical template arguments
 for synthesized CTAD guides (#99840)

This seems to be low-hanging fruit: We could remove all calls to
`Context.getCanonicalTemplateArgument()` and gain a better
diagnostic/AST.

The non-canonical template arguments shouldn't make a difference when
synthesizing a CTAD guide, so this is intended to be an NFC.

Closes https://github.com/llvm/llvm-project/issues/79798
---
 clang/lib/Sema/SemaTemplateDeductionGuide.cpp | 25 +++----
 clang/test/AST/ast-dump-ctad-alias.cpp        | 34 +++++-----
 .../temp.deduct/temp.deduct.call/p3-0x.cpp    |  2 +-
 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp  | 30 ++++----
 clang/test/SemaTemplate/deduction-guide.cpp   | 68 +++++++++----------
 5 files changed, 79 insertions(+), 80 deletions(-)

diff --git a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
index 7dff2c8f98589..0602d07c6b9b0 100644
--- a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
+++ b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
@@ -363,8 +363,7 @@ struct ConvertConstructorToDeductionGuideTransform {
           return nullptr;
         // Constraints require that we substitute depth-1 arguments
         // to match depths when substituted for evaluation later
-        Depth1Args.push_back(SemaRef.Context.getCanonicalTemplateArgument(
-            SemaRef.Context.getInjectedTemplateArg(NewParam)));
+        Depth1Args.push_back(SemaRef.Context.getInjectedTemplateArg(NewParam));
 
         if (NestedPattern) {
           TemplateDeclInstantiator Instantiator(SemaRef, DC,
@@ -379,8 +378,7 @@ struct ConvertConstructorToDeductionGuideTransform {
                "Unexpected template parameter depth");
 
         AllParams.push_back(NewParam);
-        SubstArgs.push_back(SemaRef.Context.getCanonicalTemplateArgument(
-            SemaRef.Context.getInjectedTemplateArg(NewParam)));
+        SubstArgs.push_back(SemaRef.Context.getInjectedTemplateArg(NewParam));
       }
 
       // Substitute new template parameters into requires-clause if present.
@@ -795,8 +793,8 @@ buildAssociatedConstraints(Sema &SemaRef, FunctionTemplateDecl *F,
         /*NewIndex=*/AdjustedAliasTemplateArgs.size(),
         getTemplateParameterDepth(TP) + AdjustDepth);
 
-    auto NewTemplateArgument = Context.getCanonicalTemplateArgument(
-        Context.getInjectedTemplateArg(NewParam));
+    TemplateArgument NewTemplateArgument =
+        Context.getInjectedTemplateArg(NewParam);
     AdjustedAliasTemplateArgs.push_back(NewTemplateArgument);
   }
   // Template arguments used to transform the template arguments in
@@ -822,8 +820,8 @@ buildAssociatedConstraints(Sema &SemaRef, FunctionTemplateDecl *F,
           getTemplateParameterDepth(TP) + AdjustDepth);
       FirstUndeducedParamIdx += 1;
       assert(TemplateArgsForBuildingRC[Index].isNull());
-      TemplateArgsForBuildingRC[Index] = Context.getCanonicalTemplateArgument(
-          Context.getInjectedTemplateArg(NewParam));
+      TemplateArgsForBuildingRC[Index] =
+          Context.getInjectedTemplateArg(NewParam);
       continue;
     }
     TemplateArgumentLoc Input =
@@ -923,8 +921,8 @@ Expr *buildIsDeducibleConstraint(Sema &SemaRef,
           /*NewIndex=*/TransformedTemplateArgs.size(),
           getTemplateParameterDepth(TP) + AdjustDepth);
 
-      auto NewTemplateArgument = Context.getCanonicalTemplateArgument(
-          Context.getInjectedTemplateArg(NewParam));
+      TemplateArgument NewTemplateArgument =
+          Context.getInjectedTemplateArg(NewParam);
       TransformedTemplateArgs.push_back(NewTemplateArgument);
     }
     // Transformed the ReturnType to restore the uninstantiated depth.
@@ -1087,8 +1085,8 @@ BuildDeductionGuideForTypeAlias(Sema &SemaRef,
         getTemplateParameterDepth(TP));
     FPrimeTemplateParams.push_back(NewParam);
 
-    auto NewTemplateArgument = Context.getCanonicalTemplateArgument(
-        Context.getInjectedTemplateArg(NewParam));
+    TemplateArgument NewTemplateArgument =
+        Context.getInjectedTemplateArg(NewParam);
     TransformedDeducedAliasArgs[AliasTemplateParamIdx] = NewTemplateArgument;
   }
   unsigned FirstUndeducedParamIdx = FPrimeTemplateParams.size();
@@ -1109,8 +1107,7 @@ BuildDeductionGuideForTypeAlias(Sema &SemaRef,
     assert(TemplateArgsForBuildingFPrime[FTemplateParamIdx].isNull() &&
            "The argument must be null before setting");
     TemplateArgsForBuildingFPrime[FTemplateParamIdx] =
-        Context.getCanonicalTemplateArgument(
-            Context.getInjectedTemplateArg(NewParam));
+        Context.getInjectedTemplateArg(NewParam);
   }
 
   // To form a deduction guide f' from f, we leverage clang's instantiation
diff --git a/clang/test/AST/ast-dump-ctad-alias.cpp b/clang/test/AST/ast-dump-ctad-alias.cpp
index a088c0a7b0272..b1631f7822ce0 100644
--- a/clang/test/AST/ast-dump-ctad-alias.cpp
+++ b/clang/test/AST/ast-dump-ctad-alias.cpp
@@ -32,22 +32,24 @@ Out2<double>::AInner t(1.0);
 // CHECK-NEXT: |     | |-UnresolvedLookupExpr {{.*}} '<dependent type>' lvalue (no ADL) = 'Concept'
 // CHECK-NEXT: |     | | |-TemplateArgument type 'int'
 // CHECK-NEXT: |     | | | `-BuiltinType {{.*}} 'int'
-// CHECK-NEXT: |     | | `-TemplateArgument type 'type-parameter-1-0'
-// CHECK-NEXT: |     | |   `-TemplateTypeParmType {{.*}} 'type-parameter-1-0' dependent depth 1 index 0
+// CHECK-NEXT: |     | | `-TemplateArgument type 'Y':'type-parameter-1-0'
+// CHECK-NEXT: |     | |   `-TemplateTypeParmType {{.*}} 'Y' dependent depth 1 index 0
+// CHECK-NEXT: |     | |     `-TemplateTypeParm {{.*}} 'Y'
 // CHECK-NEXT: |     | `-TypeTraitExpr {{.*}} 'bool' __is_deducible
 // CHECK-NEXT: |     |   |-DeducedTemplateSpecializationType {{.*}} 'Out2<double>::AInner' dependent
 // CHECK-NEXT: |     |   | `-name: 'Out2<double>::AInner'
 // CHECK-NEXT: |     |   |   `-TypeAliasTemplateDecl {{.+}} AInner{{$}}
-// CHECK-NEXT: |     |   `-ElaboratedType {{.*}} 'Inner<type-parameter-1-0>' sugar dependent
-// CHECK-NEXT: |     |     `-TemplateSpecializationType {{.*}} 'Inner<type-parameter-1-0>' dependent
+// CHECK-NEXT: |     |   `-ElaboratedType {{.*}} 'Inner<Y>' sugar dependent
+// CHECK-NEXT: |     |     `-TemplateSpecializationType {{.*}} 'Inner<Y>' dependent
 // CHECK-NEXT: |     |       |-name: 'Inner':'Out<int>::Inner' qualified
 // CHECK-NEXT: |     |       | `-ClassTemplateDecl {{.+}} Inner{{$}}
-// CHECK-NEXT: |     |       `-TemplateArgument type 'type-parameter-1-0'
-// CHECK-NEXT: |     |         `-SubstTemplateTypeParmType {{.*}} 'type-parameter-1-0'
+// CHECK-NEXT: |     |       `-TemplateArgument type 'Y'
+// CHECK-NEXT: |     |         `-SubstTemplateTypeParmType {{.*}} 'Y'
 // CHECK-NEXT: |     |           |-FunctionTemplate {{.*}} '<deduction guide for Inner>'
-// CHECK-NEXT: |     |           `-TemplateTypeParmType {{.*}} 'type-parameter-1-0' dependent depth 1 index 0
-// CHECK-NEXT: |     |-CXXDeductionGuideDecl {{.*}} <deduction guide for AInner> 'auto (type-parameter-0-0) -> Inner<type-parameter-0-0>'
-// CHECK-NEXT: |     | `-ParmVarDecl {{.*}} 'type-parameter-0-0'
+// CHECK-NEXT: |     |           `-TemplateTypeParmType {{.*}} 'Y' dependent depth 1 index 0
+// CHECK-NEXT: |     |             `-TemplateTypeParm {{.*}} 'Y'
+// CHECK-NEXT: |     |-CXXDeductionGuideDecl {{.*}} <deduction guide for AInner> 'auto (Y) -> Inner<Y>'
+// CHECK-NEXT: |     | `-ParmVarDecl {{.*}} 'Y'
 // CHECK-NEXT: |     `-CXXDeductionGuideDecl {{.*}} used <deduction guide for AInner> 'auto (double) -> Inner<double>' implicit_instantiation
 // CHECK-NEXT: |       |-TemplateArgument type 'double'
 // CHECK-NEXT: |       | `-BuiltinType {{.*}} 'double'
@@ -77,8 +79,8 @@ AFoo3 afoo3{0, 1};
 // CHECK-NEXT: | |-UnresolvedLookupExpr {{.*}} '<dependent type>' lvalue (no ADL) = 'Concept'
 // CHECK-NEXT: | | |-TemplateArgument type 'int'
 // CHECK-NEXT: | | | `-BuiltinType {{.*}} 'int'
-// CHECK-NEXT: | | `-TemplateArgument type 'type-parameter-0-1'
-// CHECK-NEXT: | |   `-TemplateTypeParmType {{.*}} 'type-parameter-0-1' dependent depth 0 index 1
+// CHECK-NEXT: | | `-TemplateArgument type 'V'
+// CHECK-NEXT: | |   `-TemplateTypeParmType {{.*}} 'V' dependent depth 0 index 1
 
 template <typename... T1>
 struct Foo {
@@ -88,16 +90,16 @@ struct Foo {
 template <typename...T2>
 using AFoo = Foo<T2...>;
 AFoo a(1, 2);
-// CHECK:      |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for AFoo> 'auto (type-parameter-0-0...) -> Foo<type-parameter-0-0...>'
-// CHECK-NEXT: | | `-ParmVarDecl {{.*}} 'type-parameter-0-0...' pack
+// CHECK:      |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for AFoo> 'auto (T2...) -> Foo<T2...>'
+// CHECK-NEXT: | | `-ParmVarDecl {{.*}} 'T2...' pack
 // CHECK-NEXT: | `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for AFoo> 'auto (int, int) -> Foo<int, int>' implicit_instantiation
 
 template <typename T>
 using BFoo = Foo<T, T>;
 BFoo b2(1.0, 2.0);
-// CHECK:      |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for BFoo> 'auto (type-parameter-0-0, type-parameter-0-0) -> Foo<type-parameter-0-0, type-parameter-0-0>'
-// CHECK-NEXT: | | |-ParmVarDecl {{.*}} 'type-parameter-0-0'
-// CHECK-NEXT: | | `-ParmVarDecl {{.*}} 'type-parameter-0-0'
+// CHECK:      |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for BFoo> 'auto (T, T) -> Foo<T, T>'
+// CHECK-NEXT: | | |-ParmVarDecl {{.*}} 'T'
+// CHECK-NEXT: | | `-ParmVarDecl {{.*}} 'T'
 // CHECK-NEXT: | `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for BFoo> 'auto (double, double) -> Foo<double, double>' implicit_instantiation
 
 namespace GH90209 {
diff --git a/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.call/p3-0x.cpp b/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.call/p3-0x.cpp
index ed445360c4fdd..b9ff26a7620db 100644
--- a/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.call/p3-0x.cpp
+++ b/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.call/p3-0x.cpp
@@ -78,7 +78,7 @@ namespace std_example {
   template<class T> struct A { // expected-note {{candidate}} expected-note {{implicit deduction guide}}
     template<class U>
     A(T &&, U &&, int *); // expected-note {{[with T = int, U = int] not viable: expects an rvalue}} \
-                          // expected-note {{implicit deduction guide declared as 'template <class T, class U> A(T &&, type-parameter-0-1 &&, int *) -> A<T>'}}
+                          // expected-note {{implicit deduction guide declared as 'template <class T, class U> A(T &&, U &&, int *) -> A<T>'}}
     A(T &&, int *);       // expected-note {{requires 2}} \
                           // expected-note {{implicit deduction guide declared as 'template <class T> A(T &&, int *) -> A<T>'}}
   };
diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
index 2193be03ad9a3..5392573fcdb9d 100644
--- a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
+++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -110,8 +110,8 @@ struct Foo {
 
 template <typename X, int Y>
 using Bar = Foo<X, sizeof(X)>; // expected-note {{candidate template ignored: couldn't infer template argument 'X'}} \
-                               // expected-note {{implicit deduction guide declared as 'template <typename X> requires __is_deducible(test9::Bar, Foo<type-parameter-0-0, sizeof(type-parameter-0-0)>) Bar(Foo<type-parameter-0-0, sizeof(type-parameter-0-0)>) -> Foo<type-parameter-0-0, sizeof(type-parameter-0-0)>'}} \
-                               // expected-note {{implicit deduction guide declared as 'template <typename X> requires __is_deducible(test9::Bar, Foo<type-parameter-0-0, sizeof(type-parameter-0-0)>) Bar(const type-parameter-0-0 (&)[sizeof(type-parameter-0-0)]) -> Foo<type-parameter-0-0, sizeof(type-parameter-0-0)>'}} \
+                               // expected-note {{implicit deduction guide declared as 'template <typename X> requires __is_deducible(test9::Bar, Foo<X, sizeof(X)>) Bar(Foo<X, sizeof(X)>) -> Foo<X, sizeof(X)>'}} \
+                               // expected-note {{implicit deduction guide declared as 'template <typename X> requires __is_deducible(test9::Bar, Foo<X, sizeof(X)>) Bar(const X (&)[sizeof(X)]) -> Foo<X, sizeof(X)>'}} \
                                // expected-note {{candidate template ignored: constraints not satisfied [with X = int]}} \
                                // expected-note {{cannot deduce template arguments for 'Bar' from 'Foo<int, 4UL>'}}
 
@@ -138,13 +138,13 @@ namespace test11 {
 struct A {};
 template<class T> struct Foo { T c; };
 template<class X, class Y=A>
-using AFoo = Foo<Y>; // expected-note {{candidate template ignored: could not match 'Foo<type-parameter-0-0>' against 'int'}} \
-                    // expected-note {{implicit deduction guide declared as 'template <class Y = A> requires __is_deducible(test11::AFoo, Foo<type-parameter-0-0>) AFoo(Foo<type-parameter-0-0>) -> Foo<type-parameter-0-0>'}} \
+using AFoo = Foo<Y>; // expected-note {{candidate template ignored: could not match 'Foo<Y>' against 'int'}} \
+                    // expected-note {{implicit deduction guide declared as 'template <class Y = A> requires __is_deducible(test11::AFoo, Foo<Y>) AFoo(Foo<Y>) -> Foo<Y>'}} \
                     // expected-note {{candidate template ignored: constraints not satisfied [with Y = int]}} \
                     // expected-note {{cannot deduce template arguments for 'AFoo' from 'Foo<int>'}} \
-                    // expected-note {{implicit deduction guide declared as 'template <class Y = A> requires __is_deducible(test11::AFoo, Foo<type-parameter-0-0>) AFoo(type-parameter-0-0) -> Foo<type-parameter-0-0>'}} \
+                    // expected-note {{implicit deduction guide declared as 'template <class Y = A> requires __is_deducible(test11::AFoo, Foo<Y>) AFoo(Y) -> Foo<Y>'}} \
                     // expected-note {{candidate function template not viable: requires 0 arguments, but 1 was provided}} \
-                    // expected-note {{implicit deduction guide declared as 'template <class Y = A> requires __is_deducible(test11::AFoo, Foo<type-parameter-0-0>) AFoo() -> Foo<type-parameter-0-0>'}}
+                    // expected-note {{implicit deduction guide declared as 'template <class Y = A> requires __is_deducible(test11::AFoo, Foo<Y>) AFoo() -> Foo<Y>'}}
 
 AFoo s = {1}; // expected-error {{no viable constructor or deduction guide for deduction of template arguments of 'AFoo'}}
 } // namespace test11
@@ -211,9 +211,9 @@ template<typename> concept False = false;
 template<False W>
 using BFoo = AFoo<W>; // expected-note {{candidate template ignored: constraints not satisfied [with V = int]}} \
                       // expected-note {{cannot deduce template arguments for 'BFoo' from 'Foo<int *>'}} \
-                      // expected-note {{implicit deduction guide declared as 'template <class V> requires __is_deducible(AFoo, Foo<type-parameter-0-0 *>) && __is_deducible(test15::BFoo, Foo<type-parameter-0-0 *>) BFoo(type-parameter-0-0 *) -> Foo<type-parameter-0-0 *>}} \
-                      // expected-note {{candidate template ignored: could not match 'Foo<type-parameter-0-0 *>' against 'int *'}} \
-                      // expected-note {{template <class V> requires __is_deducible(AFoo, Foo<type-parameter-0-0 *>) && __is_deducible(test15::BFoo, Foo<type-parameter-0-0 *>) BFoo(Foo<type-parameter-0-0 *>) -> Foo<type-parameter-0-0 *>}}
+                      // expected-note {{implicit deduction guide declared as 'template <class V> requires __is_deducible(AFoo, Foo<V *>) && __is_deducible(test15::BFoo, Foo<V *>) BFoo(V *) -> Foo<V *>}} \
+                      // expected-note {{candidate template ignored: could not match 'Foo<V *>' against 'int *'}} \
+                      // expected-note {{template <class V> requires __is_deducible(AFoo, Foo<V *>) && __is_deducible(test15::BFoo, Foo<V *>) BFoo(Foo<V *>) -> Foo<V *>}}
 int i = 0;
 AFoo a1(&i); // OK, deduce Foo<int *>
 
@@ -263,12 +263,12 @@ template<typename T> requires False<T> // expected-note {{because 'int' does not
 Foo(T) -> Foo<int>;
 
 template <typename U>
-using Bar = Foo<U>; // expected-note {{could not match 'Foo<type-parameter-0-0>' against 'int'}} \
-                    // expected-note {{implicit deduction guide declared as 'template <typename U> requires __is_deducible(test18::Bar, Foo<type-parameter-0-0>) Bar(Foo<type-parameter-0-0>) -> Foo<type-parameter-0-0>'}} \
+using Bar = Foo<U>; // expected-note {{could not match 'Foo<U>' against 'int'}} \
+                    // expected-note {{implicit deduction guide declared as 'template <typename U> requires __is_deducible(test18::Bar, Foo<U>) Bar(Foo<U>) -> Foo<U>'}} \
                     // expected-note {{candidate template ignored: constraints not satisfied}} \
-                    // expected-note {{implicit deduction guide declared as 'template <typename T> requires False<type-parameter-0-0> && __is_deducible(test18::Bar, Foo<int>) Bar(type-parameter-0-0) -> Foo<int>'}} \
+                    // expected-note {{implicit deduction guide declared as 'template <typename T> requires False<T> && __is_deducible(test18::Bar, Foo<int>) Bar(T) -> Foo<int>'}} \
                     // expected-note {{candidate function template not viable}} \
-                    // expected-note {{implicit deduction guide declared as 'template <typename U> requires __is_deducible(test18::Bar, Foo<type-parameter-0-0>) Bar() -> Foo<type-parameter-0-0>'}}
+                    // expected-note {{implicit deduction guide declared as 'template <typename U> requires __is_deducible(test18::Bar, Foo<U>) Bar() -> Foo<U>'}}
 
 Bar s = {1}; // expected-error {{no viable constructor or deduction guide for deduction of template arguments}}
 } // namespace test18
@@ -296,8 +296,8 @@ class Foo {};
 // Verify that template template type parameter TTP is referenced/used in the
 // template arguments of the RHS.
 template <template<typename> typename TTP>
-using Bar = Foo<K<TTP>>; // expected-note {{candidate template ignored: could not match 'Foo<K<template-parameter-0-0>>' against 'int'}} \
-                        // expected-note {{implicit deduction guide declared as 'template <template <typename> typename TTP> requires __is_deducible(test20::Bar, Foo<K<template-parameter-0-0>>) Bar(Foo<K<template-parameter-0-0>>) -> Foo<K<template-parameter-0-0>>'}}
+using Bar = Foo<K<TTP>>; // expected-note {{candidate template ignored: could not match 'Foo<K<TTP>>' against 'int'}} \
+                        // expected-note {{implicit deduction guide declared as 'template <template <typename> typename TTP> requires __is_deducible(test20::Bar, Foo<K<TTP>>) Bar(Foo<K<TTP>>) -> Foo<K<TTP>>'}}
 
 template <class T>
 class Container {};
diff --git a/clang/test/SemaTemplate/deduction-guide.cpp b/clang/test/SemaTemplate/deduction-guide.cpp
index 7b9ca4d83b020..043a4ccf56e36 100644
--- a/clang/test/SemaTemplate/deduction-guide.cpp
+++ b/clang/test/SemaTemplate/deduction-guide.cpp
@@ -68,7 +68,7 @@ using BT = B<char, 'x'>;
 // CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 0 T
 // CHECK: |-NonTypeTemplateParmDecl {{.*}} 'T' depth 0 index 1 V
 // CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 2 U
-// CHECK: |-NonTypeTemplateParmDecl {{.*}} 'type-parameter-0-2' depth 0 index 3 W
+// CHECK: |-NonTypeTemplateParmDecl {{.*}} 'U' depth 0 index 3 W
 // CHECK: |-CXXDeductionGuideDecl {{.*}} 'auto (X<W, V>) -> B<T, V>'
 // CHECK: | `-ParmVarDecl {{.*}} 'X<W, V>'
 // CHECK: `-CXXDeductionGuideDecl {{.*}} 'auto (X<nullptr, 'x'>) -> B<char, 'x'>'
@@ -81,7 +81,7 @@ using BT = B<char, 'x'>;
 // CHECK: |-InjectedClassNameType {{.*}} 'B<T, V>' dependent
 // CHECK: `-TemplateSpecializationType {{.*}} 'X<W, V>' dependent
 // CHECK:   |-TemplateArgument expr
-// CHECK:   | `-DeclRefExpr {{.*}} 'type-parameter-0-2' NonTypeTemplateParm {{.*}} 'W' 'type-parameter-0-2'
+// CHECK:   | `-DeclRefExpr {{.*}} 'U' NonTypeTemplateParm {{.*}} 'W' 'U'
 // CHECK:   `-TemplateArgument expr
 // CHECK:     `-DeclRefExpr {{.*}} 'T' NonTypeTemplateParm {{.*}} 'V' 'T'
 
@@ -99,13 +99,13 @@ using CT = C<int>;
 // CHECK: | |-TemplateTypeParmDecl {{.*}} typename depth 1 index 0 X
 // CHECK: | `-NonTypeTemplateParmDecl {{.*}} 'X' depth 1 index 1
 // CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 2 U
-// CHECK: |-NonTypeTemplateParmDecl {{.*}} 'type-parameter-0-2' depth 0 index 3 V
+// CHECK: |-NonTypeTemplateParmDecl {{.*}} 'U' depth 0 index 3 V
 // CHECK: | `-TemplateArgument {{.*}} expr
 // CHECK: |   `-IntegerLiteral {{.*}} 'int' 0
-// CHECK: |-CXXDeductionGuideDecl {{.*}} 'auto (A, Y<template-parameter-0-1>, type-parameter-0-2) -> C<A>'
+// CHECK: |-CXXDeductionGuideDecl {{.*}} 'auto (A, Y<T>, U) -> C<A>'
 // CHECK: | |-ParmVarDecl {{.*}} 'A'
-// CHECK: | |-ParmVarDecl {{.*}} 'Y<template-parameter-0-1>'
-// CHECK: | `-ParmVarDecl {{.*}} 'type-parameter-0-2'
+// CHECK: | |-ParmVarDecl {{.*}} 'Y<T>'
+// CHECK: | `-ParmVarDecl {{.*}} 'U'
 // CHECK: `-CXXDeductionGuideDecl {{.*}} 'auto (int, Y<B>, int) -> C<int>'
 // CHECK:  |-TemplateArgument type 'int'
 // CHECK:  |-TemplateArgument template 'B'
@@ -114,20 +114,20 @@ using CT = C<int>;
 // CHECK:  |-ParmVarDecl {{.*}} 'int'
 // CHECK:  |-ParmVarDecl {{.*}} 'Y<B>'
 // CHECK:  `-ParmVarDecl {{.*}} 'int'
-// CHECK: FunctionProtoType {{.*}} 'auto (A, Y<template-parameter-0-1>, type-parameter-0-2) -> C<A>' dependent trailing_return cdecl
+// CHECK: FunctionProtoType {{.*}} 'auto (A, Y<T>, U) -> C<A>' dependent trailing_return cdecl
 // CHECK: |-InjectedClassNameType {{.*}} 'C<A>' dependent
 // CHECK: |-TemplateTypeParmType {{.*}} 'A' dependent depth 0 index 0
 // CHECK: | `-TemplateTypeParm {{.*}} 'A'
-// CHECK: |-ElaboratedType {{.*}} 'Y<template-parameter-0-1>' sugar dependent
-// CHECK: | `-TemplateSpecializationType {{.*}} 'Y<template-parameter-0-1>' dependent
+// CHECK: |-ElaboratedType {{.*}} 'Y<T>' sugar dependent
+// CHECK: | `-TemplateSpecializationType {{.*}} 'Y<T>' dependent
 // CHECK: |   `-TemplateArgument template
-// CHECK: `-TemplateTypeParmType {{.*}} 'type-parameter-0-2' dependent depth 0 index 2
+// CHECK: `-TemplateTypeParmType {{.*}} 'U' dependent depth 0 index 2
 
 template<typename ...T> struct D { // expected-note {{candidate}} \
                                    // expected-note {{implicit deduction guide declared as 'template <typename ...T> D(D<T...>) -> D<T...>'}}
   template<typename... U> using B = int(int (*...p)(T, U));
   template<typename U1, typename U2> D(B<U1, U2>*); // expected-note {{candidate}} \
-                                                    // expected-note {{implicit deduction guide declared as 'template <typename ...T, typename U1, typename U2> D(B<type-parameter-0-1, type-parameter-0-2> *) -> D<T...>'}}
+                                                    // expected-note {{implicit deduction guide declared as 'template <typename ...T, typename U1, typename U2> D(B<U1, U2> *) -> D<T...>'}}
 };
 int f(int(int, int), int(int, int));
 // FIXME: We can't deduce this because we can't deduce through a
@@ -141,14 +141,14 @@ using DT = D<int, int>;
 // CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 0 ... T
 // CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 1 U1
 // CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 2 U2
-// CHECK: `-CXXDeductionGuideDecl {{.*}} 'auto (B<type-parameter-0-1, type-parameter-0-2> *) -> D<T...>'
-// CHECK:   `-ParmVarDecl {{.*}} 'B<type-parameter-0-1, type-parameter-0-2> *'
-// CHECK: FunctionProtoType {{.*}} 'auto (B<type-parameter-0-1, type-parameter-0-2> *) -> D<T...>' dependent trailing_return
+// CHECK: `-CXXDeductionGuideDecl {{.*}} 'auto (B<U1, U2> *) -> D<T...>'
+// CHECK:   `-ParmVarDecl {{.*}} 'B<U1, U2> *'
+// CHECK: FunctionProtoType {{.*}} 'auto (B<U1, U2> *) -> D<T...>' dependent trailing_return
 // CHECK: |-InjectedClassNameType {{.*}} 'D<T...>' dependent
-// CHECK: `-PointerType {{.*}} 'B<type-parameter-0-1, type-parameter-0-2> *' dependent
-// CHECK:   `-TemplateSpecializationType {{.*}} 'B<type-parameter-0-1, type-parameter-0-2>' sugar dependent alias
-// CHECK:     |-TemplateArgument type 'type-parameter-0-1'
-// CHECK:     |-TemplateArgument type 'type-parameter-0-2'
+// CHECK: `-PointerType {{.*}} 'B<U1, U2> *' dependent
+// CHECK:   `-TemplateSpecializationType {{.*}} 'B<U1, U2>' sugar dependent alias
+// CHECK:     |-TemplateArgument type 'U1'
+// CHECK:     |-TemplateArgument type 'U2'
 // CHECK:     `-FunctionProtoType {{.*}} 'int (int (*)(T, U)...)' dependent cdecl
 // CHECK:       |-BuiltinType {{.*}} 'int'
 // CHECK:       `-PackExpansionType {{.*}} 'int (*)(T, U)...' dependent expansions 2
@@ -232,17 +232,17 @@ F s(0);
 // CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 1 U
 // CHECK: |-ParenExpr {{.*}} 'bool'
 // CHECK: | `-CXXBoolLiteralExpr {{.*}} 'bool' false
-// CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (type-parameter-0-1) -> F<>'
-// CHECK: | `-ParmVarDecl {{.*}} 'type-parameter-0-1'
+// CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (U) -> F<>'
+// CHECK: | `-ParmVarDecl {{.*}} 'U'
 // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (int) -> F<>'
 // CHECK:   |-TemplateArgument integral ''x''
 // CHECK:   |-TemplateArgument type 'int'
 // CHECK:   | `-BuiltinType {{.*}} 'int'
 // CHECK:   `-ParmVarDecl {{.*}} 'int'
-// CHECK: FunctionProtoType {{.*}} 'auto (type-parameter-0-1) -> F<>' dependent trailing_return cdecl
+// CHECK: FunctionProtoType {{.*}} 'auto (U) -> F<>' dependent trailing_return cdecl
 // CHECK: |-InjectedClassNameType {{.*}} 'F<>' dependent
 // CHECK: | `-CXXRecord {{.*}} 'F'
-// CHECK: `-TemplateTypeParmType {{.*}} 'type-parameter-0-1' dependent depth 0 index 1
+// CHECK: `-TemplateTypeParmType {{.*}} 'U' dependent depth 0 index 1
 
 template<typename T>
 struct G { T t; };
@@ -259,7 +259,7 @@ AG ag = {1};
 // Verify that the aggregate deduction guide for alias templates is built.
 // CHECK-LABEL: Dumping <deduction guide for AG>
 // CHECK: FunctionTemplateDecl
-// CHECK: |-CXXDeductionGuideDecl {{.*}} 'auto (type-parameter-0-0) -> G<type-parameter-0-0>'
+// CHECK: |-CXXDeductionGuideDecl {{.*}} 'auto (X) -> G<X>'
 // CHECK: `-CXXDeductionGuideDecl {{.*}} 'auto (int) -> G<int>' implicit_instantiation
 // CHECK:   |-TemplateArgument type 'int'
 // CHECK:   | `-BuiltinType {{.*}} 'int'
@@ -281,7 +281,7 @@ struct Foo {
 template <typename U>
 using AFoo = Foo<G<U>>;
 // Verify that the require-clause from the Foo deduction guide is transformed.
-// The D occurrence should be rewritten to G<type-parameter-0-0>.
+// The D occurrence should be rewritten to G<U>.
 //
 // CHECK-LABEL: Dumping <deduction guide for AFoo>
 // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for AFoo>
@@ -289,16 +289,16 @@ using AFoo = Foo<G<U>>;
 // CHECK-NEXT: |-BinaryOperator {{.*}} '&&'
 // CHECK-NEXT: | |-ParenExpr {{.*}} 'bool'
 // CHECK-NEXT: | | `-BinaryOperator {{.*}} 'bool' '=='
-// CHECK-NEXT: | |   |-UnaryExprOrTypeTraitExpr {{.*}} 'G<type-parameter-0-0>'
+// CHECK-NEXT: | |   |-UnaryExprOrTypeTraitExpr {{.*}} 'G<U>'
 // CHECK-NEXT: | |   `-ImplicitCastExpr {{.*}}
 // CHECK-NEXT: | |     `-IntegerLiteral {{.*}}
 // CHECK-NEXT: | `-TypeTraitExpr {{.*}} 'bool' __is_deducible
 // CHECK-NEXT: |   |-DeducedTemplateSpecializationType {{.*}} 'AFoo' dependent
 // CHECK-NEXT: |   | `-name: 'AFoo'
 // CHECK-NEXT: |   |   `-TypeAliasTemplateDecl {{.+}} AFoo
-// CHECK-NEXT: |   `-TemplateSpecializationType {{.*}} 'Foo<G<type-parameter-0-0>>' dependent
-// CHECK:      |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for AFoo> 'auto (G<type-parameter-0-0>) -> Foo<G<type-parameter-0-0>>'
-// CHECK-NEXT: | `-ParmVarDecl {{.*}} 'G<type-parameter-0-0>'
+// CHECK-NEXT: |   `-TemplateSpecializationType {{.*}} 'Foo<G<U>>' dependent
+// CHECK:      |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for AFoo> 'auto (G<U>) -> Foo<G<U>>'
+// CHECK-NEXT: | `-ParmVarDecl {{.*}} 'G<U>'
 // CHECK-NEXT: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for AFoo> 'auto (G<int>) -> Foo<G<int>>' implicit_instantiation
 // CHECK-NEXT:   |-TemplateArgument type 'int'
 // CHECK-NEXT:   | `-BuiltinType {{.*}} 'int'
@@ -321,20 +321,20 @@ namespace TTP {
 // CHECK-NEXT:  |-TemplateTypeParmDecl {{.+}} class depth 0 index 0 T{{$}}
 // CHECK-NEXT:  |-TemplateTemplateParmDecl {{.+}} depth 0 index 1 TT{{$}}
 // CHECK-NEXT:  | `-TemplateTypeParmDecl {{.+}} class depth 1 index 0{{$}}
-// CHECK-NEXT:  |-CXXDeductionGuideDecl {{.+}} 'auto (template-parameter-0-1<T>) -> B<T>'{{$}}
-// CHECK-NEXT:  | `-ParmVarDecl {{.+}} 'template-parameter-0-1<T>'{{$}}
+// CHECK-NEXT:  |-CXXDeductionGuideDecl {{.+}} 'auto (TT<T>) -> B<T>'{{$}}
+// CHECK-NEXT:  | `-ParmVarDecl {{.+}} 'TT<T>'{{$}}
 // CHECK-NEXT:  `-CXXDeductionGuideDecl {{.+}} 'auto (A<int>) -> TTP::B<int>'
 // CHECK-NEXT:    |-TemplateArgument type 'int'
 // CHECK-NEXT:    | `-BuiltinType {{.+}} 'int'{{$}}
 // CHECK-NEXT:    |-TemplateArgument template 'TTP::A'{{$}}
 // CHECK-NEXT:    | `-ClassTemplateDecl {{.+}} A{{$}}
 // CHECK-NEXT:    `-ParmVarDecl {{.+}} 'A<int>':'TTP::A<int>'{{$}}
-// CHECK-NEXT:  FunctionProtoType {{.+}} 'auto (template-parameter-0-1<T>) -> B<T>' dependent trailing_return cdecl{{$}}
+// CHECK-NEXT:  FunctionProtoType {{.+}} 'auto (TT<T>) -> B<T>' dependent trailing_return cdecl{{$}}
 // CHECK-NEXT:  |-InjectedClassNameType {{.+}} 'B<T>' dependent{{$}}
 // CHECK-NEXT:  | `-CXXRecord {{.+}} 'B'{{$}}
-// CHECK-NEXT:  `-ElaboratedType {{.+}} 'template-parameter-0-1<T>' sugar dependent{{$}}
-// CHECK-NEXT:    `-TemplateSpecializationType {{.+}} 'template-parameter-0-1<T>' dependent{{$}}
-// CHECK-NEXT:      |-name: 'template-parameter-0-1' qualified
+// CHECK-NEXT:  `-ElaboratedType {{.+}} 'TT<T>' sugar dependent{{$}}
+// CHECK-NEXT:    `-TemplateSpecializationType {{.+}} 'TT<T>' dependent{{$}}
+// CHECK-NEXT:      |-name: 'TT':'template-parameter-0-1' qualified
 // CHECK-NEXT:      | `-TemplateTemplateParmDecl {{.+}} depth 0 index 1
 // CHECK-NEXT:      `-TemplateArgument type 'T':'type-parameter-0-0'{{$}}
 // CHECK-NEXT:        `-TemplateTypeParmType {{.+}} 'T' dependent depth 0 index 0{{$}}

From 20d38158ffbe1e01b74e39d36d4b3bc5e7a63d66 Mon Sep 17 00:00:00 2001
From: Mariya Podchishchaeva <mariya.podchishchaeva@intel.com>
Date: Mon, 22 Jul 2024 10:46:29 +0200
Subject: [PATCH 384/486] [clang] Fix assertion failure in `injectEmbedTokens`
 (#99624)

It seems for C++ lexer has some caching ability which doesn't expect
injecting "new" tokens in the middle of the cache. Technically #embed
tokens are not completely new since they have already been read from the
file and there was #embed annotation token in this place, so set
`reinject` flag for them to silence assertion.
---
 clang/lib/Parse/ParseExpr.cpp           | 2 +-
 clang/test/Preprocessor/embed_weird.cpp | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 3d7c58e5b3c3c..a12c375c8d48c 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -3597,7 +3597,7 @@ void Parser::injectEmbedTokens() {
     I += 2;
   }
   PP.EnterTokenStream(std::move(Toks), /*DisableMacroExpansion=*/true,
-                      /*IsReinject=*/false);
+                      /*IsReinject=*/true);
   ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
 }
 
diff --git a/clang/test/Preprocessor/embed_weird.cpp b/clang/test/Preprocessor/embed_weird.cpp
index 6eb2923152f15..90180e2d3cc70 100644
--- a/clang/test/Preprocessor/embed_weird.cpp
+++ b/clang/test/Preprocessor/embed_weird.cpp
@@ -116,6 +116,14 @@ void f1() {
 }
 #endif
 
+static_assert(_Generic(
+#embed __FILE__ limit(1)
+  , int : 1, default : 0));
+
+static_assert(alignof(typeof(
+#embed __FILE__ limit(1)
+)) == alignof(int));
+
 struct HasChar {
   signed char ch;
 };

From 7eb59a1ef98dd135a88190f9c6bcd113e0a7016f Mon Sep 17 00:00:00 2001
From: Dmitry Chernenkov <dmitryc@google.com>
Date: Mon, 22 Jul 2024 08:59:11 +0000
Subject: [PATCH 385/486] [Bazel] Fix libc Bazel build for
 29be889c2c9c9a92e7ed89bd71d141961517d7e5

---
 .../llvm-project-overlay/libc/BUILD.bazel     | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 63fd3faf38aef..eab5e28511c2c 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1076,6 +1076,22 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_osutil_pid",
+    srcs = ["src/__support/OSUtil/linux/pid.cpp"],
+    hdrs = ["src/__support/OSUtil/pid.h"],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    deps = [
+        ":__support_macros_attributes",
+        ":__support_macros_optimization",
+        ":__support_osutil_syscall",
+        ":types_pid_t",
+    ]
+)
+
 libc_support_library(
     name = "__support_stringutil",
     srcs = glob(["src/__support/StringUtil/tables/**/*.h"]) + [
@@ -3032,8 +3048,11 @@ libc_function(
     hdrs = ["src/unistd/getpid.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_config",
+        ":__support_osutil_pid",
         ":__support_osutil_syscall",
         ":errno",
+        ":types_pid_t",
     ],
 )
 

From 4db11c1f6cd6cd12b51a3220a54697b90e2e8821 Mon Sep 17 00:00:00 2001
From: Max Beck-Jones <max.beck-jones@arm.com>
Date: Mon, 22 Jul 2024 10:14:28 +0100
Subject: [PATCH 386/486] [AArch64] Lower scalable i1 vector add reduction to
 cntp (#99031)

Doing an add reduction on a vector of i1 elements is the same as
counting the number of set elements so such a reduction can be lowered
to a cntp instruction. This saves a number of instructions over
performing a UADDV. This patch only handles straightforward cases (i.e.
when vectors are not split).
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  14 ++
 .../test/CodeGen/AArch64/sve-i1-add-reduce.ll | 132 ++++++++++++++++++
 2 files changed, 146 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/sve-i1-add-reduce.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index bf205b1706a6c..c11855da3fae0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27640,6 +27640,20 @@ SDValue AArch64TargetLowering::LowerReductionToSVE(unsigned Opcode,
     VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
   }
 
+  // Lower VECREDUCE_ADD of nxv2i1-nxv16i1 to CNTP rather than UADDV.
+  if (ScalarOp.getOpcode() == ISD::VECREDUCE_ADD &&
+      VecOp.getOpcode() == ISD::ZERO_EXTEND) {
+    SDValue BoolVec = VecOp.getOperand(0);
+    if (BoolVec.getValueType().getVectorElementType() == MVT::i1) {
+      // CNTP(BoolVec & BoolVec) <=> CNTP(BoolVec & PTRUE)
+      SDValue CntpOp = DAG.getNode(
+          ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64,
+          DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64),
+          BoolVec, BoolVec);
+      return DAG.getAnyExtOrTrunc(CntpOp, DL, ScalarOp.getValueType());
+    }
+  }
+
   // UADDV always returns an i64 result.
   EVT ResVT = (Opcode == AArch64ISD::UADDV_PRED) ? MVT::i64 :
                                                    SrcVT.getVectorElementType();
diff --git a/llvm/test/CodeGen/AArch64/sve-i1-add-reduce.ll b/llvm/test/CodeGen/AArch64/sve-i1-add-reduce.ll
new file mode 100644
index 0000000000000..a748cf732e090
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-i1-add-reduce.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define i8 @uaddv_zexti8_nxv16i1(<vscale x 16 x i1> %v) {
+; CHECK-LABEL: uaddv_zexti8_nxv16i1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntp x0, p0, p0.b
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %3 = zext <vscale x 16 x i1> %v to <vscale x 16 x i8>
+  %4 = tail call i8 @llvm.vector.reduce.add.nxv16i8(<vscale x 16 x i8> %3)
+  ret i8 %4
+}
+
+define i8 @uaddv_zexti8_nxv8i1(<vscale x 8 x i1> %v) {
+; CHECK-LABEL: uaddv_zexti8_nxv8i1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntp x0, p0, p0.h
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %3 = zext <vscale x 8 x i1> %v to <vscale x 8 x i8>
+  %4 = tail call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> %3)
+  ret i8 %4
+}
+
+define i16 @uaddv_zexti16_nxv8i1(<vscale x 8 x i1> %v) {
+; CHECK-LABEL: uaddv_zexti16_nxv8i1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntp x0, p0, p0.h
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %3 = zext <vscale x 8 x i1> %v to <vscale x 8 x i16>
+  %4 = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %3)
+  ret i16 %4
+}
+
+define i8 @uaddv_zexti8_nxv4i1(<vscale x 4 x i1> %v) {
+; CHECK-LABEL: uaddv_zexti8_nxv4i1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntp x0, p0, p0.s
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %3 = zext <vscale x 4 x i1> %v to <vscale x 4 x i8>
+  %4 = tail call i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8> %3)
+  ret i8 %4
+}
+
+define i16 @uaddv_zexti16_nxv4i1(<vscale x 4 x i1> %v) {
+; CHECK-LABEL: uaddv_zexti16_nxv4i1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntp x0, p0, p0.s
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %3 = zext <vscale x 4 x i1> %v to <vscale x 4 x i16>
+  %4 = tail call i16 @llvm.vector.reduce.add.nxv4i16(<vscale x 4 x i16> %3)
+  ret i16 %4
+}
+
+define i32 @uaddv_zexti32_nxv4i1(<vscale x 4 x i1> %v) {
+; CHECK-LABEL: uaddv_zexti32_nxv4i1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntp x0, p0, p0.s
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %3 = zext <vscale x 4 x i1> %v to <vscale x 4 x i32>
+  %4 = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %3)
+  ret i32 %4
+}
+
+define i8 @uaddv_zexti8_nxv2i1(<vscale x 2 x i1> %v) {
+; CHECK-LABEL: uaddv_zexti8_nxv2i1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntp x0, p0, p0.d
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %3 = zext <vscale x 2 x i1> %v to <vscale x 2 x i8>
+  %4 = tail call i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8> %3)
+  ret i8 %4
+}
+
+define i16 @uaddv_zexti16_nxv2i1(<vscale x 2 x i1> %v) {
+; CHECK-LABEL: uaddv_zexti16_nxv2i1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntp x0, p0, p0.d
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %3 = zext <vscale x 2 x i1> %v to <vscale x 2 x i16>
+  %4 = tail call i16 @llvm.vector.reduce.add.nxv2i16(<vscale x 2 x i16> %3)
+  ret i16 %4
+}
+
+define i32 @uaddv_zexti32_nxv2i1(<vscale x 2 x i1> %v) {
+; CHECK-LABEL: uaddv_zexti32_nxv2i1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntp x0, p0, p0.d
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %3 = zext <vscale x 2 x i1> %v to <vscale x 2 x i32>
+  %4 = tail call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> %3)
+  ret i32 %4
+}
+
+define i64 @uaddv_zexti64_nxv2i1(<vscale x 2 x i1> %v) {
+; CHECK-LABEL: uaddv_zexti64_nxv2i1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntp x0, p0, p0.d
+; CHECK-NEXT:    ret
+entry:
+  %3 = zext <vscale x 2 x i1> %v to <vscale x 2 x i64>
+  %4 = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %3)
+  ret i64 %4
+}
+
+declare i8 @llvm.vector.reduce.add.nxv16i8(<vscale x 16 x i8>)
+declare i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8>)
+declare i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16>)
+declare i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8>)
+declare i16 @llvm.vector.reduce.add.nxv4i16(<vscale x 4 x i16>)
+declare i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32>)
+declare i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8>)
+declare i16 @llvm.vector.reduce.add.nxv2i16(<vscale x 2 x i16>)
+declare i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32>)
+declare i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64>)

From f138b33ddd482fc68bc8622a150e279c1449fadf Mon Sep 17 00:00:00 2001
From: Chen Zheng <czhengsz@cn.ibm.com>
Date: Mon, 22 Jul 2024 04:27:18 -0400
Subject: [PATCH 387/486] [XCOFF] refactor the XCOFF BeginSymName handling

Fixes #96810
---
 llvm/include/llvm/MC/MCContext.h              |  2 +-
 llvm/lib/MC/MCContext.cpp                     | 17 +++++---------
 llvm/lib/MC/MCObjectFileInfo.cpp              | 22 +++++++++----------
 llvm/lib/MC/MCSectionXCOFF.cpp                |  2 +-
 llvm/lib/MC/XCOFFObjectWriter.cpp             |  3 +++
 llvm/test/DebugInfo/XCOFF/empty.ll            | 20 ++++++++---------
 llvm/test/DebugInfo/XCOFF/explicit-section.ll | 12 +++++-----
 .../test/DebugInfo/XCOFF/function-sections.ll | 12 +++++-----
 8 files changed, 43 insertions(+), 47 deletions(-)

diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index 73fe6361cb866..4c2be7353cdd6 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -643,7 +643,7 @@ class MCContext {
   MCSectionXCOFF *getXCOFFSection(
       StringRef Section, SectionKind K,
       std::optional<XCOFF::CsectProperties> CsectProp = std::nullopt,
-      bool MultiSymbolsAllowed = false, const char *BeginSymName = nullptr,
+      bool MultiSymbolsAllowed = false,
       std::optional<XCOFF::DwarfSectionSubtypeFlags> DwarfSubtypeFlags =
           std::nullopt);
 
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index 2fe8369016440..44853f3bf43f3 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -795,7 +795,6 @@ bool MCContext::hasXCOFFSection(StringRef Section,
 MCSectionXCOFF *MCContext::getXCOFFSection(
     StringRef Section, SectionKind Kind,
     std::optional<XCOFF::CsectProperties> CsectProp, bool MultiSymbolsAllowed,
-    const char *BeginSymName,
     std::optional<XCOFF::DwarfSectionSubtypeFlags> DwarfSectionSubtypeFlags) {
   bool IsDwarfSec = DwarfSectionSubtypeFlags.has_value();
   assert((IsDwarfSec != CsectProp.has_value()) && "Invalid XCOFF section!");
@@ -825,35 +824,29 @@ MCSectionXCOFF *MCContext::getXCOFFSection(
         CachedName + "[" +
         XCOFF::getMappingClassString(CsectProp->MappingClass) + "]"));
 
-  MCSymbol *Begin = nullptr;
-  if (BeginSymName)
-    Begin = createTempSymbol(BeginSymName, false);
-
   // QualName->getUnqualifiedName() and CachedName are the same except when
   // CachedName contains invalid character(s) such as '$' for an XCOFF symbol.
   MCSectionXCOFF *Result = nullptr;
   if (IsDwarfSec)
     Result = new (XCOFFAllocator.Allocate()) MCSectionXCOFF(
         QualName->getUnqualifiedName(), Kind, QualName,
-        *DwarfSectionSubtypeFlags, Begin, CachedName, MultiSymbolsAllowed);
+        *DwarfSectionSubtypeFlags, QualName, CachedName, MultiSymbolsAllowed);
   else
     Result = new (XCOFFAllocator.Allocate())
         MCSectionXCOFF(QualName->getUnqualifiedName(), CsectProp->MappingClass,
-                       CsectProp->Type, Kind, QualName, Begin, CachedName,
+                       CsectProp->Type, Kind, QualName, nullptr, CachedName,
                        MultiSymbolsAllowed);
 
   Entry.second = Result;
 
   auto *F = allocInitialFragment(*Result);
-  if (Begin)
-    Begin->setFragment(F);
 
   // We might miss calculating the symbols difference as absolute value before
   // adding fixups when symbol_A without the fragment set is the csect itself
   // and symbol_B is in it.
-  // TODO: Currently we only set the fragment for XMC_PR csects because we don't
-  // have other cases that hit this problem yet.
-  if (!IsDwarfSec && CsectProp->MappingClass == XCOFF::XMC_PR)
+  // TODO: Currently we only set the fragment for XMC_PR csects and DWARF
+  // sections because we don't have other cases that hit this problem yet.
+  if (IsDwarfSec || CsectProp->MappingClass == XCOFF::XMC_PR)
     QualName->setFragment(F);
 
   return Result;
diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp
index 7a2b43a954b10..6dadd9752646f 100644
--- a/llvm/lib/MC/MCObjectFileInfo.cpp
+++ b/llvm/lib/MC/MCObjectFileInfo.cpp
@@ -948,52 +948,52 @@ void MCObjectFileInfo::initXCOFFMCObjectFileInfo(const Triple &T) {
   DwarfAbbrevSection = Ctx->getXCOFFSection(
       ".dwabrev", SectionKind::getMetadata(),
       /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwabrev", XCOFF::SSUBTYP_DWABREV);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWABREV);
 
   DwarfInfoSection = Ctx->getXCOFFSection(
       ".dwinfo", SectionKind::getMetadata(), /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwinfo", XCOFF::SSUBTYP_DWINFO);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWINFO);
 
   DwarfLineSection = Ctx->getXCOFFSection(
       ".dwline", SectionKind::getMetadata(), /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwline", XCOFF::SSUBTYP_DWLINE);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWLINE);
 
   DwarfFrameSection = Ctx->getXCOFFSection(
       ".dwframe", SectionKind::getMetadata(),
       /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwframe", XCOFF::SSUBTYP_DWFRAME);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWFRAME);
 
   DwarfPubNamesSection = Ctx->getXCOFFSection(
       ".dwpbnms", SectionKind::getMetadata(),
       /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwpbnms", XCOFF::SSUBTYP_DWPBNMS);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWPBNMS);
 
   DwarfPubTypesSection = Ctx->getXCOFFSection(
       ".dwpbtyp", SectionKind::getMetadata(),
       /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwpbtyp", XCOFF::SSUBTYP_DWPBTYP);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWPBTYP);
 
   DwarfStrSection = Ctx->getXCOFFSection(
       ".dwstr", SectionKind::getMetadata(), /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwstr", XCOFF::SSUBTYP_DWSTR);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWSTR);
 
   DwarfLocSection = Ctx->getXCOFFSection(
       ".dwloc", SectionKind::getMetadata(), /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwloc", XCOFF::SSUBTYP_DWLOC);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWLOC);
 
   DwarfARangesSection = Ctx->getXCOFFSection(
       ".dwarnge", SectionKind::getMetadata(),
       /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwarnge", XCOFF::SSUBTYP_DWARNGE);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWARNGE);
 
   DwarfRangesSection = Ctx->getXCOFFSection(
       ".dwrnges", SectionKind::getMetadata(),
       /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwrnges", XCOFF::SSUBTYP_DWRNGES);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWRNGES);
 
   DwarfMacinfoSection = Ctx->getXCOFFSection(
       ".dwmac", SectionKind::getMetadata(), /* CsectProperties */ std::nullopt,
-      /* MultiSymbolsAllowed */ true, ".dwmac", XCOFF::SSUBTYP_DWMAC);
+      /* MultiSymbolsAllowed */ true, XCOFF::SSUBTYP_DWMAC);
 }
 
 void MCObjectFileInfo::initDXContainerObjectFileInfo(const Triple &T) {
diff --git a/llvm/lib/MC/MCSectionXCOFF.cpp b/llvm/lib/MC/MCSectionXCOFF.cpp
index 00ca5d2067d58..41043b2aeb93f 100644
--- a/llvm/lib/MC/MCSectionXCOFF.cpp
+++ b/llvm/lib/MC/MCSectionXCOFF.cpp
@@ -124,7 +124,7 @@ void MCSectionXCOFF::printSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
   if (getKind().isMetadata() && isDwarfSect()) {
     OS << "\n\t.dwsect " << format("0x%" PRIx32, *getDwarfSubtypeFlags())
        << '\n';
-    OS << MAI.getPrivateLabelPrefix() << getName() << ':' << '\n';
+    OS << getName() << ':' << '\n';
     return;
   }
 
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index ba9894f18b000..c6eb537d1d546 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -596,6 +596,9 @@ void XCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm) {
     const MCSymbolXCOFF *XSym = cast<MCSymbolXCOFF>(&S);
     const MCSectionXCOFF *ContainingCsect = getContainingCsect(XSym);
 
+    if (ContainingCsect->isDwarfSect())
+      continue;
+
     if (XSym->getVisibilityType() != XCOFF::SYM_V_UNSPECIFIED)
       HasVisibility = true;
 
diff --git a/llvm/test/DebugInfo/XCOFF/empty.ll b/llvm/test/DebugInfo/XCOFF/empty.ll
index c1393907169fc..445e1a63447ef 100644
--- a/llvm/test/DebugInfo/XCOFF/empty.ll
+++ b/llvm/test/DebugInfo/XCOFF/empty.ll
@@ -79,7 +79,7 @@ entry:
 ; ASM32-NEXT:                                          # -- End function
 ; ASM32-NEXT:  L..sec_end0:
 ; ASM32:               .dwsect 0x60000
-; ASM32-NEXT:  L...dwabrev:
+; ASM32-NEXT:  .dwabrev:
 ; ASM32-NEXT:          .byte   1                               # Abbreviation Code
 ; ASM32-NEXT:          .byte   17                              # DW_TAG_compile_unit
 ; ASM32-NEXT:          .byte   1                               # DW_CHILDREN_yes
@@ -135,10 +135,10 @@ entry:
 ; ASM32-NEXT:          .byte   0                               # EOM(2)
 ; ASM32-NEXT:          .byte   0                               # EOM(3)
 ; ASM32:               .dwsect 0x10000
-; ASM32-NEXT:  L...dwinfo:
+; ASM32-NEXT:  .dwinfo:
 ; ASM32-NEXT:  L..cu_begin0:
 ; ASM32-NEXT:          .vbyte  2, 4                            # DWARF version number
-; ASM32-NEXT:          .vbyte  4, L...dwabrev                  # Offset Into Abbrev. Section
+; ASM32-NEXT:          .vbyte  4, .dwabrev                  # Offset Into Abbrev. Section
 ; ASM32-NEXT:          .byte   4                               # Address Size (in bytes)
 ; ASM32-NEXT:          .byte   1                               # Abbrev [1] 0xb:0x38 DW_TAG_compile_unit
 ; ASM32-NEXT:          .vbyte  4, L..info_string0              # DW_AT_producer
@@ -166,7 +166,7 @@ entry:
 ; ASM32-NEXT:          .byte   0                               # End Of Children Mark
 ; ASM32-NEXT:  L..debug_info_end0:
 ; ASM32:               .dwsect 0x70000
-; ASM32-NEXT:  L...dwstr:
+; ASM32-NEXT:  .dwstr:
 ; ASM32-NEXT:  L..info_string0:
 ; ASM32-NEXT:          .string "clang version 12.0.0"          # string offset=0
 ; ASM32-NEXT:  L..info_string1:
@@ -179,7 +179,7 @@ entry:
 ; ASM32-NEXT:          .string "int"                           # string offset=36
 ; ASM32-NEXT:          .toc
 ; ASM32:               .dwsect 0x20000
-; ASM32-NEXT:  L...dwline:
+; ASM32-NEXT:  .dwline:
 ; ASM32-NEXT:  L..debug_line_0:
 ; ASM32-NEXT:  .set L..line_table_start0, L..debug_line_0-4
 ; ASM32-NEXT:          .vbyte  2, 4
@@ -281,7 +281,7 @@ entry:
 ; ASM64-NEXT:                                          # -- End function
 ; ASM64-NEXT:  L..sec_end0:
 ; ASM64:               .dwsect 0x60000
-; ASM64-NEXT:  L...dwabrev:
+; ASM64-NEXT:  .dwabrev:
 ; ASM64-NEXT:          .byte   1                               # Abbreviation Code
 ; ASM64-NEXT:          .byte   17                              # DW_TAG_compile_unit
 ; ASM64-NEXT:          .byte   1                               # DW_CHILDREN_yes
@@ -337,10 +337,10 @@ entry:
 ; ASM64-NEXT:          .byte   0                               # EOM(2)
 ; ASM64-NEXT:          .byte   0                               # EOM(3)
 ; ASM64:               .dwsect 0x10000
-; ASM64-NEXT:  L...dwinfo:
+; ASM64-NEXT:  .dwinfo:
 ; ASM64-NEXT:  L..cu_begin0:
 ; ASM64-NEXT:          .vbyte  2, 4                            # DWARF version number
-; ASM64-NEXT:          .vbyte  8, L...dwabrev                  # Offset Into Abbrev. Section
+; ASM64-NEXT:          .vbyte  8, .dwabrev                  # Offset Into Abbrev. Section
 ; ASM64-NEXT:          .byte   8                               # Address Size (in bytes)
 ; ASM64-NEXT:          .byte   1                               # Abbrev [1] 0x17:0x58 DW_TAG_compile_unit
 ; ASM64-NEXT:          .vbyte  8, L..info_string0              # DW_AT_producer
@@ -368,7 +368,7 @@ entry:
 ; ASM64-NEXT:          .byte   0                               # End Of Children Mark
 ; ASM64-NEXT:  L..debug_info_end0:
 ; ASM64:               .dwsect 0x70000
-; ASM64-NEXT:  L...dwstr:
+; ASM64-NEXT:  .dwstr:
 ; ASM64-NEXT:  L..info_string0:
 ; ASM64-NEXT:          .string "clang version 12.0.0"          # string offset=0
 ; ASM64-NEXT:  L..info_string1:
@@ -381,7 +381,7 @@ entry:
 ; ASM64-NEXT:          .string "int"                           # string offset=36
 ; ASM64-NEXT:         .toc
 ; ASM64:               .dwsect 0x20000
-; ASM64-NEXT:  L...dwline:
+; ASM64-NEXT:  .dwline:
 ; ASM64-NEXT:  L..debug_line_0:
 ; ASM64-NEXT:  .set L..line_table_start0, L..debug_line_0-12
 ; ASM64-NEXT:          .vbyte  2, 4
diff --git a/llvm/test/DebugInfo/XCOFF/explicit-section.ll b/llvm/test/DebugInfo/XCOFF/explicit-section.ll
index ed2ffb709168e..530267ec634fc 100644
--- a/llvm/test/DebugInfo/XCOFF/explicit-section.ll
+++ b/llvm/test/DebugInfo/XCOFF/explicit-section.ll
@@ -130,7 +130,7 @@ entry:
 ; CHECK-NEXT:                                          # -- End function
 ; CHECK-NEXT:  L..sec_end0:
 ; CHECK:               .dwsect 0x60000
-; CHECK-NEXT:  L...dwabrev:
+; CHECK-NEXT:  .dwabrev:
 ; CHECK-NEXT:          .byte   1                               # Abbreviation Code
 ; CHECK-NEXT:          .byte   17                              # DW_TAG_compile_unit
 ; CHECK-NEXT:          .byte   1                               # DW_CHILDREN_yes
@@ -184,10 +184,10 @@ entry:
 ; CHECK-NEXT:          .byte   0                               # EOM(2)
 ; CHECK-NEXT:          .byte   0                               # EOM(3)
 ; CHECK:               .dwsect 0x10000
-; CHECK-NEXT:  L...dwinfo:
+; CHECK-NEXT:  .dwinfo:
 ; CHECK-NEXT:  L..cu_begin0:
 ; CHECK-NEXT:          .vbyte  2, 3                            # DWARF version number
-; CHECK-NEXT:          .vbyte  4, L...dwabrev                  # Offset Into Abbrev. Section
+; CHECK-NEXT:          .vbyte  4, .dwabrev                  # Offset Into Abbrev. Section
 ; CHECK-NEXT:          .byte   4                               # Address Size (in bytes)
 ; CHECK-NEXT:          .byte   1                               # Abbrev [1] 0xb:0x4f DW_TAG_compile_unit
 ; CHECK-NEXT:          .vbyte  4, L..info_string0              # DW_AT_producer
@@ -224,7 +224,7 @@ entry:
 ; CHECK-NEXT:          .byte   0                               # End Of Children Mark
 ; CHECK-NEXT:  L..debug_info_end0:
 ; CHECK:               .dwsect 0x80000
-; CHECK-NEXT:  L...dwrnges:
+; CHECK-NEXT:  .dwrnges:
 ; CHECK-NEXT:  L..debug_ranges0:
 ; CHECK-NEXT:          .vbyte  4, L..func_begin0
 ; CHECK-NEXT:          .vbyte  4, L..func_end0
@@ -233,7 +233,7 @@ entry:
 ; CHECK-NEXT:          .vbyte  4, 0
 ; CHECK-NEXT:          .vbyte  4, 0
 ; CHECK:               .dwsect 0x70000
-; CHECK-NEXT:  L...dwstr:
+; CHECK-NEXT:  .dwstr:
 ; CHECK-NEXT:  L..info_string0:
 ; CHECK-NEXT:  	.string	"clang version 13.0.0"          # string offset=0
 ; CHECK-NEXT:  L..info_string1:
@@ -248,7 +248,7 @@ entry:
 ; CHECK-NEXT:  	.string	"main"                          # string offset=39
 ; CHECK-NEXT:          .toc
 ; CHECK:               .dwsect 0x20000
-; CHECK-NEXT:  L...dwline:
+; CHECK-NEXT:  .dwline:
 ; CHECK-NEXT:  L..debug_line_0:
 ; CHECK-NEXT:  .set L..line_table_start0, L..debug_line_0-4
 ; CHECK-NEXT:          .vbyte  2, 3
diff --git a/llvm/test/DebugInfo/XCOFF/function-sections.ll b/llvm/test/DebugInfo/XCOFF/function-sections.ll
index c899089102c64..5103b66db114d 100644
--- a/llvm/test/DebugInfo/XCOFF/function-sections.ll
+++ b/llvm/test/DebugInfo/XCOFF/function-sections.ll
@@ -112,7 +112,7 @@ entry:
 ; CHECK-NEXT:                                          # -- End function
 ; CHECK-NEXT:  L..sec_end0:
 ; CHECK:               .dwsect 0x60000
-; CHECK-NEXT:  L...dwabrev:
+; CHECK-NEXT:  .dwabrev:
 ; CHECK-NEXT:          .byte   1                               # Abbreviation Code
 ; CHECK-NEXT:          .byte   17                              # DW_TAG_compile_unit
 ; CHECK-NEXT:          .byte   1                               # DW_CHILDREN_yes
@@ -168,10 +168,10 @@ entry:
 ; CHECK-NEXT:          .byte   0                               # EOM(2)
 ; CHECK-NEXT:          .byte   0                               # EOM(3)
 ; CHECK:               .dwsect 0x10000
-; CHECK-NEXT:  L...dwinfo:
+; CHECK-NEXT:  .dwinfo:
 ; CHECK-NEXT:  L..cu_begin0:
 ; CHECK-NEXT:          .vbyte  2, 3                            # DWARF version number
-; CHECK-NEXT:          .vbyte  4, L...dwabrev                  # Offset Into Abbrev. Section
+; CHECK-NEXT:          .vbyte  4, .dwabrev                  # Offset Into Abbrev. Section
 ; CHECK-NEXT:          .byte   4                               # Address Size (in bytes)
 ; CHECK-NEXT:          .byte   1                               # Abbrev [1] 0xb:0x51 DW_TAG_compile_unit
 ; CHECK-NEXT:          .vbyte  4, L..info_string0              # DW_AT_producer
@@ -210,7 +210,7 @@ entry:
 ; CHECK-NEXT:          .byte   0                               # End Of Children Mark
 ; CHECK-NEXT:  L..debug_info_end0:
 ; CHECK:               .dwsect 0x80000
-; CHECK-NEXT:  L...dwrnges:
+; CHECK-NEXT:  .dwrnges:
 ; CHECK-NEXT:  L..debug_ranges0:
 ; CHECK-NEXT:          .vbyte  4, L..func_begin0
 ; CHECK-NEXT:          .vbyte  4, L..func_end0
@@ -219,7 +219,7 @@ entry:
 ; CHECK-NEXT:          .vbyte  4, 0
 ; CHECK-NEXT:          .vbyte  4, 0
 ; CHECK:               .dwsect 0x70000
-; CHECK-NEXT:  L...dwstr:
+; CHECK-NEXT:  .dwstr:
 ; CHECK-NEXT:  L..info_string0:
 ; CHECK-NEXT:          .string "clang version 13.0.0"          # string offset=0
 ; CHECK-NEXT:  L..info_string1:
@@ -234,7 +234,7 @@ entry:
 ; CHECK-NEXT:          .string "bar"                           # string offset=39
 ; CHECK-NEXT:          .toc
 ; CHECK:               .dwsect 0x20000
-; CHECK-NEXT:  L...dwline:
+; CHECK-NEXT:  .dwline:
 ; CHECK-NEXT:  L..debug_line_0:
 ; CHECK-NEXT:  .set L..line_table_start0, L..debug_line_0-4
 ; CHECK-NEXT:          .vbyte  2, 3

From 01d783643aa750aa160bdcd097176fcde934dbd1 Mon Sep 17 00:00:00 2001
From: Tim Creech <timothy.m.creech@intel.com>
Date: Mon, 22 Jul 2024 05:22:36 -0400
Subject: [PATCH 388/486] [llvm-profgen] Add --sample-period to estimate
 absolute counts (#99826)

Without `--sample-period`, no assumptions are made about perf profile
sample frequencies. This is useful for comparing relative hotness of
different program locations within the same profile.

With `--sample-period`, LBR- and IP-based profile hit counts are
adjusted to estimate the absolute total event count for each program
location. This makes it reasonable to compare hit counts between
different profiles, e.g., between two LBR-based execution frequency
profiles with different sampling periods or between LBR-based execution
frequency profiles and IP-based branch mispredict profiles.

This functionality is in support of HWPGO[^1], which aims to enable
feedback from a wider range of hardware events.

[^1]:
https://llvm.org/devmtg/2024-04/slides/TechnicalTalks/Xiao-EnablingHW-BasedPGO.pdf
---
 .../tools/llvm-profgen/period-scaling.test    | 84 +++++++++++++++++++
 llvm/tools/llvm-profgen/PerfReader.cpp        | 14 ++++
 2 files changed, 98 insertions(+)
 create mode 100644 llvm/test/tools/llvm-profgen/period-scaling.test

diff --git a/llvm/test/tools/llvm-profgen/period-scaling.test b/llvm/test/tools/llvm-profgen/period-scaling.test
new file mode 100644
index 0000000000000..a44c9a78caeaf
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/period-scaling.test
@@ -0,0 +1,84 @@
+// RUN: llvm-profgen --format=text --perfscript=%S/Inputs/cmov_3.perfscript --binary=%S/Inputs/cmov_3.perfbin --output=%t --skip-symbolization --perf-event=br_inst_retired.near_taken:upp --sample-period=1000003
+// RUN: FileCheck %s --input-file %t --check-prefix=CHECK-RAW-PROFILE
+// RUN: llvm-profgen --format=text --perfscript=%S/Inputs/cmov_3.perfscript --binary=%S/Inputs/cmov_3.perfbin --output=%t --perf-event=br_inst_retired.near_taken:upp --sample-period=1000003
+// RUN: FileCheck %s --input-file %t --check-prefix=CHECK
+
+// RUN: llvm-profgen --format=text --perfscript=%S/Inputs/cmov_3.perfscript --binary=%S/Inputs/cmov_3.perfbin --output=%t --skip-symbolization --perf-event=br_misp_retired.all_branches:upp --leading-ip-only --sample-period=1000003
+// RUN: FileCheck %s --input-file %t --check-prefix=UNPRED-RAW-PROFILE
+// RUN: llvm-profgen --format=text --perfscript=%S/Inputs/cmov_3.perfscript --binary=%S/Inputs/cmov_3.perfbin --output=%t --perf-event=br_misp_retired.all_branches:upp --leading-ip-only --sample-period=1000003
+// RUN: FileCheck %s --input-file %t --check-prefix=UNPRED
+
+// Check that we can use perf event filtering to generate multiple types of
+// source-level profiles from a single perf profile. In this case, we generate
+// a typical execution frequency profile using br_inst_retired.near_taken LBRs,
+// and a branch mispredict profile using br_misp_retired.all_branches sample
+// IPs.
+
+// Check that we can use --sample-period to compute LBR and IP-based profiles
+// which have comparable and absolute magnitudes. For example, in this case the
+// branch of interest (at source line offset 4) is in a loop body which is
+// executed ~20M times in total, and it's mispredicted about 9M times, yielding
+// a mispredict rate of roughly 0.45.
+
+// The source example below is based on perfKernelCpp/cmov_3, except a
+// misleading builtin is used to persuade the compiler not to use cmov, which
+// induces branch mispredicts.
+
+// CHECK: sel_arr:652547082:0
+// CHECK:  3.1: 20225766
+// CHECK:  3.2: 20225766
+// CHECK:  4: 19838670
+// CHECK:  5: 20225766
+
+// UNPRED: sel_arr:18000054:0
+// UNPRED:  3.1: 0
+// UNPRED:  3.2: 0
+// UNPRED:  4: 9000027
+// UNPRED:  5: 0
+
+// CHECK-RAW-PROFILE:      3
+// CHECK-RAW-PROFILE-NEXT: 2f0-2fa:9774174
+// CHECK-RAW-PROFILE-NEXT: 2f0-310:10064496
+// CHECK-RAW-PROFILE-NEXT: 2ff-310:10161270
+
+// UNPRED-RAW-PROFILE:      1
+// UNPRED-RAW-PROFILE-NEXT: 2fa-2fa:9000027
+
+// original code:
+// icx -fprofile-sample-generate lit.c
+#include <stdlib.h>
+
+#define N 20000
+#define ITERS 10000
+
+static int *m_s1, *m_s2, *m_s3, *m_dst;
+
+void init(void) {
+    m_s1 = malloc(sizeof(int)*N);
+    m_s2 = malloc(sizeof(int)*N);
+    m_s3 = malloc(sizeof(int)*N);
+    m_dst = malloc(sizeof(int)*N);
+    srand(42);
+
+    for (int i = 0; i < N; i++) {
+        m_s1[i] = rand() % N;
+        m_s2[i] = 0;
+        m_s3[i] = 1;
+    }
+}
+
+void __attribute__((noinline)) sel_arr(int *dst, int *s1, int *s2, int *s3) {
+#pragma nounroll
+#pragma clang loop vectorize(disable) interleave(disable)
+    for (int i = 0; i < N; i++) {
+        int *p = __builtin_expect((s1[i] < 10035), 0) ? &s2[i] : &s3[i];
+        dst[i] = *p;
+    }
+}
+
+int main(void) {
+  init();
+  for(int i=0; i<ITERS; ++i)
+    sel_arr(m_dst, m_s1, m_s2, m_s3);
+  return 0;
+}
diff --git a/llvm/tools/llvm-profgen/PerfReader.cpp b/llvm/tools/llvm-profgen/PerfReader.cpp
index b4e4911fb8912..4041271cc0a82 100644
--- a/llvm/tools/llvm-profgen/PerfReader.cpp
+++ b/llvm/tools/llvm-profgen/PerfReader.cpp
@@ -53,6 +53,10 @@ static cl::alias
                           cl::desc("Comma-delimited version of -perf-event"),
                           cl::aliasopt(PerfEventFilter));
 
+static cl::opt<uint64_t>
+    SamplePeriod("sample-period", cl::init(1),
+                 cl::desc("The sampling period (-c) used for perf data"));
+
 extern cl::opt<std::string> PerfTraceFilename;
 extern cl::opt<bool> ShowDisassemblyOnly;
 extern cl::opt<bool> ShowSourceLocations;
@@ -1000,6 +1004,16 @@ void LBRPerfReader::parseSample(TraceStream &TraceIt, uint64_t Count) {
   if (extractLBRStack(TraceIt, Sample->LBRStack)) {
     warnIfMissingMMap();
     // Record LBR only samples by aggregation
+    // If a sampling period is given we can adjust the magnitude of sample
+    // counts to estimate the absolute magnitute.
+    if (SamplePeriod.getNumOccurrences()) {
+      Count *= SamplePeriod;
+      // If counts are LBR-based, as opposed to IP-based, then the magnitude is
+      // now amplified by roughly the LBR stack size. By adjusting this down, we
+      // can produce LBR-based and IP-based profiles with comparable magnitudes.
+      if (!LeadingIPOnly && Sample->LBRStack.size() > 1)
+        Count /= (Sample->LBRStack.size() - 1);
+    }
     AggregatedSamples[Hashable<PerfSample>(Sample)] += Count;
   }
 }

From 2e789900433834b3b5cd6a55ccf64da6a9ec4505 Mon Sep 17 00:00:00 2001
From: Max Beck-Jones <max.beck-jones@arm.com>
Date: Mon, 22 Jul 2024 10:28:18 +0100
Subject: [PATCH 389/486] Revert "[AArch64] Lower scalable i1 vector add
 reduction to cntp" (#99853)

Reverts llvm/llvm-project#99031
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  14 --
 .../test/CodeGen/AArch64/sve-i1-add-reduce.ll | 132 ------------------
 2 files changed, 146 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/sve-i1-add-reduce.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c11855da3fae0..bf205b1706a6c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27640,20 +27640,6 @@ SDValue AArch64TargetLowering::LowerReductionToSVE(unsigned Opcode,
     VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
   }
 
-  // Lower VECREDUCE_ADD of nxv2i1-nxv16i1 to CNTP rather than UADDV.
-  if (ScalarOp.getOpcode() == ISD::VECREDUCE_ADD &&
-      VecOp.getOpcode() == ISD::ZERO_EXTEND) {
-    SDValue BoolVec = VecOp.getOperand(0);
-    if (BoolVec.getValueType().getVectorElementType() == MVT::i1) {
-      // CNTP(BoolVec & BoolVec) <=> CNTP(BoolVec & PTRUE)
-      SDValue CntpOp = DAG.getNode(
-          ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64,
-          DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64),
-          BoolVec, BoolVec);
-      return DAG.getAnyExtOrTrunc(CntpOp, DL, ScalarOp.getValueType());
-    }
-  }
-
   // UADDV always returns an i64 result.
   EVT ResVT = (Opcode == AArch64ISD::UADDV_PRED) ? MVT::i64 :
                                                    SrcVT.getVectorElementType();
diff --git a/llvm/test/CodeGen/AArch64/sve-i1-add-reduce.ll b/llvm/test/CodeGen/AArch64/sve-i1-add-reduce.ll
deleted file mode 100644
index a748cf732e090..0000000000000
--- a/llvm/test/CodeGen/AArch64/sve-i1-add-reduce.ll
+++ /dev/null
@@ -1,132 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
-
-define i8 @uaddv_zexti8_nxv16i1(<vscale x 16 x i1> %v) {
-; CHECK-LABEL: uaddv_zexti8_nxv16i1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cntp x0, p0, p0.b
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-entry:
-  %3 = zext <vscale x 16 x i1> %v to <vscale x 16 x i8>
-  %4 = tail call i8 @llvm.vector.reduce.add.nxv16i8(<vscale x 16 x i8> %3)
-  ret i8 %4
-}
-
-define i8 @uaddv_zexti8_nxv8i1(<vscale x 8 x i1> %v) {
-; CHECK-LABEL: uaddv_zexti8_nxv8i1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cntp x0, p0, p0.h
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-entry:
-  %3 = zext <vscale x 8 x i1> %v to <vscale x 8 x i8>
-  %4 = tail call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> %3)
-  ret i8 %4
-}
-
-define i16 @uaddv_zexti16_nxv8i1(<vscale x 8 x i1> %v) {
-; CHECK-LABEL: uaddv_zexti16_nxv8i1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cntp x0, p0, p0.h
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-entry:
-  %3 = zext <vscale x 8 x i1> %v to <vscale x 8 x i16>
-  %4 = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %3)
-  ret i16 %4
-}
-
-define i8 @uaddv_zexti8_nxv4i1(<vscale x 4 x i1> %v) {
-; CHECK-LABEL: uaddv_zexti8_nxv4i1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cntp x0, p0, p0.s
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-entry:
-  %3 = zext <vscale x 4 x i1> %v to <vscale x 4 x i8>
-  %4 = tail call i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8> %3)
-  ret i8 %4
-}
-
-define i16 @uaddv_zexti16_nxv4i1(<vscale x 4 x i1> %v) {
-; CHECK-LABEL: uaddv_zexti16_nxv4i1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cntp x0, p0, p0.s
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-entry:
-  %3 = zext <vscale x 4 x i1> %v to <vscale x 4 x i16>
-  %4 = tail call i16 @llvm.vector.reduce.add.nxv4i16(<vscale x 4 x i16> %3)
-  ret i16 %4
-}
-
-define i32 @uaddv_zexti32_nxv4i1(<vscale x 4 x i1> %v) {
-; CHECK-LABEL: uaddv_zexti32_nxv4i1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cntp x0, p0, p0.s
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-entry:
-  %3 = zext <vscale x 4 x i1> %v to <vscale x 4 x i32>
-  %4 = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %3)
-  ret i32 %4
-}
-
-define i8 @uaddv_zexti8_nxv2i1(<vscale x 2 x i1> %v) {
-; CHECK-LABEL: uaddv_zexti8_nxv2i1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cntp x0, p0, p0.d
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-entry:
-  %3 = zext <vscale x 2 x i1> %v to <vscale x 2 x i8>
-  %4 = tail call i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8> %3)
-  ret i8 %4
-}
-
-define i16 @uaddv_zexti16_nxv2i1(<vscale x 2 x i1> %v) {
-; CHECK-LABEL: uaddv_zexti16_nxv2i1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cntp x0, p0, p0.d
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-entry:
-  %3 = zext <vscale x 2 x i1> %v to <vscale x 2 x i16>
-  %4 = tail call i16 @llvm.vector.reduce.add.nxv2i16(<vscale x 2 x i16> %3)
-  ret i16 %4
-}
-
-define i32 @uaddv_zexti32_nxv2i1(<vscale x 2 x i1> %v) {
-; CHECK-LABEL: uaddv_zexti32_nxv2i1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cntp x0, p0, p0.d
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-entry:
-  %3 = zext <vscale x 2 x i1> %v to <vscale x 2 x i32>
-  %4 = tail call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> %3)
-  ret i32 %4
-}
-
-define i64 @uaddv_zexti64_nxv2i1(<vscale x 2 x i1> %v) {
-; CHECK-LABEL: uaddv_zexti64_nxv2i1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cntp x0, p0, p0.d
-; CHECK-NEXT:    ret
-entry:
-  %3 = zext <vscale x 2 x i1> %v to <vscale x 2 x i64>
-  %4 = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %3)
-  ret i64 %4
-}
-
-declare i8 @llvm.vector.reduce.add.nxv16i8(<vscale x 16 x i8>)
-declare i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8>)
-declare i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16>)
-declare i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8>)
-declare i16 @llvm.vector.reduce.add.nxv4i16(<vscale x 4 x i16>)
-declare i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32>)
-declare i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8>)
-declare i16 @llvm.vector.reduce.add.nxv2i16(<vscale x 2 x i16>)
-declare i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32>)
-declare i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64>)

From 2bc38dc30bc9baad610925d7e90e724a9d09ee7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Don=C3=A1t=20Nagy?= <donat.nagy@ericsson.com>
Date: Mon, 22 Jul 2024 11:44:20 +0200
Subject: [PATCH 390/486] [analyzer] Improve bug report hashing, merge similar
 reports (#98621)

Previously there were certain situations where
alpha.security.ArrayBoundV2 produced lots of very similar and redundant
reports that only differed in their full `Description` that contained
the (negative) byte offset value. (See
https://github.com/llvm/llvm-project/issues/86969 for details.)

This change updates the `Profile()` method of `PathSensitiveBugReport`
to ensure that it uses `getShortDescription()` instead of the full
`Description` so the standard report deduplication eliminates most of
these redundant reports.

Note that the effects of this change are very limited because there are
very few checkers that specify a separate short description, and so
`getShortDescription()` practically always defaults to returning the
full `Description`.

For the sake of consistency `BasicBugReport::Profile()` is also updated
to use the short description. (Right now there are no checkers that use
`BasicBugReport` with separate long and short descriptions.)

This commit also includes some small code quality improvements in
`ArrayBoundV2` that are IMO too trivial to be moved into a separate
commit.
---
 .../Checkers/ArrayBoundCheckerV2.cpp          | 18 ++++++++--------
 clang/lib/StaticAnalyzer/Core/BugReporter.cpp |  4 ++--
 .../test/Analysis/out-of-bounds-diagnostics.c | 21 +++++++++++++++++++
 3 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
index f82288f1099e8..3f837564cf47c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
@@ -373,14 +373,14 @@ static std::optional<int64_t> getConcreteValue(std::optional<NonLoc> SV) {
 }
 
 static Messages getPrecedesMsgs(const SubRegion *Region, NonLoc Offset) {
-  std::string RegName = getRegionName(Region);
-  SmallString<128> Buf;
-  llvm::raw_svector_ostream Out(Buf);
-  Out << "Access of " << RegName << " at negative byte offset";
-  if (auto ConcreteIdx = Offset.getAs<nonloc::ConcreteInt>())
-    Out << ' ' << ConcreteIdx->getValue();
-  return {formatv("Out of bound access to memory preceding {0}", RegName),
-          std::string(Buf)};
+  std::string RegName = getRegionName(Region), OffsetStr = "";
+
+  if (auto ConcreteOffset = getConcreteValue(Offset))
+    OffsetStr = formatv(" {0}", ConcreteOffset);
+
+  return {
+      formatv("Out of bound access to memory preceding {0}", RegName),
+      formatv("Access of {0} at negative byte offset{1}", RegName, OffsetStr)};
 }
 
 /// Try to divide `Val1` and `Val2` (in place) by `Divisor` and return true if
@@ -609,7 +609,7 @@ void ArrayBoundCheckerV2::performCheck(const Expr *E, CheckerContext &C) const {
   // CHECK UPPER BOUND
   DefinedOrUnknownSVal Size = getDynamicExtent(State, Reg, SVB);
   if (auto KnownSize = Size.getAs<NonLoc>()) {
-    // In a situation where both overflow and overflow are possible (but the
+    // In a situation where both underflow and overflow are possible (but the
     // index is either tainted or known to be invalid), the logic of this
     // checker will first assume that the offset is non-negative, and then
     // (with this additional assumption) it will detect an overflow error.
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index e2002bfbe594a..d73dc40cf03fb 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -2198,7 +2198,7 @@ const Decl *PathSensitiveBugReport::getDeclWithIssue() const {
 void BasicBugReport::Profile(llvm::FoldingSetNodeID& hash) const {
   hash.AddInteger(static_cast<int>(getKind()));
   hash.AddPointer(&BT);
-  hash.AddString(Description);
+  hash.AddString(getShortDescription());
   assert(Location.isValid());
   Location.Profile(hash);
 
@@ -2213,7 +2213,7 @@ void BasicBugReport::Profile(llvm::FoldingSetNodeID& hash) const {
 void PathSensitiveBugReport::Profile(llvm::FoldingSetNodeID &hash) const {
   hash.AddInteger(static_cast<int>(getKind()));
   hash.AddPointer(&BT);
-  hash.AddString(Description);
+  hash.AddString(getShortDescription());
   PathDiagnosticLocation UL = getUniqueingLocation();
   if (UL.isValid()) {
     UL.Profile(hash);
diff --git a/clang/test/Analysis/out-of-bounds-diagnostics.c b/clang/test/Analysis/out-of-bounds-diagnostics.c
index 92f983d8b1561..de70e483add1c 100644
--- a/clang/test/Analysis/out-of-bounds-diagnostics.c
+++ b/clang/test/Analysis/out-of-bounds-diagnostics.c
@@ -17,6 +17,27 @@ int underflowWithDeref(void) {
   // expected-note@-2 {{Access of 'TenElements' at negative byte offset -4}}
 }
 
+int rng(void);
+int getIndex(void) {
+  switch (rng()) {
+    case 1: return -152;
+    case 2: return -160;
+    case 3: return -168;
+    default: return -172;
+  }
+}
+
+void gh86959(void) {
+  // Previously code like this produced many almost-identical bug reports that
+  // only differed in the offset value. Verify that now we only see one report.
+
+  // expected-note@+1 {{Entering loop body}}
+  while (rng())
+    TenElements[getIndex()] = 10;
+  // expected-warning@-1 {{Out of bound access to memory preceding 'TenElements'}}
+  // expected-note@-2 {{Access of 'TenElements' at negative byte offset -688}}
+}
+
 int scanf(const char *restrict fmt, ...);
 
 void taintedIndex(void) {

From 16e05919d94888e222d4954086f6d860839117dd Mon Sep 17 00:00:00 2001
From: AtariDreams <gfunni234@gmail.com>
Date: Mon, 22 Jul 2024 06:04:08 -0400
Subject: [PATCH 391/486] [AArch64] Use isKnownNonZero to optimize eligible
 compares to cmn and ccmn (#96349)

The problematic case for unsigned comparisons occurs only when the
second argument is zero.

The problematic case for signed comparisons occurs only when the second
argument is the signed minimum value.

We can use KnownBits to know when we don't have to worry about this.

Source: https://devblogs.microsoft.com/oldnewthing/20210607-00/?p=105288
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  44 ++--
 llvm/test/CodeGen/AArch64/cmp-chains.ll       | 243 ++++++++++++++++++
 llvm/test/CodeGen/AArch64/cmp-select-sign.ll  | 206 +++++++++++++++
 3 files changed, 476 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index bf205b1706a6c..eae0200f37f04 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -3403,6 +3403,11 @@ static bool isLegalArithImmed(uint64_t C) {
   return IsLegal;
 }
 
+static bool cannotBeIntMin(SDValue CheckedVal, SelectionDAG &DAG) {
+  KnownBits KnownSrc = DAG.computeKnownBits(CheckedVal);
+  return !KnownSrc.getSignedMinValue().isMinSignedValue();
+}
+
 // Can a (CMP op1, (sub 0, op2) be turned into a CMN instruction on
 // the grounds that "op1 - (-op2) == op1 + op2" ? Not always, the C and V flags
 // can be set differently by this operation. It comes down to whether
@@ -3410,12 +3415,14 @@ static bool isLegalArithImmed(uint64_t C) {
 // everything is fine. If not then the optimization is wrong. Thus general
 // comparisons are only valid if op2 != 0.
 //
-// So, finally, the only LLVM-native comparisons that don't mention C and V
-// are SETEQ and SETNE. They're the only ones we can safely use CMN for in
-// the absence of information about op2.
-static bool isCMN(SDValue Op, ISD::CondCode CC) {
+// So, finally, the only LLVM-native comparisons that don't mention C or V
+// are the ones that aren't unsigned comparisons. They're the only ones we can
+// safely use CMN for in the absence of information about op2.
+static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG) {
   return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
-         (CC == ISD::SETEQ || CC == ISD::SETNE);
+         (isIntEqualitySetCC(CC) ||
+          (isUnsignedIntSetCC(CC) && DAG.isKnownNeverZero(Op.getOperand(1))) ||
+          (isSignedIntSetCC(CC) && cannotBeIntMin(Op.getOperand(1), DAG)));
 }
 
 static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
@@ -3460,11 +3467,12 @@ static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
   // register to WZR/XZR if it ends up being unused.
   unsigned Opcode = AArch64ISD::SUBS;
 
-  if (isCMN(RHS, CC)) {
+  if (isCMN(RHS, CC, DAG)) {
     // Can we combine a (CMP op1, (sub 0, op2) into a CMN instruction ?
     Opcode = AArch64ISD::ADDS;
     RHS = RHS.getOperand(1);
-  } else if (isCMN(LHS, CC)) {
+  } else if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) &&
+             isIntEqualitySetCC(CC)) {
     // As we are looking for EQ/NE compares, the operands can be commuted ; can
     // we combine a (CMP (sub 0, op1), op2) into a CMN instruction ?
     Opcode = AArch64ISD::ADDS;
@@ -3566,13 +3574,15 @@ static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
       Opcode = AArch64ISD::CCMN;
       RHS = DAG.getConstant(Imm.abs(), DL, Const->getValueType(0));
     }
-  } else if (RHS.getOpcode() == ISD::SUB) {
-    SDValue SubOp0 = RHS.getOperand(0);
-    if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
-      // See emitComparison() on why we can only do this for SETEQ and SETNE.
-      Opcode = AArch64ISD::CCMN;
-      RHS = RHS.getOperand(1);
-    }
+  } else if (isCMN(RHS, CC, DAG)) {
+    Opcode = AArch64ISD::CCMN;
+    RHS = RHS.getOperand(1);
+  } else if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) &&
+             isIntEqualitySetCC(CC)) {
+    // As we are looking for EQ/NE compares, the operands can be commuted ; can
+    // we combine a (CCMP (sub 0, op1), op2) into a CCMN instruction ?
+    Opcode = AArch64ISD::CCMN;
+    LHS = LHS.getOperand(1);
   }
   if (Opcode == 0)
     Opcode = AArch64ISD::CCMP;
@@ -3890,8 +3900,8 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
   //    cmp     w12, w11, lsl #1
   if (!isa<ConstantSDNode>(RHS) ||
       !isLegalArithImmed(RHS->getAsAPIntVal().abs().getZExtValue())) {
-    bool LHSIsCMN = isCMN(LHS, CC);
-    bool RHSIsCMN = isCMN(RHS, CC);
+    bool LHSIsCMN = isCMN(LHS, CC, DAG);
+    bool RHSIsCMN = isCMN(RHS, CC, DAG);
     SDValue TheLHS = LHSIsCMN ? LHS.getOperand(1) : LHS;
     SDValue TheRHS = RHSIsCMN ? RHS.getOperand(1) : RHS;
 
@@ -3904,7 +3914,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
 
   SDValue Cmp;
   AArch64CC::CondCode AArch64CC;
-  if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) {
+  if (isIntEqualitySetCC(CC) && isa<ConstantSDNode>(RHS)) {
     const ConstantSDNode *RHSC = cast<ConstantSDNode>(RHS);
 
     // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095.
diff --git a/llvm/test/CodeGen/AArch64/cmp-chains.ll b/llvm/test/CodeGen/AArch64/cmp-chains.ll
index 14cb0c82b1c03..4b816df75a730 100644
--- a/llvm/test/CodeGen/AArch64/cmp-chains.ll
+++ b/llvm/test/CodeGen/AArch64/cmp-chains.ll
@@ -258,3 +258,246 @@ define i32 @neg_range_int(i32 %a, i32 %b, i32 %c) {
   ret i32 %retval.0
 }
 
+; (b > -(d | 1) && a < c)
+define i32 @neg_range_int_comp(i32 %a, i32 %b, i32 %c, i32 %d) {
+; SDISEL-LABEL: neg_range_int_comp:
+; SDISEL:       // %bb.0:
+; SDISEL-NEXT:    orr w8, w3, #0x1
+; SDISEL-NEXT:    cmp w0, w2
+; SDISEL-NEXT:    ccmn w1, w8, #4, lt
+; SDISEL-NEXT:    csel w0, w1, w0, gt
+; SDISEL-NEXT:    ret
+;
+; GISEL-LABEL: neg_range_int_comp:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    orr w8, w3, #0x1
+; GISEL-NEXT:    cmp w0, w2
+; GISEL-NEXT:    neg w8, w8
+; GISEL-NEXT:    ccmp w1, w8, #4, lt
+; GISEL-NEXT:    csel w0, w1, w0, gt
+; GISEL-NEXT:    ret
+  %dor = or i32 %d, 1
+  %negd = sub i32 0, %dor
+  %cmp = icmp sgt i32 %b, %negd
+  %cmp1 = icmp slt i32 %a, %c
+  %or.cond = and i1 %cmp, %cmp1
+  %retval.0 = select i1 %or.cond, i32 %b, i32 %a
+  ret i32 %retval.0
+}
+
+; (b >u -(d | 1) && a < c)
+define i32 @neg_range_int_comp_u(i32 %a, i32 %b, i32 %c, i32 %d) {
+; SDISEL-LABEL: neg_range_int_comp_u:
+; SDISEL:       // %bb.0:
+; SDISEL-NEXT:    orr w8, w3, #0x1
+; SDISEL-NEXT:    cmp w0, w2
+; SDISEL-NEXT:    ccmn w1, w8, #0, lt
+; SDISEL-NEXT:    csel w0, w1, w0, hi
+; SDISEL-NEXT:    ret
+;
+; GISEL-LABEL: neg_range_int_comp_u:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    orr w8, w3, #0x1
+; GISEL-NEXT:    cmp w0, w2
+; GISEL-NEXT:    neg w8, w8
+; GISEL-NEXT:    ccmp w1, w8, #0, lt
+; GISEL-NEXT:    csel w0, w1, w0, hi
+; GISEL-NEXT:    ret
+  %dor = or i32 %d, 1
+  %negd = sub i32 0, %dor
+  %cmp = icmp ugt i32 %b, %negd
+  %cmp1 = icmp slt i32 %a, %c
+  %or.cond = and i1 %cmp, %cmp1
+  %retval.0 = select i1 %or.cond, i32 %b, i32 %a
+  ret i32 %retval.0
+}
+
+; (b > -(d | 1) && a u < c)
+define i32 @neg_range_int_comp_ua(i32 %a, i32 %b, i32 %c, i32 %d) {
+; SDISEL-LABEL: neg_range_int_comp_ua:
+; SDISEL:       // %bb.0:
+; SDISEL-NEXT:    orr w8, w3, #0x1
+; SDISEL-NEXT:    cmp w0, w2
+; SDISEL-NEXT:    ccmn w1, w8, #4, lo
+; SDISEL-NEXT:    csel w0, w1, w0, gt
+; SDISEL-NEXT:    ret
+;
+; GISEL-LABEL: neg_range_int_comp_ua:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    orr w8, w3, #0x1
+; GISEL-NEXT:    cmp w0, w2
+; GISEL-NEXT:    neg w8, w8
+; GISEL-NEXT:    ccmp w1, w8, #4, lo
+; GISEL-NEXT:    csel w0, w1, w0, gt
+; GISEL-NEXT:    ret
+  %dor = or i32 %d, 1
+  %negd = sub i32 0, %dor
+  %cmp = icmp sgt i32 %b, %negd
+  %cmp1 = icmp ult i32 %a, %c
+  %or.cond = and i1 %cmp, %cmp1
+  %retval.0 = select i1 %or.cond, i32 %b, i32 %a
+  ret i32 %retval.0
+}
+
+; (b <= -3 && a > c)
+define i32 @neg_range_int_2(i32 %a, i32 %b, i32 %c) {
+; SDISEL-LABEL: neg_range_int_2:
+; SDISEL:       // %bb.0:
+; SDISEL-NEXT:    cmp w0, w2
+; SDISEL-NEXT:    ccmn w1, #4, #4, gt
+; SDISEL-NEXT:    csel w0, w1, w0, gt
+; SDISEL-NEXT:    ret
+;
+; GISEL-LABEL: neg_range_int_2:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmp w0, w2
+; GISEL-NEXT:    ccmn w1, #3, #8, gt
+; GISEL-NEXT:    csel w0, w1, w0, ge
+; GISEL-NEXT:    ret
+  %cmp = icmp sge i32 %b, -3
+  %cmp1 = icmp sgt i32 %a, %c
+  %or.cond = and i1 %cmp, %cmp1
+  %retval.0 = select i1 %or.cond, i32 %b, i32 %a
+  ret i32 %retval.0
+}
+
+; (b < -(d | 1) && a >= c)
+define i32 @neg_range_int_comp2(i32 %a, i32 %b, i32 %c, i32 %d) {
+; SDISEL-LABEL: neg_range_int_comp2:
+; SDISEL:       // %bb.0:
+; SDISEL-NEXT:    orr w8, w3, #0x1
+; SDISEL-NEXT:    cmp w0, w2
+; SDISEL-NEXT:    ccmn w1, w8, #0, ge
+; SDISEL-NEXT:    csel w0, w1, w0, lt
+; SDISEL-NEXT:    ret
+;
+; GISEL-LABEL: neg_range_int_comp2:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    orr w8, w3, #0x1
+; GISEL-NEXT:    cmp w0, w2
+; GISEL-NEXT:    neg w8, w8
+; GISEL-NEXT:    ccmp w1, w8, #0, ge
+; GISEL-NEXT:    csel w0, w1, w0, lt
+; GISEL-NEXT:    ret
+  %dor = or i32 %d, 1
+  %negd = sub i32 0, %dor
+  %cmp = icmp slt i32 %b, %negd
+  %cmp1 = icmp sge i32 %a, %c
+  %or.cond = and i1 %cmp, %cmp1
+  %retval.0 = select i1 %or.cond, i32 %b, i32 %a
+  ret i32 %retval.0
+}
+
+; (b <u -(d | 1) && a > c)
+define i32 @neg_range_int_comp_u2(i32 %a, i32 %b, i32 %c, i32 %d) {
+; SDISEL-LABEL: neg_range_int_comp_u2:
+; SDISEL:       // %bb.0:
+; SDISEL-NEXT:    orr w8, w3, #0x1
+; SDISEL-NEXT:    cmp w0, w2
+; SDISEL-NEXT:    ccmn w1, w8, #2, gt
+; SDISEL-NEXT:    csel w0, w1, w0, lo
+; SDISEL-NEXT:    ret
+;
+; GISEL-LABEL: neg_range_int_comp_u2:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    orr w8, w3, #0x1
+; GISEL-NEXT:    cmp w0, w2
+; GISEL-NEXT:    neg w8, w8
+; GISEL-NEXT:    ccmp w1, w8, #2, gt
+; GISEL-NEXT:    csel w0, w1, w0, lo
+; GISEL-NEXT:    ret
+  %dor = or i32 %d, 1
+  %negd = sub i32 0, %dor
+  %cmp = icmp ult i32 %b, %negd
+  %cmp1 = icmp sgt i32 %a, %c
+  %or.cond = and i1 %cmp, %cmp1
+  %retval.0 = select i1 %or.cond, i32 %b, i32 %a
+  ret i32 %retval.0
+}
+
+; (b > -(d | 1) && a u > c)
+define i32 @neg_range_int_comp_ua2(i32 %a, i32 %b, i32 %c, i32 %d) {
+; SDISEL-LABEL: neg_range_int_comp_ua2:
+; SDISEL:       // %bb.0:
+; SDISEL-NEXT:    orr w8, w3, #0x1
+; SDISEL-NEXT:    cmp w0, w2
+; SDISEL-NEXT:    ccmn w1, w8, #4, hi
+; SDISEL-NEXT:    csel w0, w1, w0, gt
+; SDISEL-NEXT:    ret
+;
+; GISEL-LABEL: neg_range_int_comp_ua2:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    orr w8, w3, #0x1
+; GISEL-NEXT:    cmp w0, w2
+; GISEL-NEXT:    neg w8, w8
+; GISEL-NEXT:    ccmp w1, w8, #4, hi
+; GISEL-NEXT:    csel w0, w1, w0, gt
+; GISEL-NEXT:    ret
+  %dor = or i32 %d, 1
+  %negd = sub i32 0, %dor
+  %cmp = icmp sgt i32 %b, %negd
+  %cmp1 = icmp ugt i32 %a, %c
+  %or.cond = and i1 %cmp, %cmp1
+  %retval.0 = select i1 %or.cond, i32 %b, i32 %a
+  ret i32 %retval.0
+}
+
+; (b > -(d | 1) && a u == c)
+define i32 @neg_range_int_comp_ua3(i32 %a, i32 %b, i32 %c, i32 %d) {
+; SDISEL-LABEL: neg_range_int_comp_ua3:
+; SDISEL:       // %bb.0:
+; SDISEL-NEXT:    orr w8, w3, #0x1
+; SDISEL-NEXT:    cmp w0, w2
+; SDISEL-NEXT:    ccmn w1, w8, #4, eq
+; SDISEL-NEXT:    csel w0, w1, w0, gt
+; SDISEL-NEXT:    ret
+;
+; GISEL-LABEL: neg_range_int_comp_ua3:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    orr w8, w3, #0x1
+; GISEL-NEXT:    cmp w0, w2
+; GISEL-NEXT:    neg w8, w8
+; GISEL-NEXT:    ccmp w1, w8, #4, eq
+; GISEL-NEXT:    csel w0, w1, w0, gt
+; GISEL-NEXT:    ret
+  %dor = or i32 %d, 1
+  %negd = sub i32 0, %dor
+  %cmp = icmp sgt i32 %b, %negd
+  %cmp1 = icmp eq i32 %a, %c
+  %or.cond = and i1 %cmp, %cmp1
+  %retval.0 = select i1 %or.cond, i32 %b, i32 %a
+  ret i32 %retval.0
+}
+
+; -(a | 1) > (b | 3) && a < c
+define i32 @neg_range_int_c(i32 %a, i32 %b, i32 %c) {
+; SDISEL-LABEL: neg_range_int_c:
+; SDISEL:       // %bb.0: // %entry
+; SDISEL-NEXT:    orr w8, w0, #0x1
+; SDISEL-NEXT:    orr w9, w1, #0x3
+; SDISEL-NEXT:    cmn w9, w8
+; SDISEL-NEXT:    ccmp w2, w0, #2, lo
+; SDISEL-NEXT:    cset w0, lo
+; SDISEL-NEXT:    ret
+;
+; GISEL-LABEL: neg_range_int_c:
+; GISEL:       // %bb.0: // %entry
+; GISEL-NEXT:    orr w8, w0, #0x1
+; GISEL-NEXT:    orr w9, w1, #0x3
+; GISEL-NEXT:    neg w8, w8
+; GISEL-NEXT:    cmp w9, w8
+; GISEL-NEXT:    cset w8, lo
+; GISEL-NEXT:    cmp w2, w0
+; GISEL-NEXT:    cset w9, lo
+; GISEL-NEXT:    and w0, w8, w9
+; GISEL-NEXT:    ret
+entry:
+  %or = or i32 %a, 1
+  %sub = sub i32 0, %or
+  %or1 = or i32 %b, 3
+  %cmp = icmp ult i32 %or1, %sub
+  %cmp2 = icmp ult i32 %c, %a
+  %0 = and i1 %cmp, %cmp2
+  %land.ext = zext i1 %0 to i32
+  ret i32 %land.ext
+}
diff --git a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
index 09a6e26fe5a40..22440b79bdcd4 100644
--- a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
+++ b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
@@ -262,4 +262,210 @@ define <4 x i65> @sign_4xi65(<4 x i65> %a) {
   ret <4 x i65> %res
 }
 
+define i32 @or_neg(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w8, w0, #0x1
+; CHECK-NEXT:    cmn w1, w8
+; CHECK-NEXT:    cset w0, lt
+; CHECK-NEXT:    ret
+  %3 = or i32 %x, 1
+  %4 = sub i32 0, %3
+  %5 = icmp sgt i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+define i32 @or_neg_ugt(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w8, w0, #0x1
+; CHECK-NEXT:    cmn w1, w8
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %3 = or i32 %x, 1
+  %4 = sub i32 0, %3
+  %5 = icmp ugt i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+; Negative test
+
+define i32 @or_neg_no_smin(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg_no_smin:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w8, w0
+; CHECK-NEXT:    cmp w8, w1
+; CHECK-NEXT:    cset w0, gt
+; CHECK-NEXT:    ret
+  %4 = sub i32 0, %x
+  %5 = icmp sgt i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+; Negative test
+
+define i32 @or_neg_ult_no_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg_ult_no_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w8, w0
+; CHECK-NEXT:    cmp w8, w1
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %4 = sub i32 0, %x
+  %5 = icmp ult i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+define i32 @or_neg_no_smin_but_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg_no_smin_but_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bic w8, w0, w0, asr #31
+; CHECK-NEXT:    cmn w1, w8
+; CHECK-NEXT:    cset w0, lt
+; CHECK-NEXT:    ret
+  %3 = call i32 @llvm.smax.i32(i32 %x, i32 0)
+  %4 = sub i32 0, %3
+  %5 = icmp sgt i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+define i32 @or_neg_slt_zero_but_no_smin(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg_slt_zero_but_no_smin:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #9 // =0x9
+; CHECK-NEXT:    cmp w0, #9
+; CHECK-NEXT:    csel w8, w0, w8, lo
+; CHECK-NEXT:    neg w8, w8
+; CHECK-NEXT:    cmp w8, w1
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    ret
+  %3 = call i32 @llvm.umin.i32(i32 %x, i32 9)
+  %4 = sub i32 0, %3
+  %5 = icmp ugt i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+define i32 @or_neg2(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w8, w0, #0x1
+; CHECK-NEXT:    cmn w1, w8
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
+  %3 = or i32 %x, 1
+  %4 = sub i32 0, %3
+  %5 = icmp sge i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+define i32 @or_neg3(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w8, w0, #0x1
+; CHECK-NEXT:    cmn w1, w8
+; CHECK-NEXT:    cset w0, gt
+; CHECK-NEXT:    ret
+  %3 = or i32 %x, 1
+  %4 = sub i32 0, %3
+  %5 = icmp slt i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+define i32 @or_neg4(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w8, w0, #0x1
+; CHECK-NEXT:    cmn w1, w8
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
+  %3 = or i32 %x, 1
+  %4 = sub i32 0, %3
+  %5 = icmp sle i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+define i32 @or_neg_ult(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w8, w0, #0x1
+; CHECK-NEXT:    cmn w1, w8
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %3 = or i32 %x, 1
+  %4 = sub i32 0, %3
+  %5 = icmp ugt i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+define i32 @or_neg_no_smin2(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg_no_smin2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w8, w0
+; CHECK-NEXT:    cmp w8, w1
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
+  %4 = sub i32 0, %x
+  %5 = icmp sge i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+; Negative test
+
+define i32 @or_neg_ult_no_zero2(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg_ult_no_zero2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w8, w0
+; CHECK-NEXT:    cmp w8, w1
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %4 = sub i32 0, %x
+  %5 = icmp ult i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+define i32 @or_neg_no_smin_but_zero2(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg_no_smin_but_zero2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bic w8, w0, w0, asr #31
+; CHECK-NEXT:    cmn w1, w8
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
+  %3 = call i32 @llvm.smax.i32(i32 %x, i32 0)
+  %4 = sub i32 0, %3
+  %5 = icmp sle i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+define i32 @or_neg_slt_zero_but_no_smin2(i32 %x, i32 %y) {
+; CHECK-LABEL: or_neg_slt_zero_but_no_smin2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #9 // =0x9
+; CHECK-NEXT:    cmp w0, #9
+; CHECK-NEXT:    csel w8, w0, w8, lo
+; CHECK-NEXT:    neg w8, w8
+; CHECK-NEXT:    cmp w8, w1
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
+  %3 = call i32 @llvm.umin.i32(i32 %x, i32 9)
+  %4 = sub i32 0, %3
+  %5 = icmp uge i32 %4, %y
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+declare i32 @llvm.smax.i32(i32, i32)
+declare i32 @llvm.umax.i32(i32, i32)
 declare void @use_4xi1(<4 x i1>)

From 46223b5eae1ae25ad0d8ca47ea210c1781cb03fc Mon Sep 17 00:00:00 2001
From: Romaric Jodin <rjodin@chromium.org>
Date: Mon, 22 Jul 2024 12:08:56 +0200
Subject: [PATCH 392/486] libclc: add half version of 'sign' (#99841)

---
 libclc/generic/lib/common/sign.cl | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/libclc/generic/lib/common/sign.cl b/libclc/generic/lib/common/sign.cl
index 25832e0b4f8b9..7b6b793be79c1 100644
--- a/libclc/generic/lib/common/sign.cl
+++ b/libclc/generic/lib/common/sign.cl
@@ -26,3 +26,12 @@ SIGN(double, )
 _CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, sign, double)
 
 #endif
+
+#ifdef cl_khr_fp16
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+SIGN(half,)
+_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, sign, half)
+
+#endif

From 85e74285626191108a78e797179374219b3a67d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Mon, 22 Jul 2024 11:11:52 +0100
Subject: [PATCH 393/486] [mlir][vector] Refactor vector-transfer-flatten.mlir
 (nfc) (3/n) (#95745)

The main goal of this and subsequent PRs is to unify and categorize
tests in:
  * vector-transfer-flatten.mlir

This should make it easier to identify the edge cases being tested (and
how they differ), remove duplicates and to add tests for scalable
vectors.

The main contributions of this PR:

1. For consistency with other tests,
   `@transfer_read_flattenable_with_dynamic_dims_and_indices` is renamed
   as `@transfer_read_leading_dynamic_dims`. It is also moved near other
   tests for `xfer_read`, variable names are updated to match other
   `xfer_read` tests

2. `@transfer_write_dims_mismatch_non_zero_indices_trailing_dynamic_dim`
   is renamed as `@negative_transfer_read_dynamic_dim_to_flatten` to
   better highlight that it's a negative test and to contrast it with
   `@transfer_read_leading_dynamic_dims` (and to emphasise the
   difference between the two).

3. Similar changes for tests for `xfer_write`.

4. Make sure that we consistently use `%idx_N` (as opposed to `%idxN`).

Follow-up for #95743 and #95744
---
 .../Vector/vector-transfer-flatten.mlir       | 147 ++++++++++--------
 1 file changed, 82 insertions(+), 65 deletions(-)

diff --git a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
index 303f841e8a828..621baef82319f 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
@@ -110,12 +110,12 @@ func.func @transfer_read_dims_mismatch_non_zero_indices(
 
 func.func @transfer_read_dims_mismatch_non_contiguous_non_zero_indices(
     %arg : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>,
-    %idx0 : index,
-    %idx1 : index) -> vector<2x2xf32> {
+    %idx_1 : index,
+    %idx_2 : index) -> vector<2x2xf32> {
 
   %c0 = arith.constant 0 : index
   %cst_1 = arith.constant 0.000000e+00 : f32
-  %8 = vector.transfer_read %arg[%c0, %idx0, %idx1, %c0], %cst_1 {in_bounds = [true, true]} :
+  %8 = vector.transfer_read %arg[%c0, %idx_1, %idx_2, %c0], %cst_1 {in_bounds = [true, true]} :
     memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>, vector<2x2xf32>
   return %8 : vector<2x2xf32>
 }
@@ -123,7 +123,8 @@ func.func @transfer_read_dims_mismatch_non_contiguous_non_zero_indices(
 // CHECK: #[[$MAP:.+]] = affine_map<()[s0] -> (s0 * 2)>
 
 // CHECK-LABEL:  func.func @transfer_read_dims_mismatch_non_contiguous_non_zero_indices(
-// CHECK:         %[[COLLAPSE:.+]] = memref.collapse_shape %{{.*}} {{\[}}[0], [1], [2, 3]] : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>> into memref<1x3x6xf32, strided<[40, 10, 1], offset: ?>>
+// CHECK:         %[[COLLAPSE:.+]] = memref.collapse_shape %{{.*}} {{\[}}[0], [1], [2, 3]]
+// CHECK-SAME:      : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>> into memref<1x3x6xf32, strided<[40, 10, 1], offset: ?>>
 // CHECK:         %[[APPLY:.*]] = affine.apply #[[$MAP]]()
 
 // CHECK-128B-LABEL: func @transfer_read_dims_mismatch_non_contiguous_non_zero_indices(
@@ -131,10 +132,42 @@ func.func @transfer_read_dims_mismatch_non_contiguous_non_zero_indices(
 
 // -----
 
-// The input memref has a dynamic trailing shape and hence is not flattened.
-// TODO: This case could be supported via memref.dim
+// The leading dynamic shapes don't affect whether this example is flattenable
+// or not. Indeed, those dynamic shapes are not candidates for flattening anyway.
 
-func.func @transfer_read_dims_mismatch_non_zero_indices_dynamic_shapes(
+func.func @transfer_read_leading_dynamic_dims(
+    %arg : memref<?x?x8x4xi8, strided<[?, 32, 4, 1], offset: ?>>,
+    %idx_1 : index,
+    %idx_2 : index) -> vector<8x4xi8> {
+
+  %c0_i8 = arith.constant 0 : i8
+  %c0 = arith.constant 0 : index
+  %result = vector.transfer_read %arg[%idx_1, %idx_2, %c0, %c0], %c0_i8 {in_bounds = [true, true]} :
+    memref<?x?x8x4xi8, strided<[?, 32, 4, 1], offset: ?>>, vector<8x4xi8>
+  return %result : vector<8x4xi8>
+}
+
+// CHECK-LABEL: func @transfer_read_leading_dynamic_dims
+// CHECK-SAME:    %[[ARG0:.+]]: memref<?x?x8x4xi8, {{.+}}>, %[[ARG1:.+]]: index, %[[ARG2:.+]]: index
+// CHECK:         %[[C0_I8:.+]] = arith.constant 0 : i8
+// CHECK:         %[[C0:.+]] = arith.constant 0 : index
+// CHECK:         %[[COLLAPSED:.+]] = memref.collapse_shape %[[ARG0]] {{\[}}[0], [1], [2, 3]{{\]}}
+// CHECK-SAME:      : memref<?x?x8x4xi8, {{.+}}> into memref<?x?x32xi8, {{.+}}>
+// CHECK:         %[[VEC1D:.+]] = vector.transfer_read %[[COLLAPSED]]
+// CHECK-SAME:    [%[[ARG1]], %[[ARG2]], %[[C0]]], %[[C0_I8]]
+// CHECK-SAME:    {in_bounds = [true]}
+// CHECK-SAME:      : memref<?x?x32xi8, {{.+}}>, vector<32xi8>
+// CHECK:         %[[VEC2D:.+]] = vector.shape_cast %[[VEC1D]] : vector<32xi8> to vector<8x4xi8>
+// CHECK:         return %[[VEC2D]] : vector<8x4xi8>
+
+// CHECK-128B-LABEL: func @transfer_read_leading_dynamic_dims
+//       CHECK-128B:   memref.collapse_shape
+
+// -----
+
+// One of the dims to be flattened is dynamic - not supported ATM.
+
+func.func @negative_transfer_read_dynamic_dim_to_flatten(
     %idx_1: index,
     %idx_2: index,
     %m_in: memref<1x?x4x6xi32>) -> vector<1x2x6xi32> {
@@ -146,11 +179,11 @@ func.func @transfer_read_dims_mismatch_non_zero_indices_dynamic_shapes(
   return %v : vector<1x2x6xi32>
 }
 
-// CHECK-LABEL: func.func @transfer_read_dims_mismatch_non_zero_indices_dynamic_shapes(
+// CHECK-LABEL: func.func @negative_transfer_read_dynamic_dim_to_flatten
 // CHECK-NOT: memref.collapse_shape
 // CHECK-NOT: vector.shape_cast
 
-// CHECK-128B-LABEL: func @transfer_read_dims_mismatch_non_zero_indices_dynamic_shapes(
+// CHECK-128B-LABEL: func @negative_transfer_read_dynamic_dim_to_flatten
 //   CHECK-128B-NOT:   memref.collapse_shape
 
 // -----
@@ -326,11 +359,11 @@ func.func @transfer_write_dims_mismatch_non_zero_indices(
 func.func @transfer_write_dims_mismatch_non_contiguous_non_zero_indices(
     %value : vector<2x2xf32>,
     %subview : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>,
-    %idx0 : index,
-    %idx1 : index) {
+    %idx_1 : index,
+    %idx_2 : index) {
 
   %c0 = arith.constant 0 : index
-  vector.transfer_write %value, %subview[%c0, %idx0, %idx1, %c0] {in_bounds = [true, true]} : vector<2x2xf32>, memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>
+  vector.transfer_write %value, %subview[%c0, %idx_1, %idx_2, %c0] {in_bounds = [true, true]} : vector<2x2xf32>, memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>
   return
 }
 
@@ -345,10 +378,40 @@ func.func @transfer_write_dims_mismatch_non_contiguous_non_zero_indices(
 
 // -----
 
-// The input memref has a dynamic trailing shape and hence is not flattened.
-// TODO: This case could be supported via memref.dim
+// The leading dynamic shapes don't affect whether this example is flattenable
+// or not. Indeed, those dynamic shapes are not candidates for flattening anyway.
+
+func.func @transfer_write_leading_dynamic_dims(
+    %vec : vector<8x4xi8>,
+    %arg : memref<?x?x8x4xi8, strided<[?, 32, 4, 1], offset: ?>>,
+    %idx_1 : index,
+    %idx_2 : index) {
+
+  %c0 = arith.constant 0 : index
+  vector.transfer_write %vec, %arg[%idx_1, %idx_2, %c0, %c0] {in_bounds = [true, true]} :
+    vector<8x4xi8>, memref<?x?x8x4xi8, strided<[?, 32, 4, 1], offset: ?>>
+  return
+}
+
+// CHECK-LABEL: func @transfer_write_leading_dynamic_dims
+// CHECK-SAME:    %[[ARG0:.+]]: vector<8x4xi8>, %[[ARG1:.+]]: memref<?x?x8x4xi8, {{.+}}>, %[[ARG2:.+]]: index, %[[ARG3:.+]]: index
+// CHECK:         %[[C0:.+]] = arith.constant 0 : index
+// CHECK:         %[[COLLAPSED:.+]] = memref.collapse_shape %[[ARG1]] {{\[}}[0], [1], [2, 3]{{\]}}
+// CHECK-SAME:      : memref<?x?x8x4xi8, {{.+}}> into memref<?x?x32xi8, {{.+}}>
+// CHECK:         %[[VEC1D:.+]] = vector.shape_cast %[[ARG0]] : vector<8x4xi8> to vector<32xi8>
+// CHECK:         vector.transfer_write %[[VEC1D]], %[[COLLAPSED]]
+// CHECK-SAME:      [%[[ARG2]], %[[ARG3]], %[[C0]]]
+// CHECK-SAME:      {in_bounds = [true]}
+// CHECK-SAME:      : vector<32xi8>, memref<?x?x32xi8, {{.+}}>
+
+// CHECK-128B-LABEL: func @transfer_write_leading_dynamic_dims
+//       CHECK-128B:   memref.collapse_shape
 
-func.func @transfer_write_dims_mismatch_non_zero_indices_dynamic_shapes(
+// -----
+
+// One of the dims to be flattened is dynamic - not supported ATM.
+
+func.func @negative_transfer_write_dynamic_to_flatten(
     %idx_1: index,
     %idx_2: index,
     %vec : vector<1x2x6xi32>,
@@ -361,11 +424,11 @@ func.func @transfer_write_dims_mismatch_non_zero_indices_dynamic_shapes(
   return
 }
 
-// CHECK-LABEL: func.func @transfer_write_dims_mismatch_non_zero_indices_dynamic_shapes(
+// CHECK-LABEL: func.func @negative_transfer_write_dynamic_to_flatten
 // CHECK-NOT: memref.collapse_shape
 // CHECK-NOT: vector.shape_cast
 
-// CHECK-128B-LABEL: func @transfer_write_dims_mismatch_non_zero_indices_dynamic_shapes(
+// CHECK-128B-LABEL: func @negative_transfer_write_dynamic_to_flatten
 //   CHECK-128B-NOT:   memref.collapse_shape
 
 // -----
@@ -434,56 +497,10 @@ func.func @transfer_write_non_contiguous_src(
 // -----
 
 ///----------------------------------------------------------------------------------------
-/// TODO: Categorize + re-format
+/// [Pattern: DropUnitDimFromElementwiseOps]
+/// TODO: Move to a dedicated file - there's no "flattening" in the following tests
 ///----------------------------------------------------------------------------------------
 
-func.func @transfer_read_flattenable_with_dynamic_dims_and_indices(%arg0 : memref<?x?x8x4xi8, strided<[?, 32, 4, 1], offset: ?>>, %arg1 : index, %arg2 : index) -> vector<8x4xi8> {
-    %c0_i8 = arith.constant 0 : i8
-    %c0 = arith.constant 0 : index
-    %result = vector.transfer_read %arg0[%arg1, %arg2, %c0, %c0], %c0_i8 {in_bounds = [true, true]} : memref<?x?x8x4xi8, strided<[?, 32, 4, 1], offset: ?>>, vector<8x4xi8>
-    return %result : vector<8x4xi8>
-}
-
-// CHECK-LABEL: func @transfer_read_flattenable_with_dynamic_dims_and_indices
-// CHECK-SAME:    %[[ARG0:.+]]: memref<?x?x8x4xi8, {{.+}}>, %[[ARG1:.+]]: index, %[[ARG2:.+]]: index
-// CHECK:       %[[C0_I8:.+]] = arith.constant 0 : i8
-// CHECK:       %[[C0:.+]] = arith.constant 0 : index
-// CHECK:       %[[COLLAPSED:.+]] = memref.collapse_shape %[[ARG0]] {{\[}}[0], [1], [2, 3]{{\]}}
-// CHECK-SAME:    : memref<?x?x8x4xi8, {{.+}}> into memref<?x?x32xi8, {{.+}}>
-// CHECK:       %[[VEC1D:.+]] = vector.transfer_read %[[COLLAPSED]]
-// CHECK-SAME:    [%[[ARG1]], %[[ARG2]], %[[C0]]], %[[C0_I8]]
-// CHECK-SAME:    {in_bounds = [true]}
-// CHECK-SAME:    : memref<?x?x32xi8, {{.+}}>, vector<32xi8>
-// CHECK:       %[[VEC2D:.+]] = vector.shape_cast %[[VEC1D]] : vector<32xi8> to vector<8x4xi8>
-// CHECK:       return %[[VEC2D]] : vector<8x4xi8>
-
-// CHECK-128B-LABEL: func @transfer_read_flattenable_with_dynamic_dims_and_indices(
-//       CHECK-128B:   memref.collapse_shape
-
-// -----
-
-func.func @transfer_write_flattenable_with_dynamic_dims_and_indices(%vec : vector<8x4xi8>, %dst : memref<?x?x8x4xi8, strided<[?, 32, 4, 1], offset: ?>>, %arg1 : index, %arg2 : index) {
-    %c0 = arith.constant 0 : index
-    vector.transfer_write %vec, %dst[%arg1, %arg2, %c0, %c0] {in_bounds = [true, true]} : vector<8x4xi8>, memref<?x?x8x4xi8, strided<[?, 32, 4, 1], offset: ?>>
-    return
-}
-
-// CHECK-LABEL: func @transfer_write_flattenable_with_dynamic_dims_and_indices
-// CHECK-SAME:    %[[ARG0:.+]]: vector<8x4xi8>, %[[ARG1:.+]]: memref<?x?x8x4xi8, {{.+}}>, %[[ARG2:.+]]: index, %[[ARG3:.+]]: index
-// CHECK:       %[[C0:.+]] = arith.constant 0 : index
-// CHECK:       %[[COLLAPSED:.+]] = memref.collapse_shape %[[ARG1]] {{\[}}[0], [1], [2, 3]{{\]}}
-// CHECK-SAME:    : memref<?x?x8x4xi8, {{.+}}> into memref<?x?x32xi8, {{.+}}>
-// CHECK:       %[[VEC1D:.+]] = vector.shape_cast %[[ARG0]] : vector<8x4xi8> to vector<32xi8>
-// CHECK:       vector.transfer_write %[[VEC1D]], %[[COLLAPSED]]
-// CHECK-SAME:    [%[[ARG2]], %[[ARG3]], %[[C0]]]
-// CHECK-SAME:    {in_bounds = [true]}
-// CHECK-SAME:    : vector<32xi8>, memref<?x?x32xi8, {{.+}}>
-
-// CHECK-128B-LABEL: func @transfer_write_flattenable_with_dynamic_dims_and_indices(
-//       CHECK-128B:   memref.collapse_shape
-
-// -----
-
 func.func @fold_unit_dim_add_basic(%arg0 : vector<1x8xi32>) -> vector<1x8xi32> {
    %add = arith.addi %arg0, %arg0 : vector<1x8xi32>
    return %add : vector<1x8xi32>

From 45c0decdda5eee6b4d13d4bf241a45c56e412c05 Mon Sep 17 00:00:00 2001
From: Vikash Gupta <Vikash.Gupta@amd.com>
Date: Mon, 22 Jul 2024 15:42:15 +0530
Subject: [PATCH 394/486] [LLD] [COFF] Zero-intialization & proper constructor
 invocation in COFF's Symbol (#98447)

It happened due to lld's COFF linker multiple regression tests failure.
It got reliably reproduced after the needed intialization of
isUsedinRegularObject bit in the Symbol's ctor, but not handled at
replaceSymbol API properly while creating a specific symbol to insert in
symbol table.

So, now while creating the specific symbol using replaceSymbol, by
explicitly setting the value of isUsedinRegularObject to newly created
symbol around the ctor call of symbol would solve the regression failure
---
 lld/COFF/Symbols.h | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/lld/COFF/Symbols.h b/lld/COFF/Symbols.h
index 5ef46f5af6a6c..56b137d56873a 100644
--- a/lld/COFF/Symbols.h
+++ b/lld/COFF/Symbols.h
@@ -98,10 +98,10 @@ class Symbol {
   friend SymbolTable;
   explicit Symbol(Kind k, StringRef n = "")
       : symbolKind(k), isExternal(true), isCOMDAT(false),
-        writtenToSymtab(false), pendingArchiveLoad(false), isGCRoot(false),
-        isRuntimePseudoReloc(false), deferUndefined(false), canInline(true),
-        isWeak(false), nameSize(n.size()),
-        nameData(n.empty() ? nullptr : n.data()) {
+        writtenToSymtab(false), isUsedInRegularObj(false),
+        pendingArchiveLoad(false), isGCRoot(false), isRuntimePseudoReloc(false),
+        deferUndefined(false), canInline(true), isWeak(false),
+        nameSize(n.size()), nameData(n.empty() ? nullptr : n.data()) {
     assert((!n.empty() || k <= LastDefinedCOFFKind) &&
            "If the name is empty, the Symbol must be a DefinedCOFF.");
   }
@@ -499,8 +499,10 @@ void replaceSymbol(Symbol *s, ArgT &&... arg) {
   assert(static_cast<Symbol *>(static_cast<T *>(nullptr)) == nullptr &&
          "Not a Symbol");
   bool canInline = s->canInline;
+  bool isUsedInRegularObj = s->isUsedInRegularObj;
   new (s) T(std::forward<ArgT>(arg)...);
   s->canInline = canInline;
+  s->isUsedInRegularObj = isUsedInRegularObj;
 }
 } // namespace coff
 

From ef2af7f85616b48029dce55069c3faa949d46454 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin@arm.com>
Date: Mon, 22 Jul 2024 11:21:35 +0100
Subject: [PATCH 395/486] [AArch64][SME] Make use of Arm Optimised Routines in
 compiler-rt (#99326)

A number of streaming-compatible versions of standard C functions
were added to compiler-rt, however there are already optimised
versions of most of these in libc which are valid in streaming-SVE
mode. This patch replaces the implementations of __arm_sc_mem* with
these versions where possible.
---
 compiler-rt/lib/builtins/CMakeLists.txt       |   2 +-
 .../builtins/aarch64/sme-libc-mem-routines.S  | 344 ++++++++++++++++++
 .../lib/builtins/aarch64/sme-libc-routines.c  |  75 ----
 3 files changed, 345 insertions(+), 76 deletions(-)
 create mode 100644 compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S

diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 744bbfacf32f1..88a5998fd4610 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -571,7 +571,7 @@ set(aarch64_SOURCES
 
 if (COMPILER_RT_HAS_AARCH64_SME)
   if (NOT COMPILER_RT_DISABLE_AARCH64_FMV AND COMPILER_RT_HAS_FNO_BUILTIN_FLAG AND (COMPILER_RT_HAS_AUXV OR COMPILER_RT_BAREMETAL_BUILD))
-    list(APPEND aarch64_SOURCES aarch64/sme-abi.S aarch64/sme-abi-init.c aarch64/sme-abi-vg.c aarch64/sme-libc-routines.c)
+    list(APPEND aarch64_SOURCES aarch64/sme-abi.S aarch64/sme-libc-mem-routines.S aarch64/sme-abi-init.c aarch64/sme-abi-vg.c aarch64/sme-libc-routines.c)
     message(STATUS "AArch64 SME ABI routines enabled")
     set_source_files_properties(aarch64/sme-libc-routines.c PROPERTIES COMPILE_FLAGS "-fno-builtin")
   else()
diff --git a/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S b/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S
new file mode 100644
index 0000000000000..926ad3b1b6331
--- /dev/null
+++ b/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S
@@ -0,0 +1,344 @@
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Routines taken from libc/AOR_v20.02/string/aarch64
+
+#include "../assembly.h"
+
+#ifdef __aarch64__
+
+#define L(l) .L ## l
+
+//
+//  __arm_sc_memcpy / __arm_sc_memmove
+//
+
+#define dstin    x0
+#define src      x1
+#define count    x2
+#define dst      x3
+#define srcend1  x4
+#define dstend1  x5
+#define A_l      x6
+#define A_lw     w6
+#define A_h      x7
+#define B_l      x8
+#define B_lw     w8
+#define B_h      x9
+#define C_l      x10
+#define C_lw     w10
+#define C_h      x11
+#define D_l      x12
+#define D_h      x13
+#define E_l      x14
+#define E_h      x15
+#define F_l      x16
+#define F_h      x17
+#define G_l      count
+#define G_h      dst
+#define H_l      src
+#define H_h      srcend1
+#define tmp1     x14
+
+/* This implementation handles overlaps and supports both memcpy and memmove
+   from a single entry point.  It uses unaligned accesses and branchless
+   sequences to keep the code small, simple and improve performance.
+
+   Copies are split into 3 main cases: small copies of up to 32 bytes, medium
+   copies of up to 128 bytes, and large copies.  The overhead of the overlap
+   check is negligible since it is only required for large copies.
+
+   Large copies use a software pipelined loop processing 64 bytes per iteration.
+   The destination pointer is 16-byte aligned to minimize unaligned accesses.
+   The loop tail is handled by always copying 64 bytes from the end.
+*/
+
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_sc_memcpy)
+        add     srcend1, src, count
+        add     dstend1, dstin, count
+        cmp     count, 128
+        b.hi    L(copy_long)
+        cmp     count, 32
+        b.hi    L(copy32_128)
+
+        /* Small copies: 0..32 bytes.  */
+        cmp     count, 16
+        b.lo    L(copy16)
+        ldp     A_l, A_h, [src]
+        ldp     D_l, D_h, [srcend1, -16]
+        stp     A_l, A_h, [dstin]
+        stp     D_l, D_h, [dstend1, -16]
+        ret
+
+        /* Copy 8-15 bytes.  */
+L(copy16):
+        tbz     count, 3, L(copy8)
+        ldr     A_l, [src]
+        ldr     A_h, [srcend1, -8]
+        str     A_l, [dstin]
+        str     A_h, [dstend1, -8]
+        ret
+
+        .p2align 3
+        /* Copy 4-7 bytes.  */
+L(copy8):
+        tbz     count, 2, L(copy4)
+        ldr     A_lw, [src]
+        ldr     B_lw, [srcend1, -4]
+        str     A_lw, [dstin]
+        str     B_lw, [dstend1, -4]
+        ret
+
+        /* Copy 0..3 bytes using a branchless sequence.  */
+L(copy4):
+        cbz     count, L(copy0)
+        lsr     tmp1, count, 1
+        ldrb    A_lw, [src]
+        ldrb    C_lw, [srcend1, -1]
+        ldrb    B_lw, [src, tmp1]
+        strb    A_lw, [dstin]
+        strb    B_lw, [dstin, tmp1]
+        strb    C_lw, [dstend1, -1]
+L(copy0):
+        ret
+
+        .p2align 4
+        /* Medium copies: 33..128 bytes.  */
+L(copy32_128):
+        ldp     A_l, A_h, [src]
+        ldp     B_l, B_h, [src, 16]
+        ldp     C_l, C_h, [srcend1, -32]
+        ldp     D_l, D_h, [srcend1, -16]
+        cmp     count, 64
+        b.hi    L(copy128)
+        stp     A_l, A_h, [dstin]
+        stp     B_l, B_h, [dstin, 16]
+        stp     C_l, C_h, [dstend1, -32]
+        stp     D_l, D_h, [dstend1, -16]
+        ret
+
+        .p2align 4
+        /* Copy 65..128 bytes.  */
+L(copy128):
+        ldp     E_l, E_h, [src, 32]
+        ldp     F_l, F_h, [src, 48]
+        cmp     count, 96
+        b.ls    L(copy96)
+        ldp     G_l, G_h, [srcend1, -64]
+        ldp     H_l, H_h, [srcend1, -48]
+        stp     G_l, G_h, [dstend1, -64]
+        stp     H_l, H_h, [dstend1, -48]
+L(copy96):
+        stp     A_l, A_h, [dstin]
+        stp     B_l, B_h, [dstin, 16]
+        stp     E_l, E_h, [dstin, 32]
+        stp     F_l, F_h, [dstin, 48]
+        stp     C_l, C_h, [dstend1, -32]
+        stp     D_l, D_h, [dstend1, -16]
+        ret
+
+        .p2align 4
+        /* Copy more than 128 bytes.  */
+L(copy_long):
+        /* Use backwards copy if there is an overlap.  */
+        sub     tmp1, dstin, src
+        cbz     tmp1, L(copy0)
+        cmp     tmp1, count
+        b.lo    L(copy_long_backwards)
+
+        /* Copy 16 bytes and then align dst to 16-byte alignment.  */
+
+        ldp     D_l, D_h, [src]
+        and     tmp1, dstin, 15
+        bic     dst, dstin, 15
+        sub     src, src, tmp1
+        add     count, count, tmp1      /* Count is now 16 too large.  */
+        ldp     A_l, A_h, [src, 16]
+        stp     D_l, D_h, [dstin]
+        ldp     B_l, B_h, [src, 32]
+        ldp     C_l, C_h, [src, 48]
+        ldp     D_l, D_h, [src, 64]!
+        subs    count, count, 128 + 16  /* Test and readjust count.  */
+        b.ls    L(copy64_from_end)
+L(loop64):
+        stp     A_l, A_h, [dst, 16]
+        ldp     A_l, A_h, [src, 16]
+        stp     B_l, B_h, [dst, 32]
+        ldp     B_l, B_h, [src, 32]
+        stp     C_l, C_h, [dst, 48]
+        ldp     C_l, C_h, [src, 48]
+        stp     D_l, D_h, [dst, 64]!
+        ldp     D_l, D_h, [src, 64]!
+        subs    count, count, 64
+        b.hi    L(loop64)
+
+        /* Write the last iteration and copy 64 bytes from the end.  */
+L(copy64_from_end):
+        ldp     E_l, E_h, [srcend1, -64]
+        stp     A_l, A_h, [dst, 16]
+        ldp     A_l, A_h, [srcend1, -48]
+        stp     B_l, B_h, [dst, 32]
+        ldp     B_l, B_h, [srcend1, -32]
+        stp     C_l, C_h, [dst, 48]
+        ldp     C_l, C_h, [srcend1, -16]
+        stp     D_l, D_h, [dst, 64]
+        stp     E_l, E_h, [dstend1, -64]
+        stp     A_l, A_h, [dstend1, -48]
+        stp     B_l, B_h, [dstend1, -32]
+        stp     C_l, C_h, [dstend1, -16]
+        ret
+
+        .p2align 4
+
+        /* Large backwards copy for overlapping copies.
+           Copy 16 bytes and then align dst to 16-byte alignment.  */
+L(copy_long_backwards):
+        ldp     D_l, D_h, [srcend1, -16]
+        and     tmp1, dstend1, 15
+        sub     srcend1, srcend1, tmp1
+        sub     count, count, tmp1
+        ldp     A_l, A_h, [srcend1, -16]
+        stp     D_l, D_h, [dstend1, -16]
+        ldp     B_l, B_h, [srcend1, -32]
+        ldp     C_l, C_h, [srcend1, -48]
+        ldp     D_l, D_h, [srcend1, -64]!
+        sub     dstend1, dstend1, tmp1
+        subs    count, count, 128
+        b.ls    L(copy64_from_start)
+
+L(loop64_backwards):
+        stp     A_l, A_h, [dstend1, -16]
+        ldp     A_l, A_h, [srcend1, -16]
+        stp     B_l, B_h, [dstend1, -32]
+        ldp     B_l, B_h, [srcend1, -32]
+        stp     C_l, C_h, [dstend1, -48]
+        ldp     C_l, C_h, [srcend1, -48]
+        stp     D_l, D_h, [dstend1, -64]!
+        ldp     D_l, D_h, [srcend1, -64]!
+        subs    count, count, 64
+        b.hi    L(loop64_backwards)
+
+        /* Write the last iteration and copy 64 bytes from the start.  */
+L(copy64_from_start):
+        ldp     G_l, G_h, [src, 48]
+        stp     A_l, A_h, [dstend1, -16]
+        ldp     A_l, A_h, [src, 32]
+        stp     B_l, B_h, [dstend1, -32]
+        ldp     B_l, B_h, [src, 16]
+        stp     C_l, C_h, [dstend1, -48]
+        ldp     C_l, C_h, [src]
+        stp     D_l, D_h, [dstend1, -64]
+        stp     G_l, G_h, [dstin, 48]
+        stp     A_l, A_h, [dstin, 32]
+        stp     B_l, B_h, [dstin, 16]
+        stp     C_l, C_h, [dstin]
+        ret
+END_COMPILERRT_OUTLINE_FUNCTION(__arm_sc_memcpy)
+
+DEFINE_COMPILERRT_FUNCTION_ALIAS(__arm_sc_memmove, __arm_sc_memcpy)
+
+
+//
+//  __arm_sc_memset
+//
+
+#define dstin    x0
+#define val      x1
+#define valw     w1
+#define count    x2
+#define dst      x3
+#define dstend2  x4
+#define zva_val  x5
+
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_sc_memset)
+        dup     v0.16B, valw
+        add     dstend2, dstin, count
+
+        cmp     count, 96
+        b.hi    L(set_long)
+        cmp     count, 16
+        b.hs    L(set_medium)
+        mov     val, v0.D[0]
+
+        /* Set 0..15 bytes.  */
+        tbz     count, 3, 1f
+        str     val, [dstin]
+        str     val, [dstend2, -8]
+        ret
+        nop
+1:      tbz     count, 2, 2f
+        str     valw, [dstin]
+        str     valw, [dstend2, -4]
+        ret
+2:      cbz     count, 3f
+        strb    valw, [dstin]
+        tbz     count, 1, 3f
+        strh    valw, [dstend2, -2]
+3:      ret
+
+        /* Set 17..96 bytes.  */
+L(set_medium):
+        str     q0, [dstin]
+        tbnz    count, 6, L(set96)
+        str     q0, [dstend2, -16]
+        tbz     count, 5, 1f
+        str     q0, [dstin, 16]
+        str     q0, [dstend2, -32]
+1:      ret
+
+        .p2align 4
+        /* Set 64..96 bytes.  Write 64 bytes from the start and
+           32 bytes from the end.  */
+L(set96):
+        str     q0, [dstin, 16]
+        stp     q0, q0, [dstin, 32]
+        stp     q0, q0, [dstend2, -32]
+        ret
+
+        .p2align 4
+L(set_long):
+        and     valw, valw, 255
+        bic     dst, dstin, 15
+        str     q0, [dstin]
+        cmp     count, 160
+        ccmp    valw, 0, 0, hs
+        b.ne    L(no_zva)
+
+#ifndef SKIP_ZVA_CHECK
+        mrs     zva_val, dczid_el0
+        and     zva_val, zva_val, 31
+        cmp     zva_val, 4              /* ZVA size is 64 bytes.  */
+        b.ne    L(no_zva)
+#endif
+        str     q0, [dst, 16]
+        stp     q0, q0, [dst, 32]
+        bic     dst, dst, 63
+        sub     count, dstend2, dst      /* Count is now 64 too large.  */
+        sub     count, count, 128       /* Adjust count and bias for loop.  */
+
+        .p2align 4
+L(zva_loop):
+        add     dst, dst, 64
+        dc      zva, dst
+        subs    count, count, 64
+        b.hi    L(zva_loop)
+        stp     q0, q0, [dstend2, -64]
+        stp     q0, q0, [dstend2, -32]
+        ret
+
+L(no_zva):
+        sub     count, dstend2, dst      /* Count is 16 too large.  */
+        sub     dst, dst, 16            /* Dst is biased by -32.  */
+        sub     count, count, 64 + 16   /* Adjust count and bias for loop.  */
+L(no_zva_loop):
+        stp     q0, q0, [dst, 32]
+        stp     q0, q0, [dst, 64]!
+        subs    count, count, 64
+        b.hi    L(no_zva_loop)
+        stp     q0, q0, [dstend2, -64]
+        stp     q0, q0, [dstend2, -32]
+        ret
+END_COMPILERRT_OUTLINE_FUNCTION(__arm_sc_memset)
+
+#endif // __aarch64__
diff --git a/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c b/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
index 89b52b0d1a880..315490e73ea2b 100644
--- a/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
+++ b/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
@@ -1,80 +1,5 @@
 #include <stddef.h>
 
-// WARNING: When building the scalar versions of these functions you need to
-// use the compiler flag "-mllvm -disable-loop-idiom-all" to prevent clang
-// from recognising a loop idiom and planting calls to memcpy!
-
-static void *__arm_sc_memcpy_fwd(void *dest, const void *src,
-                                 size_t n) __arm_streaming_compatible {
-  unsigned char *destp = (unsigned char *)dest;
-  const unsigned char *srcp = (const unsigned char *)src;
-  for (size_t i = 0; i < n; ++i)
-    destp[i] = srcp[i];
-
-  return dest;
-}
-
-// If dest and src overlap then behaviour is undefined, hence we can add the
-// restrict keywords here. This also matches the definition of the libc memcpy
-// according to the man page.
-void *__arm_sc_memcpy(void *__restrict__ dest, const void *__restrict__ src,
-                      size_t n) __arm_streaming_compatible {
-  return __arm_sc_memcpy_fwd(dest, src, n);
-}
-
-void *__arm_sc_memset(void *dest, int c, size_t n) __arm_streaming_compatible {
-  unsigned char *destp = (unsigned char *)dest;
-  unsigned char c8 = (unsigned char)c;
-  for (size_t i = 0; i < n; ++i)
-    destp[i] = c8;
-
-  return dest;
-}
-
-static void *__arm_sc_memcpy_rev(void *dest, const void *src,
-                                 size_t n) __arm_streaming_compatible {
-  unsigned char *destp = (unsigned char *)dest;
-  const unsigned char *srcp = (const unsigned char *)src;
-  // TODO: Improve performance by copying larger chunks in reverse, or by
-  // using SVE.
-  while (n > 0) {
-    --n;
-    destp[n] = srcp[n];
-  }
-  return dest;
-}
-
-// Semantically a memmove is equivalent to the following:
-//   1. Copy the entire contents of src to a temporary array that does not
-//      overlap with src or dest.
-//   2. Copy the contents of the temporary array into dest.
-void *__arm_sc_memmove(void *dest, const void *src,
-                       size_t n) __arm_streaming_compatible {
-  unsigned char *destp = (unsigned char *)dest;
-  const unsigned char *srcp = (const unsigned char *)src;
-
-  // If src and dest don't overlap then just invoke memcpy
-  if ((srcp > (destp + n)) || (destp > (srcp + n)))
-    return __arm_sc_memcpy_fwd(dest, src, n);
-
-  // Overlap case 1:
-  //     src: Low     |   ->   |     High
-  //    dest: Low  |   ->   |        High
-  // Here src is always ahead of dest at a higher addres. If we first read a
-  // chunk of data from src we can safely write the same chunk to dest without
-  // corrupting future reads of src.
-  if (srcp > destp)
-    return __arm_sc_memcpy_fwd(dest, src, n);
-
-  // Overlap case 2:
-  //     src: Low  |   ->   |        High
-  //    dest: Low     |   ->   |     High
-  // While we're in the overlap region we're always corrupting future reads of
-  // src when writing to dest. An efficient way to do this is to copy the data
-  // in reverse by starting at the highest address.
-  return __arm_sc_memcpy_rev(dest, src, n);
-}
-
 const void *__arm_sc_memchr(const void *src, int c,
                             size_t n) __arm_streaming_compatible {
   const unsigned char *srcp = (const unsigned char *)src;

From f406d83d9553c0ade8f9b2dea961e5a8caa220f3 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 18 Jul 2024 17:16:28 +0100
Subject: [PATCH 396/486] [DAG] widenCtPop - reuse existing SDLoc. NFC.

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 9b2153c68ccae..8595f466d51e1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -13782,9 +13782,10 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
 /// Given an extending node with a pop-count operand, if the target does not
 /// support a pop-count in the narrow source type but does support it in the
 /// destination type, widen the pop-count to the destination type.
-static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG) {
+static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG, const SDLoc &DL) {
   assert((Extend->getOpcode() == ISD::ZERO_EXTEND ||
-          Extend->getOpcode() == ISD::ANY_EXTEND) && "Expected extend op");
+          Extend->getOpcode() == ISD::ANY_EXTEND) &&
+         "Expected extend op");
 
   SDValue CtPop = Extend->getOperand(0);
   if (CtPop.getOpcode() != ISD::CTPOP || !CtPop.hasOneUse())
@@ -13797,7 +13798,6 @@ static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG) {
     return SDValue();
 
   // zext (ctpop X) --> ctpop (zext X)
-  SDLoc DL(Extend);
   SDValue NewZext = DAG.getZExtOrTrunc(CtPop.getOperand(0), DL, VT);
   return DAG.getNode(ISD::CTPOP, DL, VT, NewZext);
 }
@@ -14138,7 +14138,7 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
     return NewVSel;
 
-  if (SDValue NewCtPop = widenCtPop(N, DAG))
+  if (SDValue NewCtPop = widenCtPop(N, DAG, DL))
     return NewCtPop;
 
   if (SDValue V = widenAbs(N, DAG))
@@ -14316,7 +14316,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
       return SCC;
   }
 
-  if (SDValue NewCtPop = widenCtPop(N, DAG))
+  if (SDValue NewCtPop = widenCtPop(N, DAG, DL))
     return NewCtPop;
 
   if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))

From 102d16809b2cf79b5e1ff4728ef6164cd92f90d5 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Mon, 22 Jul 2024 11:31:50 +0100
Subject: [PATCH 397/486] [Analysis] Bail out for negative offsets in
 isDereferenceableAndAlignedInLoop (#99490)

This patch now bails out explicitly for negative offsets so that it's
more consistent with the unsigned remainder and add calculations,
and it fixes a genuine bug as shown with the new test.
---
 llvm/lib/Analysis/Loads.cpp                   |   7 +
 .../LoopVectorize/load-deref-pred-align.ll    | 128 ++++++++++++++++--
 2 files changed, 120 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 255a7ae56dd3b..4f0b92f51777b 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -313,6 +313,13 @@ bool llvm::isDereferenceableAndAlignedInLoop(LoadInst *LI, Loop *L,
     const auto *Offset = dyn_cast<SCEVConstant>(StartS->getOperand(0));
     const auto *NewBase = dyn_cast<SCEVUnknown>(StartS->getOperand(1));
     if (StartS->getNumOperands() == 2 && Offset && NewBase) {
+      // The following code below assumes the offset is unsigned, but GEP
+      // offsets are treated as signed so we can end up with a signed value
+      // here too. For example, suppose the initial PHI value is (i8 255),
+      // the offset will be treated as (i8 -1) and sign-extended to (i64 -1).
+      if (Offset->getAPInt().isNegative())
+        return false;
+
       // For the moment, restrict ourselves to the case where the offset is a
       // multiple of the requested alignment and the base is aligned.
       // TODO: generalize if a case found which warrants
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
index b85d68c574a82..a7c9a18127ade 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1-p:16:16:16:16"
 
 declare void @init(ptr nocapture nofree)
 
@@ -17,7 +17,7 @@ define i16 @test_access_size_not_multiple_of_align(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[PRED_LOAD_CONTINUE2]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_LOAD_CONTINUE2]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
@@ -43,16 +43,16 @@ define i16 @test_access_size_not_multiple_of_align(i64 %len, ptr %test_base) {
 ; CHECK:       pred.load.continue2:
 ; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x i16> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ]
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> [[TMP14]], <2 x i16> zeroinitializer
-; CHECK-NEXT:    [[TMP16]] = add <2 x i16> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP15]] = add <2 x i16> [[VEC_PHI]], [[PREDPHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP18:%.*]] = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> [[TMP16]])
+; CHECK-NEXT:    [[TMP17:%.*]] = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> [[TMP15]])
 ; CHECK-NEXT:    br i1 true, label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
@@ -72,7 +72,7 @@ define i16 @test_access_size_not_multiple_of_align(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp eq i64 [[IV]], 4095
 ; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       loop_exit:
-; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i16 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i16 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i16 [[ACCUM_NEXT_LCSSA]]
 ;
 entry:
@@ -114,7 +114,7 @@ define i32 @test_access_size_multiple_of_align_but_offset_by_1(i64 %len, ptr %te
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[PRED_LOAD_CONTINUE2]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_LOAD_CONTINUE2]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
@@ -140,16 +140,16 @@ define i32 @test_access_size_multiple_of_align_but_offset_by_1(i64 %len, ptr %te
 ; CHECK:       pred.load.continue2:
 ; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ]
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP14]], <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP16]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP15]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP16]])
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP15]])
 ; CHECK-NEXT:    br i1 true, label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
@@ -169,7 +169,7 @@ define i32 @test_access_size_multiple_of_align_but_offset_by_1(i64 %len, ptr %te
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp eq i64 [[IV]], 4095
 ; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       loop_exit:
-; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
 ;
 entry:
@@ -198,3 +198,101 @@ latch:
 loop_exit:
   ret i32 %accum.next
 }
+
+
+; Test where offset relative to alloca is negative and we shouldn't
+; treat predicated loads as being always dereferenceable.
+define i8 @test_negative_off(i16 %len, ptr %test_base) {
+; CHECK-LABEL: @test_negative_off(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [64638 x i8], align 1
+; CHECK-NEXT:    call void @init(ptr [[ALLOCA]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i8> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[PRED_LOAD_CONTINUE2]] ]
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 -1000, [[DOTCAST]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE:%.*]], i16 [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i16 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i1, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load i1, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i1> poison, i1 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i1> [[TMP6]], i1 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK:       pred.load.if:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[TMP0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i8> poison, i8 [[TMP10]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; CHECK:       pred.load.continue:
+; CHECK-NEXT:    [[TMP12:%.*]] = phi <2 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
+; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
+; CHECK:       pred.load.if1:
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[TMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr [[TMP14]], align 1
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i8> [[TMP12]], i8 [[TMP15]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
+; CHECK:       pred.load.continue2:
+; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i8> [ [[TMP12]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i8> [[TMP17]], <2 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP18]] = add <2 x i8> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP20:%.*]] = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> [[TMP18]])
+; CHECK-NEXT:    br i1 true, label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ -988, [[MIDDLE_BLOCK]] ], [ -1000, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i8 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i16 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i16 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i8, ptr [[ADDR]], align 1
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i8 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i8 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i16 [[IV]], -990
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i8 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i8 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [64638 x i8]
+  call void @init(ptr %alloca)
+  br label %loop
+loop:
+  %iv = phi i16 [ -1000, %entry ], [ %iv.next, %latch ]
+  %accum = phi i8 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i16 %iv, 1
+  %test_addr = getelementptr inbounds i1, ptr %test_base, i16 %iv
+  %earlycnd = load i1, ptr %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr i8, ptr %alloca, i16 %iv
+  %val = load i8, ptr %addr
+  br label %latch
+latch:
+  %val.phi = phi i8 [ 0, %loop ], [ %val, %pred ]
+  %accum.next = add i8 %accum, %val.phi
+  %exit = icmp ugt i16 %iv, -990
+  br i1 %exit, label %loop_exit, label %loop
+loop_exit:
+  ret i8 %accum.next
+}

From bf08d0e1182c94b6fe14b8915df6a7e5e755e5f2 Mon Sep 17 00:00:00 2001
From: jeanPerier <jperier@nvidia.com>
Date: Mon, 22 Jul 2024 12:51:30 +0200
Subject: [PATCH 398/486] [flang] fix cg-rewrite DCE (#99653)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

cg-rewrite runs regionDCE to get rid of the unused fir.shape/shift/slice
before codegen since those operations have no codegen.
I came across an issue where unreachable code would cause the pass to
fail with `error: loc(...): null operand found`.

It turns out `mlir::RegionDCE` does not work properly in presence of
unreachable code because it delete operations in reachable code that are
unused in reachable code, but still used in unreachable code (like the
constant in the added test case). It seems `mlir::RegionDCE` is always
run after `mlir::eraseUnreachableBlock` outside of this pass.

A solution could be to run `mlir::eraseUnreachableBlock` here or to try
modifying `mlir::RegionDCE`. But the current behavior may be
intentional, and both of these calls are actually quite expensive. For
instance, RegionDCE will does liveness analysis, and removes unused
block arguments, which is way more than what is needed here. I am not
very found of having this rather heavy transformation inside this pass
(they should be run after or before if they matter in the overall
pipeline).

Do a naïve backward deletion of the trivially dead operations instead.
It is cheaper, and works with unreachable code.
---
 flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp | 23 ++++++++++++++++++--
 flang/test/Fir/declare-codegen.fir           | 14 ++++++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
index c57f5140c29eb..fdf24028c5f9b 100644
--- a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
@@ -18,8 +18,8 @@
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
+#include "mlir/IR/Iterators.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Debug.h"
 
@@ -325,6 +325,25 @@ class DummyScopeOpConversion
   }
 };
 
+/// Simple DCE to erase fir.shape/shift/slice/unused shape operands after this
+/// pass (fir.shape and like have no codegen).
+/// mlir::RegionDCE is expensive and requires running
+/// mlir::eraseUnreachableBlocks. It does things that are not needed here, like
+/// removing unused block arguments. fir.shape/shift/slice cannot be block
+/// arguments.
+/// This helper does a naive backward walk of the IR. It is not even guaranteed
+/// to walk blocks according to backward dominance, but that is good enough for
+/// what is done here, fir.shape/shift/slice have no usages anymore. The
+/// backward walk allows getting rid of most of the unused operands, it is not a
+/// problem to leave some in the weird cases.
+static void simpleDCE(mlir::RewriterBase &rewriter, mlir::Operation *op) {
+  op->walk<mlir::WalkOrder::PostOrder, mlir::ReverseIterator>(
+      [&](mlir::Operation *subOp) {
+        if (mlir::isOpTriviallyDead(subOp))
+          rewriter.eraseOp(subOp);
+      });
+}
+
 class CodeGenRewrite : public fir::impl::CodeGenRewriteBase<CodeGenRewrite> {
 public:
   using CodeGenRewriteBase<CodeGenRewrite>::CodeGenRewriteBase;
@@ -356,7 +375,7 @@ class CodeGenRewrite : public fir::impl::CodeGenRewriteBase<CodeGenRewrite> {
     }
     // Erase any residual (fir.shape, fir.slice...).
     mlir::IRRewriter rewriter(&context);
-    (void)mlir::runRegionDCE(rewriter, mod->getRegions());
+    simpleDCE(rewriter, mod.getOperation());
   }
 };
 
diff --git a/flang/test/Fir/declare-codegen.fir b/flang/test/Fir/declare-codegen.fir
index c5879facb1572..fe8d84ef2d19f 100644
--- a/flang/test/Fir/declare-codegen.fir
+++ b/flang/test/Fir/declare-codegen.fir
@@ -38,3 +38,17 @@ func.func @useless_shape_with_duplicate_extent_operand(%arg0: !fir.ref<!fir.arra
 
 // DECL-LABEL: func.func @useless_shape_with_duplicate_extent_operand(
 // DECL: fircg.ext_declare
+
+// Test DCE does not crash because of unreachable code.
+func.func @unreachable_code(%arg0: !fir.ref<!fir.char<1,10>>) {
+  %c10 = arith.constant 10 : index
+  %2 = fir.declare %arg0 typeparams %c10 {uniq_name = "live_code"} : (!fir.ref<!fir.char<1,10>>, index) -> (!fir.ref<!fir.char<1,10>>)
+  return
+^bb2:  // no predecessors
+  %3 = fir.declare %arg0 typeparams %c10 {uniq_name = "dead_code"} : (!fir.ref<!fir.char<1,10>>, index) -> (!fir.ref<!fir.char<1,10>>)
+  fir.unreachable
+}
+// NODECL-LABEL:   func.func @unreachable_code(
+// NODECL-NOT:       uniq_name = "live_code"
+// DECL-LABEL:    func.func @unreachable_code(
+// DECL:             uniq_name = "live_code"

From 462d084241616627be1ac2b967a7fcba9b0facfe Mon Sep 17 00:00:00 2001
From: jeanPerier <jperier@nvidia.com>
Date: Mon, 22 Jul 2024 12:51:58 +0200
Subject: [PATCH 399/486] [flang] fix sequence association of polymorphic
 actual arguments (#99294)

When passing a polymorphic actual array argument to an non polymorphic
explicit or assumed shape argument, copy-in/copy-out may be required and
should be made according to the dummy dynamic type.

The code that was creating the descriptor to drive this copy-in/out was
not handling properly the case where the dummy and actual rank do not
match (possible according to sequence association rules), it tried to
make the copy-in/out according to the dummy argument shape (which we may
not even know if the dummy is assumed-size). Fix this by using the
actual shape when creating this new descriptor with the dummy argument
dynamic type.
---
 flang/lib/Lower/ConvertCall.cpp               | 54 +++++++++----------
 .../Lower/HLFIR/calls-poly-to-nonpoly.f90     | 26 +++++++++
 2 files changed, 53 insertions(+), 27 deletions(-)
 create mode 100644 flang/test/Lower/HLFIR/calls-poly-to-nonpoly.f90

diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index ba65b644e5a93..fd873f55dd844 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -1227,26 +1227,32 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
     return hlfir::Entity{copyIn.getCopiedIn()};
   };
 
+  auto genSetDynamicTypeToDummyType = [&](hlfir::Entity var) -> hlfir::Entity {
+    fir::BaseBoxType boxType = fir::BoxType::get(
+        hlfir::getFortranElementOrSequenceType(dummyTypeWithActualRank));
+    if (actualIsAssumedRank)
+      return hlfir::Entity{builder.create<fir::ReboxAssumedRankOp>(
+          loc, boxType, var, fir::LowerBoundModifierAttribute::SetToOnes)};
+    // Use actual shape when creating descriptor with dummy type, the dummy
+    // shape may be unknown in case of sequence association.
+    mlir::Type actualTy =
+        hlfir::getFortranElementOrSequenceType(actual.getType());
+    boxType = boxType.getBoxTypeWithNewShape(actualTy);
+    return hlfir::Entity{builder.create<fir::ReboxOp>(loc, boxType, var,
+                                                      /*shape=*/mlir::Value{},
+                                                      /*slice=*/mlir::Value{})};
+  };
+
   // Step 2: prepare the storage for the dummy arguments, ensuring that it
   // matches the dummy requirements (e.g., must be contiguous or must be
   // a temporary).
   hlfir::Entity entity =
       hlfir::derefPointersAndAllocatables(loc, builder, actual);
   if (entity.isVariable()) {
-    if (mustSetDynamicTypeToDummyType) {
-      // Note: this is important to do this before any copy-in or copy so
-      // that the dummy is contiguous according to the dummy type.
-      mlir::Type boxType = fir::BoxType::get(
-          hlfir::getFortranElementOrSequenceType(dummyTypeWithActualRank));
-      if (actualIsAssumedRank) {
-        entity = hlfir::Entity{builder.create<fir::ReboxAssumedRankOp>(
-            loc, boxType, entity, fir::LowerBoundModifierAttribute::SetToOnes)};
-      } else {
-        entity = hlfir::Entity{builder.create<fir::ReboxOp>(
-            loc, boxType, entity, /*shape=*/mlir::Value{},
-            /*slice=*/mlir::Value{})};
-      }
-    }
+    // Set dynamic type if needed before any copy-in or copy so that the dummy
+    // is contiguous according to the dummy type.
+    if (mustSetDynamicTypeToDummyType)
+      entity = genSetDynamicTypeToDummyType(entity);
     if (arg.hasValueAttribute() ||
         // Constant expressions might be lowered as variables with
         // 'parameter' attribute. Even though the constant expressions
@@ -1285,20 +1291,14 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
         loc, builder, entity, storageType, "", byRefAttr);
     entity = hlfir::Entity{associate.getBase()};
     preparedDummy.pushExprAssociateCleanUp(associate);
+    // Rebox the actual argument to the dummy argument's type, and make sure
+    // that we pass a contiguous entity (i.e. make copy-in, if needed).
+    //
+    // TODO: this can probably be optimized by associating the expression with
+    // properly typed temporary, but this needs either a new operation or
+    // making the hlfir.associate more complex.
     if (mustSetDynamicTypeToDummyType) {
-      // Rebox the actual argument to the dummy argument's type, and make
-      // sure that we pass a contiguous entity (i.e. make copy-in,
-      // if needed).
-      //
-      // TODO: this can probably be optimized by associating the expression
-      // with properly typed temporary, but this needs either a new operation
-      // or making the hlfir.associate more complex.
-      assert(!actualIsAssumedRank && "only variables are assumed-rank");
-      mlir::Type boxType = fir::BoxType::get(
-          hlfir::getFortranElementOrSequenceType(dummyTypeWithActualRank));
-      entity = hlfir::Entity{builder.create<fir::ReboxOp>(
-          loc, boxType, entity, /*shape=*/mlir::Value{},
-          /*slice=*/mlir::Value{})};
+      entity = genSetDynamicTypeToDummyType(entity);
       entity = genCopyIn(entity, /*doCopyOut=*/false);
     }
   }
diff --git a/flang/test/Lower/HLFIR/calls-poly-to-nonpoly.f90 b/flang/test/Lower/HLFIR/calls-poly-to-nonpoly.f90
new file mode 100644
index 0000000000000..3c60a84692bdb
--- /dev/null
+++ b/flang/test/Lower/HLFIR/calls-poly-to-nonpoly.f90
@@ -0,0 +1,26 @@
+! Test passing polymorphic variable for non-polymorphic dummy arguments:
+! RUN: bbc -emit-hlfir -o - -I nowhere %s | FileCheck %s
+
+subroutine test_sequence_association(x)
+  type t
+    integer :: i
+  end type
+  interface
+    subroutine sequence_assoc(x, n)
+      import :: t
+      type(t) :: x(n)
+    end subroutine
+  end interface
+  class(t) :: x(:, :)
+  call sequence_assoc(x, 100)
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest_sequence_association(
+! CHECK-SAME:                                            %[[VAL_0:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>
+! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>>
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[REBOX:.*]] = fir.rebox %[[VAL_3]]#0 : (!fir.class<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>) -> !fir.box<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.copy_in %[[REBOX]] to %[[VAL_1]] : (!fir.box<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>>>) -> (!fir.box<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>, i1)
+! CHECK:           %[[VAL_6:.*]] = fir.box_addr %[[VAL_5]]#0 : (!fir.box<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>) -> !fir.ref<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>) -> !fir.ref<!fir.array<?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>
+! CHECK:           fir.call @_QPsequence_assoc(%[[VAL_7]], %{{.*}})
+! CHECK:           hlfir.copy_out %[[VAL_1]], %[[VAL_5]]#1 to %[[REBOX]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>>>, i1, !fir.box<!fir.array<?x?x!fir.type<_QFtest_sequence_associationTt{i:i32}>>>) -> ()

From dc82c774a74ad7e94d4c555e4cae1025e30f3876 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Mon, 22 Jul 2024 12:03:59 +0100
Subject: [PATCH 400/486] [Clang][NEON] Add neon target guard to intrinsics
 (#98624)

This patch improves reported error when NEON intrinsics are used without
neon target feature.
---
 clang/include/clang/Basic/arm_neon.td         | 64 +++++++++----------
 clang/include/clang/Basic/arm_neon_incl.td    |  2 +-
 .../aarch64-neon-without-target-feature.cpp   | 34 ++++++++++
 clang/utils/TableGen/NeonEmitter.cpp          |  6 +-
 4 files changed, 70 insertions(+), 36 deletions(-)
 create mode 100644 clang/test/Sema/aarch64-neon-without-target-feature.cpp

diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index 6390ba3f9fe5e..3098fa67e6a51 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -289,7 +289,7 @@ def SPLATQ : WInst<"splat_laneq", ".(!Q)I",
                    "UcUsUicsilPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUlhdQhQdPlQPl"> {
   let isLaneQ = 1;
 }
-let TargetGuard = "bf16" in {
+let TargetGuard = "bf16,neon" in {
   def SPLAT_BF  : WInst<"splat_lane", ".(!q)I", "bQb">;
   def SPLATQ_BF : WInst<"splat_laneq", ".(!Q)I", "bQb"> {
     let isLaneQ = 1;
@@ -323,7 +323,7 @@ def VMLSL    : SOpInst<"vmlsl", "(>Q)(>Q)..", "csiUcUsUi", OP_MLSL>;
 def VQDMULH  : SInst<"vqdmulh", "...", "siQsQi">;
 def VQRDMULH : SInst<"vqrdmulh", "...", "siQsQi">;
 
-let TargetGuard = "v8.1a" in {
+let TargetGuard = "v8.1a,neon" in {
 def VQRDMLAH : SInst<"vqrdmlah", "....", "siQsQi">;
 def VQRDMLSH : SInst<"vqrdmlsh", "....", "siQsQi">;
 }
@@ -614,7 +614,7 @@ def A64_VQDMULH_LANE  : SInst<"vqdmulh_lane", "..(!q)I", "siQsQi">;
 def A64_VQRDMULH_LANE : SInst<"vqrdmulh_lane", "..(!q)I", "siQsQi">;
 }
 
-let TargetGuard = "v8.1a" in {
+let TargetGuard = "v8.1a,neon" in {
 def VQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "...qI", "siQsQi", OP_QRDMLAH_LN>;
 def VQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "...qI", "siQsQi", OP_QRDMLSH_LN>;
 }
@@ -957,7 +957,7 @@ def VQDMLAL_HIGH : SOpInst<"vqdmlal_high", "(>Q)(>Q)QQ", "si", OP_QDMLALHi>;
 def VQDMLAL_HIGH_N : SOpInst<"vqdmlal_high_n", "(>Q)(>Q)Q1", "si", OP_QDMLALHi_N>;
 def VQDMLSL_HIGH : SOpInst<"vqdmlsl_high", "(>Q)(>Q)QQ", "si", OP_QDMLSLHi>;
 def VQDMLSL_HIGH_N : SOpInst<"vqdmlsl_high_n", "(>Q)(>Q)Q1", "si", OP_QDMLSLHi_N>;
-let TargetGuard = "aes" in {
+let TargetGuard = "aes,neon" in {
   def VMULL_P64    : SInst<"vmull", "(1>)11", "Pl">;
   def VMULL_HIGH_P64 : SOpInst<"vmull_high", "(1>)..", "HPl", OP_MULLHi_P64>;
 }
@@ -1091,7 +1091,7 @@ let isLaneQ = 1 in {
 def VQDMULH_LANEQ  : SInst<"vqdmulh_laneq", "..QI", "siQsQi">;
 def VQRDMULH_LANEQ : SInst<"vqrdmulh_laneq", "..QI", "siQsQi">;
 }
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a,neon" in {
 def VQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "...QI", "siQsQi", OP_QRDMLAH_LN> {
   let isLaneQ = 1;
 }
@@ -1122,14 +1122,14 @@ def VEXT_A64 : WInst<"vext", "...I", "dQdPlQPl">;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Crypto
-let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "aes" in {
+let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "aes,neon" in {
 def AESE : SInst<"vaese", "...", "QUc">;
 def AESD : SInst<"vaesd", "...", "QUc">;
 def AESMC : SInst<"vaesmc", "..", "QUc">;
 def AESIMC : SInst<"vaesimc", "..", "QUc">;
 }
 
-let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "sha2" in {
+let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "sha2,neon" in {
 def SHA1H : SInst<"vsha1h", "11", "Ui">;
 def SHA1SU1 : SInst<"vsha1su1", "...", "QUi">;
 def SHA256SU0 : SInst<"vsha256su0", "...", "QUi">;
@@ -1143,7 +1143,7 @@ def SHA256H2 : SInst<"vsha256h2", "....", "QUi">;
 def SHA256SU1 : SInst<"vsha256su1", "....", "QUi">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3,neon" in {
 def BCAX : SInst<"vbcax", "....", "QUcQUsQUiQUlQcQsQiQl">;
 def EOR3 : SInst<"veor3", "....", "QUcQUsQUiQUlQcQsQiQl">;
 def RAX1 : SInst<"vrax1", "...", "QUl">;
@@ -1153,14 +1153,14 @@ def XAR :  SInst<"vxar", "...I", "QUl">;
 }
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3,neon" in {
 def SHA512SU0 : SInst<"vsha512su0", "...", "QUl">;
 def SHA512su1 : SInst<"vsha512su1", "....", "QUl">;
 def SHA512H : SInst<"vsha512h", "....", "QUl">;
 def SHA512H2 : SInst<"vsha512h2", "....", "QUl">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4,neon" in {
 def SM3SS1 : SInst<"vsm3ss1", "....", "QUi">;
 def SM3TT1A : SInst<"vsm3tt1a", "....I", "QUi">;
 def SM3TT1B : SInst<"vsm3tt1b", "....I", "QUi">;
@@ -1170,7 +1170,7 @@ def SM3PARTW1 : SInst<"vsm3partw1", "....", "QUi">;
 def SM3PARTW2 : SInst<"vsm3partw2", "....", "QUi">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4,neon" in {
 def SM4E : SInst<"vsm4e", "...", "QUi">;
 def SM4EKEY : SInst<"vsm4ekey", "...", "QUi">;
 }
@@ -1227,7 +1227,7 @@ def FRINTZ_S64 : SInst<"vrnd", "..", "dQd">;
 def FRINTI_S64 : SInst<"vrndi", "..", "dQd">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.5a" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.5a,neon" in {
 def FRINT32X_S32 : SInst<"vrnd32x", "..", "fQf">;
 def FRINT32Z_S32 : SInst<"vrnd32z", "..", "fQf">;
 def FRINT64X_S32 : SInst<"vrnd64x", "..", "fQf">;
@@ -1401,7 +1401,7 @@ def SCALAR_SQDMULH : SInst<"vqdmulh", "111", "SsSi">;
 // Scalar Integer Saturating Rounding Doubling Multiply Half High
 def SCALAR_SQRDMULH : SInst<"vqrdmulh", "111", "SsSi">;
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a,neon" in {
 ////////////////////////////////////////////////////////////////////////////////
 // Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
 def SCALAR_SQRDMLAH : SInst<"vqrdmlah", "1111", "SsSi">;
@@ -1632,7 +1632,7 @@ def SCALAR_SQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "11QI", "SsSi", OP_SCALAR_
   let isLaneQ = 1;
 }
 
-let TargetGuard = "v8.1a" in {
+let TargetGuard = "v8.1a,neon" in {
 // Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
 def SCALAR_SQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "111.I", "SsSi", OP_SCALAR_QRDMLAH_LN>;
 def SCALAR_SQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "111QI", "SsSi", OP_SCALAR_QRDMLAH_LN> {
@@ -1654,7 +1654,7 @@ def SCALAR_VDUP_LANEQ : IInst<"vdup_laneq", "1QI", "ScSsSiSlSfSdSUcSUsSUiSUlSPcS
 } // ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)"
 
 // ARMv8.2-A FP16 vector intrinsics for A32/A64.
-let TargetGuard = "fullfp16" in {
+let TargetGuard = "fullfp16,neon" in {
 
   // ARMv8.2-A FP16 one-operand vector intrinsics.
 
@@ -1679,7 +1679,7 @@ let TargetGuard = "fullfp16" in {
   def VCVTP_U16    : SInst<"vcvtp_u16", "U.", "hQh">;
 
   // Vector rounding
-  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)", TargetGuard = "fullfp16" in {
+  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)", TargetGuard = "fullfp16,neon" in {
     def FRINTZH      : SInst<"vrnd",  "..", "hQh">;
     def FRINTNH      : SInst<"vrndn", "..", "hQh">;
     def FRINTAH      : SInst<"vrnda", "..", "hQh">;
@@ -1728,7 +1728,7 @@ let TargetGuard = "fullfp16" in {
   // Max/Min
   def VMAXH         : SInst<"vmax", "...", "hQh">;
   def VMINH         : SInst<"vmin", "...", "hQh">;
-  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN)", TargetGuard = "fullfp16" in {
+  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN)", TargetGuard = "fullfp16,neon" in {
     def FMAXNMH       : SInst<"vmaxnm", "...", "hQh">;
     def FMINNMH       : SInst<"vminnm", "...", "hQh">;
   }
@@ -1775,7 +1775,7 @@ def VEXTH      : WInst<"vext", "...I", "hQh">;
 def VREV64H    : WOpInst<"vrev64", "..", "hQh", OP_REV64>;
 
 // ARMv8.2-A FP16 vector intrinsics for A64 only.
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fullfp16" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fullfp16,neon" in {
 
   // Vector rounding
   def FRINTIH      : SInst<"vrndi", "..", "hQh">;
@@ -1872,11 +1872,11 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)" in {
 }
 
 // v8.2-A dot product instructions.
-let TargetGuard = "dotprod" in {
+let TargetGuard = "dotprod,neon" in {
   def DOT : SInst<"vdot", "..(<<)(<<)", "iQiUiQUi">;
   def DOT_LANE : SOpInst<"vdot_lane", "..(<<)(<<q)I", "iUiQiQUi", OP_DOT_LN>;
 }
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "dotprod" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "dotprod,neon" in {
   // Variants indexing into a 128-bit vector are A64 only.
   def UDOT_LANEQ : SOpInst<"vdot_laneq", "..(<<)(<<Q)I", "iUiQiQUi", OP_DOT_LNQ> {
     let isLaneQ = 1;
@@ -1884,7 +1884,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "d
 }
 
 // v8.2-A FP16 fused multiply-add long instructions.
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fp16fml" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fp16fml,neon" in {
   def VFMLAL_LOW  : SInst<"vfmlal_low",  ">>..", "hQh">;
   def VFMLSL_LOW  : SInst<"vfmlsl_low",  ">>..", "hQh">;
   def VFMLAL_HIGH : SInst<"vfmlal_high", ">>..", "hQh">;
@@ -1909,7 +1909,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "f
   }
 }
 
-let TargetGuard = "i8mm" in {
+let TargetGuard = "i8mm,neon" in {
   def VMMLA   : SInst<"vmmla", "..(<<)(<<)", "QUiQi">;
   def VUSMMLA : SInst<"vusmmla", "..(<<U)(<<)", "Qi">;
 
@@ -1926,7 +1926,7 @@ let TargetGuard = "i8mm" in {
   }
 }
 
-let TargetGuard = "bf16" in {
+let TargetGuard = "bf16,neon" in {
   def VDOT_BF : SInst<"vbfdot", "..BB", "fQf">;
   def VDOT_LANE_BF : SOpInst<"vbfdot_lane", "..B(Bq)I", "fQf", OP_BFDOT_LN>;
   def VDOT_LANEQ_BF : SOpInst<"vbfdot_laneq", "..B(BQ)I", "fQf", OP_BFDOT_LNQ> {
@@ -1970,7 +1970,7 @@ multiclass VCMLA_ROTS<string type, string lanety, string laneqty> {
 }
 
 // v8.3-A Vector complex addition intrinsics
-let TargetGuard = "v8.3a,fullfp16" in {
+let TargetGuard = "v8.3a,fullfp16,neon" in {
   def VCADD_ROT90_FP16   : SInst<"vcadd_rot90", "...", "h">;
   def VCADD_ROT270_FP16  : SInst<"vcadd_rot270", "...", "h">;
   def VCADDQ_ROT90_FP16  : SInst<"vcaddq_rot90", "QQQ", "h">;
@@ -1978,7 +1978,7 @@ let TargetGuard = "v8.3a,fullfp16" in {
 
   defm VCMLA_FP16  : VCMLA_ROTS<"h", "uint32x2_t", "uint32x4_t">;
 }
-let TargetGuard = "v8.3a" in {
+let TargetGuard = "v8.3a,neon" in {
   def VCADD_ROT90   : SInst<"vcadd_rot90", "...", "f">;
   def VCADD_ROT270  : SInst<"vcadd_rot270", "...", "f">;
   def VCADDQ_ROT90  : SInst<"vcaddq_rot90", "QQQ", "f">;
@@ -1986,7 +1986,7 @@ let TargetGuard = "v8.3a" in {
 
   defm VCMLA_F32        : VCMLA_ROTS<"f", "uint64x1_t", "uint64x2_t">;
 }
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.3a" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.3a,neon" in {
   def VCADDQ_ROT90_FP64  : SInst<"vcaddq_rot90", "QQQ", "d">;
   def VCADDQ_ROT270_FP64 : SInst<"vcaddq_rot270", "QQQ", "d">;
 
@@ -1994,7 +1994,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v
 }
 
 // V8.2-A BFloat intrinsics
-let TargetGuard = "bf16" in {
+let TargetGuard = "bf16,neon" in {
   def VCREATE_BF : NoTestOpInst<"vcreate", ".(IU>)", "b", OP_CAST> {
     let BigEndianSafe = 1;
   }
@@ -2058,14 +2058,14 @@ let TargetGuard = "bf16" in {
   def SCALAR_CVT_F32_BF16 : SOpInst<"vcvtah_f32", "(1F>)(1!)", "b", OP_CVT_F32_BF16>;
 }
 
-let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16" in {
+let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
   def VCVT_BF16_F32_A32_INTERNAL : WInst<"__a32_vcvt_bf16", "BQ", "f">;
   def VCVT_BF16_F32_A32 : SOpInst<"vcvt_bf16", "BQ", "f", OP_VCVT_BF16_F32_A32>;
   def VCVT_LOW_BF16_F32_A32 : SOpInst<"vcvt_low_bf16",  "BQ", "Qf", OP_VCVT_BF16_F32_LO_A32>;
   def VCVT_HIGH_BF16_F32_A32 : SOpInst<"vcvt_high_bf16", "BBQ", "Qf", OP_VCVT_BF16_F32_HI_A32>;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
   def VCVT_LOW_BF16_F32_A64_INTERNAL : WInst<"__a64_vcvtq_low_bf16", "BQ", "Hf">;
   def VCVT_LOW_BF16_F32_A64 : SOpInst<"vcvt_low_bf16", "BQ", "Qf", OP_VCVT_BF16_F32_LO_A64>;
   def VCVT_HIGH_BF16_F32_A64 : SInst<"vcvt_high_bf16", "BBQ", "Qf">;
@@ -2077,14 +2077,14 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "b
   def COPYQ_LANEQ_BF16 : IOpInst<"vcopy_laneq", "..I.I", "Qb", OP_COPY_LN>;
 }
 
-let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16" in {
+let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
   let BigEndianSafe = 1 in {
     defm VREINTERPRET_BF : REINTERPRET_CROSS_TYPES<
         "csilUcUsUiUlhfPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQPcQPsQPl", "bQb">;
   }
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
   let BigEndianSafe = 1 in {
     defm VVREINTERPRET_BF : REINTERPRET_CROSS_TYPES<
         "csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk", "bQb">;
@@ -2092,7 +2092,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "b
 }
 
 // v8.9a/v9.4a LRCPC3 intrinsics
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "rcpc3" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "rcpc3,neon" in {
   def VLDAP1_LANE : WInst<"vldap1_lane", ".(c*!).I", "QUlQlUlldQdPlQPl">;
   def VSTL1_LANE  : WInst<"vstl1_lane", "v*(.!)I", "QUlQlUlldQdPlQPl">;
 }
diff --git a/clang/include/clang/Basic/arm_neon_incl.td b/clang/include/clang/Basic/arm_neon_incl.td
index b8155c187d1bc..3b8015daee6d9 100644
--- a/clang/include/clang/Basic/arm_neon_incl.td
+++ b/clang/include/clang/Basic/arm_neon_incl.td
@@ -265,7 +265,7 @@ class Inst <string n, string p, string t, Operation o> {
   string Prototype = p;
   string Types = t;
   string ArchGuard = "";
-  string TargetGuard = "";
+  string TargetGuard = "neon";
 
   Operation Operation = o;
   bit BigEndianSafe = 0;
diff --git a/clang/test/Sema/aarch64-neon-without-target-feature.cpp b/clang/test/Sema/aarch64-neon-without-target-feature.cpp
new file mode 100644
index 0000000000000..64f76a7900faf
--- /dev/null
+++ b/clang/test/Sema/aarch64-neon-without-target-feature.cpp
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +dotprod  -target-feature +fullfp16 -target-feature +fp16fml -target-feature +i8mm -target-feature +bf16 -verify -emit-llvm -o - %s
+
+// This test is testing the diagnostics that Clang emits when compiling without '+neon'.
+
+#include <arm_neon.h>
+
+void undefined(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t v16i8, uint8x8_t v8i8, float32x2_t v2f32, float32x4_t v4f32, float16x4_t v4f16, float64x2_t v2f64, bfloat16x4_t v4bf16, __bf16 bf16, poly64_t poly64, poly64x2_t poly64x2) {
+  // dotprod
+  vdot_u32(v2i32, v8i8, v8i8); // expected-error {{always_inline function 'vdot_u32' requires target feature 'neon'}}
+  vdot_laneq_u32(v2i32, v8i8, v16i8, 1); // expected-error {{always_inline function 'vdot_u32' requires target feature 'neon'}} expected-error {{'__builtin_neon_splat_laneq_v' needs target feature neon}}
+  // fp16
+  vceqz_f16(v4f16); // expected-error {{always_inline function 'vceqz_f16' requires target feature 'neon'}}
+  vrnd_f16(v4f16); // expected-error {{always_inline function 'vrnd_f16' requires target feature 'neon'}}
+  vmaxnm_f16(v4f16, v4f16); // expected-error {{always_inline function 'vmaxnm_f16' requires target feature 'neon'}}
+  vrndi_f16(v4f16); // expected-error {{always_inline function 'vrndi_f16' requires target feature 'neon'}}
+  // fp16fml depends on fp-armv8
+  vfmlal_low_f16(v2f32, v4f16, v4f16); // expected-error {{always_inline function 'vfmlal_low_f16' requires target feature 'neon'}}
+  // i8mm
+  vmmlaq_s32(v4i32, v8i16, v8i16); // expected-error {{always_inline function 'vmmlaq_s32' requires target feature 'neon'}}
+  vusdot_laneq_s32(v2i32, v8i8, v8i16, 0); // expected-error {{always_inline function 'vusdot_s32' requires target feature 'neon'}} expected-error {{'__builtin_neon_splat_laneq_v' needs target feature neon}}
+  // bf16
+  vbfdot_f32(v2f32, v4bf16, v4bf16); // expected-error {{always_inline function 'vbfdot_f32' requires target feature 'neon'}}
+  vcreate_bf16(10);
+  vdup_lane_bf16(v4bf16, 2); // expected-error {{'__builtin_neon_splat_lane_bf16' needs target feature bf16,neon}}
+  vdup_n_bf16(bf16); // expected-error {{always_inline function 'vdup_n_bf16' requires target feature 'neon'}}
+  vld1_bf16(0); // expected-error {{'__builtin_neon_vld1_bf16' needs target feature bf16,neon}}
+  vcvt_f32_bf16(v4bf16); // expected-error {{always_inline function 'vcvt_f32_bf16' requires target feature 'neon'}}
+  vcvt_bf16_f32(v4f32); // expected-error {{always_inline function 'vcvt_bf16_f32' requires target feature 'neon'}}
+  vmull_p64(poly64, poly64);  // expected-error {{always_inline function 'vmull_p64' requires target feature 'neon'}}
+  vmull_high_p64(poly64x2, poly64x2);  // expected-error {{always_inline function 'vmull_high_p64' requires target feature 'neon'}}
+  vtrn1_s8(v8i8, v8i8); // expected-error {{always_inline function 'vtrn1_s8' requires target feature 'neon'}}
+  vqabsq_s16(v8i16); // expected-error {{always_inline function 'vqabsq_s16' requires target feature 'neon'}}
+  vbslq_s16(v8i16, v8i16, v8i16);// expected-error {{always_inline function 'vbslq_s16' requires target feature 'neon'}}
+}
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 626031d38cf00..30fbb8c5d65e5 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -952,7 +952,7 @@ std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
   char typeCode = '\0';
   bool printNumber = true;
 
-  if (CK == ClassB && TargetGuard == "")
+  if (CK == ClassB && TargetGuard == "neon")
     return "";
 
   if (T.isBFloat16())
@@ -976,7 +976,7 @@ std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
       break;
     }
   }
-  if (CK == ClassB && TargetGuard == "") {
+  if (CK == ClassB && TargetGuard == "neon") {
     typeCode = '\0';
   }
 
@@ -1078,7 +1078,7 @@ std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const {
     S += "_" + getInstTypeCode(InBaseType, LocalCK);
   }
 
-  if (LocalCK == ClassB && TargetGuard == "")
+  if (LocalCK == ClassB && TargetGuard == "neon")
     S += "_v";
 
   // Insert a 'q' before the first '_' character so that it ends up before

From 15b41d207e15c96cf6f6a7e72cbf3c0044762df5 Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <christudasan.devadasan@amd.com>
Date: Mon, 22 Jul 2024 16:49:39 +0530
Subject: [PATCH 401/486] [CodeGen] change prototype of regalloc filter
 function (#93525)

[CodeGen] Change the prototype of regalloc filter function

Change the prototype of the filter function so that we can
filter not just by RegClass. We need to implement more
complicated filter based upon some other info associated
with each register.

Patch provided by: Gang Chen (gangc@amd.com)
---
 llvm/include/llvm/CodeGen/Passes.h             |  6 +++---
 llvm/include/llvm/CodeGen/RegAllocCommon.h     |  6 ++++--
 llvm/include/llvm/CodeGen/RegAllocFast.h       |  2 +-
 llvm/include/llvm/Passes/PassBuilder.h         | 10 +++++-----
 llvm/lib/CodeGen/RegAllocBase.h                |  9 +++++----
 llvm/lib/CodeGen/RegAllocBasic.cpp             | 10 ++++------
 llvm/lib/CodeGen/RegAllocFast.cpp              | 16 ++++++++--------
 llvm/lib/CodeGen/RegAllocGreedy.cpp            | 10 ++++------
 llvm/lib/CodeGen/RegAllocGreedy.h              |  2 +-
 llvm/lib/Passes/PassBuilder.cpp                |  4 ++--
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 15 +++++++++------
 llvm/lib/Target/RISCV/RISCVTargetMachine.cpp   |  6 ++++--
 llvm/lib/Target/X86/X86TargetMachine.cpp       |  6 ++++--
 13 files changed, 54 insertions(+), 48 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index f850767270a4f..cafb9781698a2 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -205,20 +205,20 @@ namespace llvm {
   /// possible. It is best suited for debug code where live ranges are short.
   ///
   FunctionPass *createFastRegisterAllocator();
-  FunctionPass *createFastRegisterAllocator(RegClassFilterFunc F,
+  FunctionPass *createFastRegisterAllocator(RegAllocFilterFunc F,
                                             bool ClearVirtRegs);
 
   /// BasicRegisterAllocation Pass - This pass implements a degenerate global
   /// register allocator using the basic regalloc framework.
   ///
   FunctionPass *createBasicRegisterAllocator();
-  FunctionPass *createBasicRegisterAllocator(RegClassFilterFunc F);
+  FunctionPass *createBasicRegisterAllocator(RegAllocFilterFunc F);
 
   /// Greedy register allocation pass - This pass implements a global register
   /// allocator for optimized builds.
   ///
   FunctionPass *createGreedyRegisterAllocator();
-  FunctionPass *createGreedyRegisterAllocator(RegClassFilterFunc F);
+  FunctionPass *createGreedyRegisterAllocator(RegAllocFilterFunc F);
 
   /// PBQPRegisterAllocation Pass - This pass implements the Partitioned Boolean
   /// Quadratic Prograaming (PBQP) based register allocator.
diff --git a/llvm/include/llvm/CodeGen/RegAllocCommon.h b/llvm/include/llvm/CodeGen/RegAllocCommon.h
index ad533eab1861c..6c5cb295f5290 100644
--- a/llvm/include/llvm/CodeGen/RegAllocCommon.h
+++ b/llvm/include/llvm/CodeGen/RegAllocCommon.h
@@ -9,18 +9,20 @@
 #ifndef LLVM_CODEGEN_REGALLOCCOMMON_H
 #define LLVM_CODEGEN_REGALLOCCOMMON_H
 
+#include "llvm/CodeGen/Register.h"
 #include <functional>
 
 namespace llvm {
 
 class TargetRegisterClass;
 class TargetRegisterInfo;
+class MachineRegisterInfo;
 
 /// Filter function for register classes during regalloc. Default register class
 /// filter is nullptr, where all registers should be allocated.
 typedef std::function<bool(const TargetRegisterInfo &TRI,
-                           const TargetRegisterClass &RC)>
-    RegClassFilterFunc;
+                           const MachineRegisterInfo &MRI, const Register Reg)>
+    RegAllocFilterFunc;
 }
 
 #endif // LLVM_CODEGEN_REGALLOCCOMMON_H
diff --git a/llvm/include/llvm/CodeGen/RegAllocFast.h b/llvm/include/llvm/CodeGen/RegAllocFast.h
index c62bd14d0b4cb..c99c715daacf9 100644
--- a/llvm/include/llvm/CodeGen/RegAllocFast.h
+++ b/llvm/include/llvm/CodeGen/RegAllocFast.h
@@ -15,7 +15,7 @@
 namespace llvm {
 
 struct RegAllocFastPassOptions {
-  RegClassFilterFunc Filter = nullptr;
+  RegAllocFilterFunc Filter = nullptr;
   StringRef FilterName = "all";
   bool ClearVRegs = true;
 };
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index 551d297e0c089..474a19531ff5d 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -390,9 +390,9 @@ class PassBuilder {
   /// returns false.
   Error parseAAPipeline(AAManager &AA, StringRef PipelineText);
 
-  /// Parse RegClassFilterName to get RegClassFilterFunc.
-  std::optional<RegClassFilterFunc>
-  parseRegAllocFilter(StringRef RegClassFilterName);
+  /// Parse RegAllocFilterName to get RegAllocFilterFunc.
+  std::optional<RegAllocFilterFunc>
+  parseRegAllocFilter(StringRef RegAllocFilterName);
 
   /// Print pass names.
   void printPassNames(raw_ostream &OS);
@@ -586,7 +586,7 @@ class PassBuilder {
   /// needs it. E.g. AMDGPU requires regalloc passes can handle sgpr and vgpr
   /// separately.
   void registerRegClassFilterParsingCallback(
-      const std::function<RegClassFilterFunc(StringRef)> &C) {
+      const std::function<RegAllocFilterFunc(StringRef)> &C) {
     RegClassFilterParsingCallbacks.push_back(C);
   }
 
@@ -807,7 +807,7 @@ class PassBuilder {
               2>
       MachineFunctionPipelineParsingCallbacks;
   // Callbacks to parse `filter` parameter in register allocation passes
-  SmallVector<std::function<RegClassFilterFunc(StringRef)>, 2>
+  SmallVector<std::function<RegAllocFilterFunc(StringRef)>, 2>
       RegClassFilterParsingCallbacks;
 };
 
diff --git a/llvm/lib/CodeGen/RegAllocBase.h b/llvm/lib/CodeGen/RegAllocBase.h
index 643094671d682..a1ede08a15356 100644
--- a/llvm/lib/CodeGen/RegAllocBase.h
+++ b/llvm/lib/CodeGen/RegAllocBase.h
@@ -72,7 +72,7 @@ class RegAllocBase {
 
 private:
   /// Private, callees should go through shouldAllocateRegister
-  const RegClassFilterFunc ShouldAllocateClass;
+  const RegAllocFilterFunc shouldAllocateRegisterImpl;
 
 protected:
   /// Inst which is a def of an original reg and whose defs are already all
@@ -81,7 +81,8 @@ class RegAllocBase {
   /// always available for the remat of all the siblings of the original reg.
   SmallPtrSet<MachineInstr *, 32> DeadRemats;
 
-  RegAllocBase(const RegClassFilterFunc F = nullptr) : ShouldAllocateClass(F) {}
+  RegAllocBase(const RegAllocFilterFunc F = nullptr)
+      : shouldAllocateRegisterImpl(F) {}
 
   virtual ~RegAllocBase() = default;
 
@@ -90,9 +91,9 @@ class RegAllocBase {
 
   /// Get whether a given register should be allocated
   bool shouldAllocateRegister(Register Reg) {
-    if (!ShouldAllocateClass)
+    if (!shouldAllocateRegisterImpl)
       return true;
-    return ShouldAllocateClass(*TRI, *MRI->getRegClass(Reg));
+    return shouldAllocateRegisterImpl(*TRI, *MRI, Reg);
   }
 
   // The top-level driver. The output is a VirtRegMap that us updated with
diff --git a/llvm/lib/CodeGen/RegAllocBasic.cpp b/llvm/lib/CodeGen/RegAllocBasic.cpp
index e78a447969fb2..caf9c32a5a349 100644
--- a/llvm/lib/CodeGen/RegAllocBasic.cpp
+++ b/llvm/lib/CodeGen/RegAllocBasic.cpp
@@ -74,7 +74,7 @@ class RABasic : public MachineFunctionPass,
   void LRE_WillShrinkVirtReg(Register) override;
 
 public:
-  RABasic(const RegClassFilterFunc F = nullptr);
+  RABasic(const RegAllocFilterFunc F = nullptr);
 
   /// Return the pass name.
   StringRef getPassName() const override { return "Basic Register Allocator"; }
@@ -168,10 +168,8 @@ void RABasic::LRE_WillShrinkVirtReg(Register VirtReg) {
   enqueue(&LI);
 }
 
-RABasic::RABasic(RegClassFilterFunc F):
-  MachineFunctionPass(ID),
-  RegAllocBase(F) {
-}
+RABasic::RABasic(RegAllocFilterFunc F)
+    : MachineFunctionPass(ID), RegAllocBase(F) {}
 
 void RABasic::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesCFG();
@@ -333,6 +331,6 @@ FunctionPass* llvm::createBasicRegisterAllocator() {
   return new RABasic();
 }
 
-FunctionPass* llvm::createBasicRegisterAllocator(RegClassFilterFunc F) {
+FunctionPass *llvm::createBasicRegisterAllocator(RegAllocFilterFunc F) {
   return new RABasic(F);
 }
diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp
index b4660d4085ffd..6e5ce72240d27 100644
--- a/llvm/lib/CodeGen/RegAllocFast.cpp
+++ b/llvm/lib/CodeGen/RegAllocFast.cpp
@@ -177,9 +177,9 @@ class InstrPosIndexes {
 
 class RegAllocFastImpl {
 public:
-  RegAllocFastImpl(const RegClassFilterFunc F = nullptr,
+  RegAllocFastImpl(const RegAllocFilterFunc F = nullptr,
                    bool ClearVirtRegs_ = true)
-      : ShouldAllocateClass(F), StackSlotForVirtReg(-1),
+      : ShouldAllocateRegisterImpl(F), StackSlotForVirtReg(-1),
         ClearVirtRegs(ClearVirtRegs_) {}
 
 private:
@@ -188,7 +188,7 @@ class RegAllocFastImpl {
   const TargetRegisterInfo *TRI = nullptr;
   const TargetInstrInfo *TII = nullptr;
   RegisterClassInfo RegClassInfo;
-  const RegClassFilterFunc ShouldAllocateClass;
+  const RegAllocFilterFunc ShouldAllocateRegisterImpl;
 
   /// Basic block currently being allocated.
   MachineBasicBlock *MBB = nullptr;
@@ -397,7 +397,7 @@ class RegAllocFast : public MachineFunctionPass {
 public:
   static char ID;
 
-  RegAllocFast(const RegClassFilterFunc F = nullptr, bool ClearVirtRegs_ = true)
+  RegAllocFast(const RegAllocFilterFunc F = nullptr, bool ClearVirtRegs_ = true)
       : MachineFunctionPass(ID), Impl(F, ClearVirtRegs_) {}
 
   bool runOnMachineFunction(MachineFunction &MF) override {
@@ -440,10 +440,10 @@ INITIALIZE_PASS(RegAllocFast, "regallocfast", "Fast Register Allocator", false,
 
 bool RegAllocFastImpl::shouldAllocateRegister(const Register Reg) const {
   assert(Reg.isVirtual());
-  if (!ShouldAllocateClass)
+  if (!ShouldAllocateRegisterImpl)
     return true;
-  const TargetRegisterClass &RC = *MRI->getRegClass(Reg);
-  return ShouldAllocateClass(*TRI, RC);
+
+  return ShouldAllocateRegisterImpl(*TRI, *MRI, Reg);
 }
 
 void RegAllocFastImpl::setPhysRegState(MCPhysReg PhysReg, unsigned NewState) {
@@ -1841,7 +1841,7 @@ void RegAllocFastPass::printPipeline(
 
 FunctionPass *llvm::createFastRegisterAllocator() { return new RegAllocFast(); }
 
-FunctionPass *llvm::createFastRegisterAllocator(RegClassFilterFunc Ftor,
+FunctionPass *llvm::createFastRegisterAllocator(RegAllocFilterFunc Ftor,
                                                 bool ClearVirtRegs) {
   return new RegAllocFast(Ftor, ClearVirtRegs);
 }
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 310050d92d744..5001b4fec58f2 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -192,14 +192,12 @@ FunctionPass* llvm::createGreedyRegisterAllocator() {
   return new RAGreedy();
 }
 
-FunctionPass *llvm::createGreedyRegisterAllocator(RegClassFilterFunc Ftor) {
+FunctionPass *llvm::createGreedyRegisterAllocator(RegAllocFilterFunc Ftor) {
   return new RAGreedy(Ftor);
 }
 
-RAGreedy::RAGreedy(RegClassFilterFunc F):
-  MachineFunctionPass(ID),
-  RegAllocBase(F) {
-}
+RAGreedy::RAGreedy(RegAllocFilterFunc F)
+    : MachineFunctionPass(ID), RegAllocBase(F) {}
 
 void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesCFG();
@@ -2306,7 +2304,7 @@ void RAGreedy::tryHintRecoloring(const LiveInterval &VirtReg) {
     if (Reg.isPhysical())
       continue;
 
-    // This may be a skipped class
+    // This may be a skipped register.
     if (!VRM->hasPhys(Reg)) {
       assert(!shouldAllocateRegister(Reg) &&
              "We have an unallocated variable which should have been handled");
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index ac300c0024f5a..2e7608a53e9ce 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -281,7 +281,7 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public MachineFunctionPass,
   bool ReverseLocalAssignment = false;
 
 public:
-  RAGreedy(const RegClassFilterFunc F = nullptr);
+  RAGreedy(const RegAllocFilterFunc F = nullptr);
 
   /// Return the pass name.
   StringRef getPassName() const override { return "Greedy Register Allocator"; }
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index a9d3f8ec3a4ec..63d2f633784e9 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1180,7 +1180,7 @@ parseRegAllocFastPassOptions(PassBuilder &PB, StringRef Params) {
     std::tie(ParamName, Params) = Params.split(';');
 
     if (ParamName.consume_front("filter=")) {
-      std::optional<RegClassFilterFunc> Filter =
+      std::optional<RegAllocFilterFunc> Filter =
           PB.parseRegAllocFilter(ParamName);
       if (!Filter) {
         return make_error<StringError>(
@@ -2190,7 +2190,7 @@ Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
   return Error::success();
 }
 
-std::optional<RegClassFilterFunc>
+std::optional<RegAllocFilterFunc>
 PassBuilder::parseRegAllocFilter(StringRef FilterName) {
   if (FilterName == "all")
     return nullptr;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 192996f84c5f3..c8fb68d1c0b0c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -85,16 +85,19 @@ class VGPRRegisterRegAlloc : public RegisterRegAllocBase<VGPRRegisterRegAlloc> {
 };
 
 static bool onlyAllocateSGPRs(const TargetRegisterInfo &TRI,
-                              const TargetRegisterClass &RC) {
-  return static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(&RC);
+                              const MachineRegisterInfo &MRI,
+                              const Register Reg) {
+  const TargetRegisterClass *RC = MRI.getRegClass(Reg);
+  return static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
 }
 
 static bool onlyAllocateVGPRs(const TargetRegisterInfo &TRI,
-                              const TargetRegisterClass &RC) {
-  return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(&RC);
+                              const MachineRegisterInfo &MRI,
+                              const Register Reg) {
+  const TargetRegisterClass *RC = MRI.getRegClass(Reg);
+  return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
 }
 
-
 /// -{sgpr|vgpr}-regalloc=... command line option.
 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
 
@@ -749,7 +752,7 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
       });
 
   PB.registerRegClassFilterParsingCallback(
-      [](StringRef FilterName) -> RegClassFilterFunc {
+      [](StringRef FilterName) -> RegAllocFilterFunc {
         if (FilterName == "sgpr")
           return onlyAllocateSGPRs;
         if (FilterName == "vgpr")
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index bc3699db6f91e..21fbf47875e68 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -279,8 +279,10 @@ class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
 };
 
 static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
-                               const TargetRegisterClass &RC) {
-  return RISCVRegisterInfo::isRVVRegClass(&RC);
+                               const MachineRegisterInfo &MRI,
+                               const Register Reg) {
+  const TargetRegisterClass *RC = MRI.getRegClass(Reg);
+  return RISCVRegisterInfo::isRVVRegClass(RC);
 }
 
 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 4c77f40fd32a3..d2d59ff3b93cf 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -676,8 +676,10 @@ std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const {
 }
 
 static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI,
-                                      const TargetRegisterClass &RC) {
-  return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(&RC);
+                                      const MachineRegisterInfo &MRI,
+                                      const Register Reg) {
+  const TargetRegisterClass *RC = MRI.getRegClass(Reg);
+  return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(RC);
 }
 
 bool X86PassConfig::addRegAssignAndRewriteOptimized() {

From c1622cae10c953f327c7ef24d57508eebd84f244 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Mon, 22 Jul 2024 12:32:47 +0100
Subject: [PATCH 402/486] Revert "[Clang][NEON] Add neon target guard to
 intrinsics" (#99864)

Reverts llvm/llvm-project#98624
---
 clang/include/clang/Basic/arm_neon.td         | 64 +++++++++----------
 clang/include/clang/Basic/arm_neon_incl.td    |  2 +-
 .../aarch64-neon-without-target-feature.cpp   | 34 ----------
 clang/utils/TableGen/NeonEmitter.cpp          |  6 +-
 4 files changed, 36 insertions(+), 70 deletions(-)
 delete mode 100644 clang/test/Sema/aarch64-neon-without-target-feature.cpp

diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index 3098fa67e6a51..6390ba3f9fe5e 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -289,7 +289,7 @@ def SPLATQ : WInst<"splat_laneq", ".(!Q)I",
                    "UcUsUicsilPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUlhdQhQdPlQPl"> {
   let isLaneQ = 1;
 }
-let TargetGuard = "bf16,neon" in {
+let TargetGuard = "bf16" in {
   def SPLAT_BF  : WInst<"splat_lane", ".(!q)I", "bQb">;
   def SPLATQ_BF : WInst<"splat_laneq", ".(!Q)I", "bQb"> {
     let isLaneQ = 1;
@@ -323,7 +323,7 @@ def VMLSL    : SOpInst<"vmlsl", "(>Q)(>Q)..", "csiUcUsUi", OP_MLSL>;
 def VQDMULH  : SInst<"vqdmulh", "...", "siQsQi">;
 def VQRDMULH : SInst<"vqrdmulh", "...", "siQsQi">;
 
-let TargetGuard = "v8.1a,neon" in {
+let TargetGuard = "v8.1a" in {
 def VQRDMLAH : SInst<"vqrdmlah", "....", "siQsQi">;
 def VQRDMLSH : SInst<"vqrdmlsh", "....", "siQsQi">;
 }
@@ -614,7 +614,7 @@ def A64_VQDMULH_LANE  : SInst<"vqdmulh_lane", "..(!q)I", "siQsQi">;
 def A64_VQRDMULH_LANE : SInst<"vqrdmulh_lane", "..(!q)I", "siQsQi">;
 }
 
-let TargetGuard = "v8.1a,neon" in {
+let TargetGuard = "v8.1a" in {
 def VQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "...qI", "siQsQi", OP_QRDMLAH_LN>;
 def VQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "...qI", "siQsQi", OP_QRDMLSH_LN>;
 }
@@ -957,7 +957,7 @@ def VQDMLAL_HIGH : SOpInst<"vqdmlal_high", "(>Q)(>Q)QQ", "si", OP_QDMLALHi>;
 def VQDMLAL_HIGH_N : SOpInst<"vqdmlal_high_n", "(>Q)(>Q)Q1", "si", OP_QDMLALHi_N>;
 def VQDMLSL_HIGH : SOpInst<"vqdmlsl_high", "(>Q)(>Q)QQ", "si", OP_QDMLSLHi>;
 def VQDMLSL_HIGH_N : SOpInst<"vqdmlsl_high_n", "(>Q)(>Q)Q1", "si", OP_QDMLSLHi_N>;
-let TargetGuard = "aes,neon" in {
+let TargetGuard = "aes" in {
   def VMULL_P64    : SInst<"vmull", "(1>)11", "Pl">;
   def VMULL_HIGH_P64 : SOpInst<"vmull_high", "(1>)..", "HPl", OP_MULLHi_P64>;
 }
@@ -1091,7 +1091,7 @@ let isLaneQ = 1 in {
 def VQDMULH_LANEQ  : SInst<"vqdmulh_laneq", "..QI", "siQsQi">;
 def VQRDMULH_LANEQ : SInst<"vqrdmulh_laneq", "..QI", "siQsQi">;
 }
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a" in {
 def VQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "...QI", "siQsQi", OP_QRDMLAH_LN> {
   let isLaneQ = 1;
 }
@@ -1122,14 +1122,14 @@ def VEXT_A64 : WInst<"vext", "...I", "dQdPlQPl">;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Crypto
-let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "aes,neon" in {
+let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "aes" in {
 def AESE : SInst<"vaese", "...", "QUc">;
 def AESD : SInst<"vaesd", "...", "QUc">;
 def AESMC : SInst<"vaesmc", "..", "QUc">;
 def AESIMC : SInst<"vaesimc", "..", "QUc">;
 }
 
-let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "sha2,neon" in {
+let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "sha2" in {
 def SHA1H : SInst<"vsha1h", "11", "Ui">;
 def SHA1SU1 : SInst<"vsha1su1", "...", "QUi">;
 def SHA256SU0 : SInst<"vsha256su0", "...", "QUi">;
@@ -1143,7 +1143,7 @@ def SHA256H2 : SInst<"vsha256h2", "....", "QUi">;
 def SHA256SU1 : SInst<"vsha256su1", "....", "QUi">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3" in {
 def BCAX : SInst<"vbcax", "....", "QUcQUsQUiQUlQcQsQiQl">;
 def EOR3 : SInst<"veor3", "....", "QUcQUsQUiQUlQcQsQiQl">;
 def RAX1 : SInst<"vrax1", "...", "QUl">;
@@ -1153,14 +1153,14 @@ def XAR :  SInst<"vxar", "...I", "QUl">;
 }
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3" in {
 def SHA512SU0 : SInst<"vsha512su0", "...", "QUl">;
 def SHA512su1 : SInst<"vsha512su1", "....", "QUl">;
 def SHA512H : SInst<"vsha512h", "....", "QUl">;
 def SHA512H2 : SInst<"vsha512h2", "....", "QUl">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4" in {
 def SM3SS1 : SInst<"vsm3ss1", "....", "QUi">;
 def SM3TT1A : SInst<"vsm3tt1a", "....I", "QUi">;
 def SM3TT1B : SInst<"vsm3tt1b", "....I", "QUi">;
@@ -1170,7 +1170,7 @@ def SM3PARTW1 : SInst<"vsm3partw1", "....", "QUi">;
 def SM3PARTW2 : SInst<"vsm3partw2", "....", "QUi">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4" in {
 def SM4E : SInst<"vsm4e", "...", "QUi">;
 def SM4EKEY : SInst<"vsm4ekey", "...", "QUi">;
 }
@@ -1227,7 +1227,7 @@ def FRINTZ_S64 : SInst<"vrnd", "..", "dQd">;
 def FRINTI_S64 : SInst<"vrndi", "..", "dQd">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.5a,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.5a" in {
 def FRINT32X_S32 : SInst<"vrnd32x", "..", "fQf">;
 def FRINT32Z_S32 : SInst<"vrnd32z", "..", "fQf">;
 def FRINT64X_S32 : SInst<"vrnd64x", "..", "fQf">;
@@ -1401,7 +1401,7 @@ def SCALAR_SQDMULH : SInst<"vqdmulh", "111", "SsSi">;
 // Scalar Integer Saturating Rounding Doubling Multiply Half High
 def SCALAR_SQRDMULH : SInst<"vqrdmulh", "111", "SsSi">;
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a" in {
 ////////////////////////////////////////////////////////////////////////////////
 // Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
 def SCALAR_SQRDMLAH : SInst<"vqrdmlah", "1111", "SsSi">;
@@ -1632,7 +1632,7 @@ def SCALAR_SQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "11QI", "SsSi", OP_SCALAR_
   let isLaneQ = 1;
 }
 
-let TargetGuard = "v8.1a,neon" in {
+let TargetGuard = "v8.1a" in {
 // Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
 def SCALAR_SQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "111.I", "SsSi", OP_SCALAR_QRDMLAH_LN>;
 def SCALAR_SQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "111QI", "SsSi", OP_SCALAR_QRDMLAH_LN> {
@@ -1654,7 +1654,7 @@ def SCALAR_VDUP_LANEQ : IInst<"vdup_laneq", "1QI", "ScSsSiSlSfSdSUcSUsSUiSUlSPcS
 } // ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)"
 
 // ARMv8.2-A FP16 vector intrinsics for A32/A64.
-let TargetGuard = "fullfp16,neon" in {
+let TargetGuard = "fullfp16" in {
 
   // ARMv8.2-A FP16 one-operand vector intrinsics.
 
@@ -1679,7 +1679,7 @@ let TargetGuard = "fullfp16,neon" in {
   def VCVTP_U16    : SInst<"vcvtp_u16", "U.", "hQh">;
 
   // Vector rounding
-  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)", TargetGuard = "fullfp16,neon" in {
+  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)", TargetGuard = "fullfp16" in {
     def FRINTZH      : SInst<"vrnd",  "..", "hQh">;
     def FRINTNH      : SInst<"vrndn", "..", "hQh">;
     def FRINTAH      : SInst<"vrnda", "..", "hQh">;
@@ -1728,7 +1728,7 @@ let TargetGuard = "fullfp16,neon" in {
   // Max/Min
   def VMAXH         : SInst<"vmax", "...", "hQh">;
   def VMINH         : SInst<"vmin", "...", "hQh">;
-  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN)", TargetGuard = "fullfp16,neon" in {
+  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN)", TargetGuard = "fullfp16" in {
     def FMAXNMH       : SInst<"vmaxnm", "...", "hQh">;
     def FMINNMH       : SInst<"vminnm", "...", "hQh">;
   }
@@ -1775,7 +1775,7 @@ def VEXTH      : WInst<"vext", "...I", "hQh">;
 def VREV64H    : WOpInst<"vrev64", "..", "hQh", OP_REV64>;
 
 // ARMv8.2-A FP16 vector intrinsics for A64 only.
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fullfp16,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fullfp16" in {
 
   // Vector rounding
   def FRINTIH      : SInst<"vrndi", "..", "hQh">;
@@ -1872,11 +1872,11 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)" in {
 }
 
 // v8.2-A dot product instructions.
-let TargetGuard = "dotprod,neon" in {
+let TargetGuard = "dotprod" in {
   def DOT : SInst<"vdot", "..(<<)(<<)", "iQiUiQUi">;
   def DOT_LANE : SOpInst<"vdot_lane", "..(<<)(<<q)I", "iUiQiQUi", OP_DOT_LN>;
 }
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "dotprod,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "dotprod" in {
   // Variants indexing into a 128-bit vector are A64 only.
   def UDOT_LANEQ : SOpInst<"vdot_laneq", "..(<<)(<<Q)I", "iUiQiQUi", OP_DOT_LNQ> {
     let isLaneQ = 1;
@@ -1884,7 +1884,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "d
 }
 
 // v8.2-A FP16 fused multiply-add long instructions.
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fp16fml,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fp16fml" in {
   def VFMLAL_LOW  : SInst<"vfmlal_low",  ">>..", "hQh">;
   def VFMLSL_LOW  : SInst<"vfmlsl_low",  ">>..", "hQh">;
   def VFMLAL_HIGH : SInst<"vfmlal_high", ">>..", "hQh">;
@@ -1909,7 +1909,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "f
   }
 }
 
-let TargetGuard = "i8mm,neon" in {
+let TargetGuard = "i8mm" in {
   def VMMLA   : SInst<"vmmla", "..(<<)(<<)", "QUiQi">;
   def VUSMMLA : SInst<"vusmmla", "..(<<U)(<<)", "Qi">;
 
@@ -1926,7 +1926,7 @@ let TargetGuard = "i8mm,neon" in {
   }
 }
 
-let TargetGuard = "bf16,neon" in {
+let TargetGuard = "bf16" in {
   def VDOT_BF : SInst<"vbfdot", "..BB", "fQf">;
   def VDOT_LANE_BF : SOpInst<"vbfdot_lane", "..B(Bq)I", "fQf", OP_BFDOT_LN>;
   def VDOT_LANEQ_BF : SOpInst<"vbfdot_laneq", "..B(BQ)I", "fQf", OP_BFDOT_LNQ> {
@@ -1970,7 +1970,7 @@ multiclass VCMLA_ROTS<string type, string lanety, string laneqty> {
 }
 
 // v8.3-A Vector complex addition intrinsics
-let TargetGuard = "v8.3a,fullfp16,neon" in {
+let TargetGuard = "v8.3a,fullfp16" in {
   def VCADD_ROT90_FP16   : SInst<"vcadd_rot90", "...", "h">;
   def VCADD_ROT270_FP16  : SInst<"vcadd_rot270", "...", "h">;
   def VCADDQ_ROT90_FP16  : SInst<"vcaddq_rot90", "QQQ", "h">;
@@ -1978,7 +1978,7 @@ let TargetGuard = "v8.3a,fullfp16,neon" in {
 
   defm VCMLA_FP16  : VCMLA_ROTS<"h", "uint32x2_t", "uint32x4_t">;
 }
-let TargetGuard = "v8.3a,neon" in {
+let TargetGuard = "v8.3a" in {
   def VCADD_ROT90   : SInst<"vcadd_rot90", "...", "f">;
   def VCADD_ROT270  : SInst<"vcadd_rot270", "...", "f">;
   def VCADDQ_ROT90  : SInst<"vcaddq_rot90", "QQQ", "f">;
@@ -1986,7 +1986,7 @@ let TargetGuard = "v8.3a,neon" in {
 
   defm VCMLA_F32        : VCMLA_ROTS<"f", "uint64x1_t", "uint64x2_t">;
 }
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.3a,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.3a" in {
   def VCADDQ_ROT90_FP64  : SInst<"vcaddq_rot90", "QQQ", "d">;
   def VCADDQ_ROT270_FP64 : SInst<"vcaddq_rot270", "QQQ", "d">;
 
@@ -1994,7 +1994,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v
 }
 
 // V8.2-A BFloat intrinsics
-let TargetGuard = "bf16,neon" in {
+let TargetGuard = "bf16" in {
   def VCREATE_BF : NoTestOpInst<"vcreate", ".(IU>)", "b", OP_CAST> {
     let BigEndianSafe = 1;
   }
@@ -2058,14 +2058,14 @@ let TargetGuard = "bf16,neon" in {
   def SCALAR_CVT_F32_BF16 : SOpInst<"vcvtah_f32", "(1F>)(1!)", "b", OP_CVT_F32_BF16>;
 }
 
-let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
+let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16" in {
   def VCVT_BF16_F32_A32_INTERNAL : WInst<"__a32_vcvt_bf16", "BQ", "f">;
   def VCVT_BF16_F32_A32 : SOpInst<"vcvt_bf16", "BQ", "f", OP_VCVT_BF16_F32_A32>;
   def VCVT_LOW_BF16_F32_A32 : SOpInst<"vcvt_low_bf16",  "BQ", "Qf", OP_VCVT_BF16_F32_LO_A32>;
   def VCVT_HIGH_BF16_F32_A32 : SOpInst<"vcvt_high_bf16", "BBQ", "Qf", OP_VCVT_BF16_F32_HI_A32>;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16" in {
   def VCVT_LOW_BF16_F32_A64_INTERNAL : WInst<"__a64_vcvtq_low_bf16", "BQ", "Hf">;
   def VCVT_LOW_BF16_F32_A64 : SOpInst<"vcvt_low_bf16", "BQ", "Qf", OP_VCVT_BF16_F32_LO_A64>;
   def VCVT_HIGH_BF16_F32_A64 : SInst<"vcvt_high_bf16", "BBQ", "Qf">;
@@ -2077,14 +2077,14 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "b
   def COPYQ_LANEQ_BF16 : IOpInst<"vcopy_laneq", "..I.I", "Qb", OP_COPY_LN>;
 }
 
-let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
+let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16" in {
   let BigEndianSafe = 1 in {
     defm VREINTERPRET_BF : REINTERPRET_CROSS_TYPES<
         "csilUcUsUiUlhfPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQPcQPsQPl", "bQb">;
   }
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16" in {
   let BigEndianSafe = 1 in {
     defm VVREINTERPRET_BF : REINTERPRET_CROSS_TYPES<
         "csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk", "bQb">;
@@ -2092,7 +2092,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "b
 }
 
 // v8.9a/v9.4a LRCPC3 intrinsics
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "rcpc3,neon" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "rcpc3" in {
   def VLDAP1_LANE : WInst<"vldap1_lane", ".(c*!).I", "QUlQlUlldQdPlQPl">;
   def VSTL1_LANE  : WInst<"vstl1_lane", "v*(.!)I", "QUlQlUlldQdPlQPl">;
 }
diff --git a/clang/include/clang/Basic/arm_neon_incl.td b/clang/include/clang/Basic/arm_neon_incl.td
index 3b8015daee6d9..b8155c187d1bc 100644
--- a/clang/include/clang/Basic/arm_neon_incl.td
+++ b/clang/include/clang/Basic/arm_neon_incl.td
@@ -265,7 +265,7 @@ class Inst <string n, string p, string t, Operation o> {
   string Prototype = p;
   string Types = t;
   string ArchGuard = "";
-  string TargetGuard = "neon";
+  string TargetGuard = "";
 
   Operation Operation = o;
   bit BigEndianSafe = 0;
diff --git a/clang/test/Sema/aarch64-neon-without-target-feature.cpp b/clang/test/Sema/aarch64-neon-without-target-feature.cpp
deleted file mode 100644
index 64f76a7900faf..0000000000000
--- a/clang/test/Sema/aarch64-neon-without-target-feature.cpp
+++ /dev/null
@@ -1,34 +0,0 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +dotprod  -target-feature +fullfp16 -target-feature +fp16fml -target-feature +i8mm -target-feature +bf16 -verify -emit-llvm -o - %s
-
-// This test is testing the diagnostics that Clang emits when compiling without '+neon'.
-
-#include <arm_neon.h>
-
-void undefined(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t v16i8, uint8x8_t v8i8, float32x2_t v2f32, float32x4_t v4f32, float16x4_t v4f16, float64x2_t v2f64, bfloat16x4_t v4bf16, __bf16 bf16, poly64_t poly64, poly64x2_t poly64x2) {
-  // dotprod
-  vdot_u32(v2i32, v8i8, v8i8); // expected-error {{always_inline function 'vdot_u32' requires target feature 'neon'}}
-  vdot_laneq_u32(v2i32, v8i8, v16i8, 1); // expected-error {{always_inline function 'vdot_u32' requires target feature 'neon'}} expected-error {{'__builtin_neon_splat_laneq_v' needs target feature neon}}
-  // fp16
-  vceqz_f16(v4f16); // expected-error {{always_inline function 'vceqz_f16' requires target feature 'neon'}}
-  vrnd_f16(v4f16); // expected-error {{always_inline function 'vrnd_f16' requires target feature 'neon'}}
-  vmaxnm_f16(v4f16, v4f16); // expected-error {{always_inline function 'vmaxnm_f16' requires target feature 'neon'}}
-  vrndi_f16(v4f16); // expected-error {{always_inline function 'vrndi_f16' requires target feature 'neon'}}
-  // fp16fml depends on fp-armv8
-  vfmlal_low_f16(v2f32, v4f16, v4f16); // expected-error {{always_inline function 'vfmlal_low_f16' requires target feature 'neon'}}
-  // i8mm
-  vmmlaq_s32(v4i32, v8i16, v8i16); // expected-error {{always_inline function 'vmmlaq_s32' requires target feature 'neon'}}
-  vusdot_laneq_s32(v2i32, v8i8, v8i16, 0); // expected-error {{always_inline function 'vusdot_s32' requires target feature 'neon'}} expected-error {{'__builtin_neon_splat_laneq_v' needs target feature neon}}
-  // bf16
-  vbfdot_f32(v2f32, v4bf16, v4bf16); // expected-error {{always_inline function 'vbfdot_f32' requires target feature 'neon'}}
-  vcreate_bf16(10);
-  vdup_lane_bf16(v4bf16, 2); // expected-error {{'__builtin_neon_splat_lane_bf16' needs target feature bf16,neon}}
-  vdup_n_bf16(bf16); // expected-error {{always_inline function 'vdup_n_bf16' requires target feature 'neon'}}
-  vld1_bf16(0); // expected-error {{'__builtin_neon_vld1_bf16' needs target feature bf16,neon}}
-  vcvt_f32_bf16(v4bf16); // expected-error {{always_inline function 'vcvt_f32_bf16' requires target feature 'neon'}}
-  vcvt_bf16_f32(v4f32); // expected-error {{always_inline function 'vcvt_bf16_f32' requires target feature 'neon'}}
-  vmull_p64(poly64, poly64);  // expected-error {{always_inline function 'vmull_p64' requires target feature 'neon'}}
-  vmull_high_p64(poly64x2, poly64x2);  // expected-error {{always_inline function 'vmull_high_p64' requires target feature 'neon'}}
-  vtrn1_s8(v8i8, v8i8); // expected-error {{always_inline function 'vtrn1_s8' requires target feature 'neon'}}
-  vqabsq_s16(v8i16); // expected-error {{always_inline function 'vqabsq_s16' requires target feature 'neon'}}
-  vbslq_s16(v8i16, v8i16, v8i16);// expected-error {{always_inline function 'vbslq_s16' requires target feature 'neon'}}
-}
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 30fbb8c5d65e5..626031d38cf00 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -952,7 +952,7 @@ std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
   char typeCode = '\0';
   bool printNumber = true;
 
-  if (CK == ClassB && TargetGuard == "neon")
+  if (CK == ClassB && TargetGuard == "")
     return "";
 
   if (T.isBFloat16())
@@ -976,7 +976,7 @@ std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
       break;
     }
   }
-  if (CK == ClassB && TargetGuard == "neon") {
+  if (CK == ClassB && TargetGuard == "") {
     typeCode = '\0';
   }
 
@@ -1078,7 +1078,7 @@ std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const {
     S += "_" + getInstTypeCode(InBaseType, LocalCK);
   }
 
-  if (LocalCK == ClassB && TargetGuard == "neon")
+  if (LocalCK == ClassB && TargetGuard == "")
     S += "_v";
 
   // Insert a 'q' before the first '_' character so that it ends up before

From f39bd0a24e8aaa55c957c179607c071a86590ec1 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 22 Jul 2024 07:07:51 -0500
Subject: [PATCH 403/486] [AMDGPU] Do not print `kernel-resource-usage`
 information on non-kernels (#99720)

Summary:
This pass is used to get helpful information about the kernel resources
without needing to insepct the binary. However, it currently prints on
every function. These values will always be zero, so it's just spam on
the terminal, at best an indication that a function wasn't internalized
/ optimized out. This patch makes it only print for kernels to make it
more useful in practice.
---
 llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp   |  4 ++++
 .../AMDGPU/resource-optimization-remarks.ll   | 21 ++-----------------
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 3154dc6fe433d..e64e28e01d3d1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -1487,6 +1487,10 @@ void AMDGPUAsmPrinter::emitResourceUsageRemarks(
   if (!Ctx.getDiagHandlerPtr()->isAnalysisRemarkEnabled(Name))
     return;
 
+  // Currently non-kernel functions have no resources to emit.
+  if (!isEntryFunctionCC(MF.getFunction().getCallingConv()))
+    return;
+
   auto EmitResourceUsageRemark = [&](StringRef RemarkName,
                                      StringRef RemarkLabel, auto Argument) {
     // Add an indent for every line besides the line with the kernel name. This
diff --git a/llvm/test/CodeGen/AMDGPU/resource-optimization-remarks.ll b/llvm/test/CodeGen/AMDGPU/resource-optimization-remarks.ll
index a640ac985ade4..002de8bb4eb51 100644
--- a/llvm/test/CodeGen/AMDGPU/resource-optimization-remarks.ll
+++ b/llvm/test/CodeGen/AMDGPU/resource-optimization-remarks.ll
@@ -113,16 +113,7 @@ define amdgpu_kernel void @test_kernel() !dbg !3 {
   ret void
 }
 
-; STDERR: remark: foo.cl:42:0: Function Name: test_func
-; STDERR-NEXT: remark: foo.cl:42:0:     SGPRs: 0
-; STDERR-NEXT: remark: foo.cl:42:0:     VGPRs: 0
-; STDERR-NEXT: remark: foo.cl:42:0:     AGPRs: 0
-; STDERR-NEXT: remark: foo.cl:42:0:     ScratchSize [bytes/lane]: 0
-; STDERR-NEXT: remark: foo.cl:42:0:     Dynamic Stack: False
-; STDERR-NEXT: remark: foo.cl:42:0:     Occupancy [waves/SIMD]: 0
-; STDERR-NEXT: remark: foo.cl:42:0:     SGPRs Spill: 0
-; STDERR-NEXT: remark: foo.cl:42:0:     VGPRs Spill: 0
-; STDERR-NOT: LDS Size
+; STDERR-NOT: test_func
 define void @test_func() !dbg !6 {
   call void asm sideeffect "; clobber v17", "~{v17}"()
   call void asm sideeffect "; clobber s11", "~{s11}"()
@@ -144,15 +135,7 @@ define amdgpu_kernel void @empty_kernel() !dbg !7 {
   ret void
 }
 
-; STDERR: remark: foo.cl:52:0: Function Name: empty_func
-; STDERR-NEXT: remark: foo.cl:52:0:     SGPRs: 0
-; STDERR-NEXT: remark: foo.cl:52:0:     VGPRs: 0
-; STDERR-NEXT: remark: foo.cl:52:0:     AGPRs: 0
-; STDERR-NEXT: remark: foo.cl:52:0:     ScratchSize [bytes/lane]: 0
-; STDERR-NEXT: remark: foo.cl:52:0:     Dynamic Stack: False
-; STDERR-NEXT: remark: foo.cl:52:0:     Occupancy [waves/SIMD]: 0
-; STDERR-NEXT: remark: foo.cl:52:0:     SGPRs Spill: 0
-; STDERR-NEXT: remark: foo.cl:52:0:     VGPRs Spill: 0
+; STDERR-NOT: empty_func
 define void @empty_func() !dbg !8 {
   ret void
 }

From cffe1153f4466b1602d4778e57c3bd8663cef5ec Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Mon, 22 Jul 2024 14:13:21 +0200
Subject: [PATCH 404/486] [llvm-dlltool] Use EXPORTAS name type for renamed
 imports on ARM64EC. (#99346)

Renamed entries are more tricky on ARM64EC than on other targets due
to additional symbols (we need `__imp_aux_*` in addition to `__imp_*`
and both mangled and unmangled symbol thunks). While we could extend
weak aliases to add them, it seems cleaner to just always use EXPORTAS
name type on ARM64EC targets. Unlike other targets, linkers supporting
ARM64EC need to support EXPORTAS, so there is no compatibility problem
with that.
---
 llvm/lib/Object/COFFImportFile.cpp        |  5 +++-
 llvm/test/tools/llvm-dlltool/arm64ec.test | 29 +++++++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Object/COFFImportFile.cpp b/llvm/lib/Object/COFFImportFile.cpp
index 2458a53cb6d54..e67b02405a3a1 100644
--- a/llvm/lib/Object/COFFImportFile.cpp
+++ b/llvm/lib/Object/COFFImportFile.cpp
@@ -729,7 +729,10 @@ Error writeImportLibrary(StringRef ImportName, StringRef Path,
         else if (Machine == IMAGE_FILE_MACHINE_I386 &&
                  applyNameType(IMPORT_NAME_NOPREFIX, Name) == E.ImportName)
           NameType = IMPORT_NAME_NOPREFIX;
-        else if (Name == E.ImportName)
+        else if (isArm64EC(M)) {
+          NameType = IMPORT_NAME_EXPORTAS;
+          ExportName = E.ImportName;
+        } else if (Name == E.ImportName)
           NameType = IMPORT_NAME;
         else {
           Deferred D;
diff --git a/llvm/test/tools/llvm-dlltool/arm64ec.test b/llvm/test/tools/llvm-dlltool/arm64ec.test
index b03b4eaf7b2d0..4ed0f4a4387d6 100644
--- a/llvm/test/tools/llvm-dlltool/arm64ec.test
+++ b/llvm/test/tools/llvm-dlltool/arm64ec.test
@@ -44,6 +44,29 @@ RUN: llvm-nm --print-armap test3.lib | FileCheck --check-prefix=ARMAP2 %s
 RUN: not llvm-dlltool -m arm64 -d test.def -N test2.def -l test4.lib 2>&1 | FileCheck --check-prefix=ERR %s
 ERR: native .def file is supported only on arm64ec target
 
+RUN: llvm-dlltool -m arm64ec -d test3.def -l test3.lib
+RUN: llvm-readobj test3.lib | FileCheck --check-prefix=ALIAS %s
+
+ALIAS:      File: test.dll
+ALIAS-NEXT: Format: COFF-import-file-ARM64EC
+ALIAS-NEXT: Type: code
+ALIAS-NEXT: Name type: export as
+ALIAS-NEXT: Export name: efunc
+ALIAS-NEXT: Symbol: __imp_func
+ALIAS-NEXT: Symbol: func
+ALIAS-NEXT: Symbol: __imp_aux_func
+ALIAS-NEXT: Symbol: #func
+ALIAS-EMPTY:
+ALIAS-NEXT: File: test.dll
+ALIAS-NEXT: Format: COFF-import-file-ARM64EC
+ALIAS-NEXT: Type: code
+ALIAS-NEXT: Name type: export as
+ALIAS-NEXT: Export name: efunc
+ALIAS-NEXT: Symbol: __imp_efunc
+ALIAS-NEXT: Symbol: efunc
+ALIAS-NEXT: Symbol: __imp_aux_efunc
+ALIAS-NEXT: Symbol: #efunc
+
 #--- test.def
 LIBRARY test.dll
 EXPORTS
@@ -53,3 +76,9 @@ EXPORTS
 LIBRARY test.dll
 EXPORTS
     otherfunc
+
+#--- test3.def
+LIBRARY test.dll
+EXPORTS
+    func == efunc
+    efunc

From 5b8479bc28a8641f02be3d64f87770b9e0b1a427 Mon Sep 17 00:00:00 2001
From: OverMighty <its.overmighty@gmail.com>
Date: Mon, 22 Jul 2024 14:42:12 +0200
Subject: [PATCH 405/486] [libc][CMake] Set library type of
 libc_diff_test_utils to STATIC (#99869)

Fixes linker errors due to hidden symbols when running CMake with
-DBUILD_SHARED_LIBS=ON.
---
 libc/test/src/math/performance_testing/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/test/src/math/performance_testing/CMakeLists.txt b/libc/test/src/math/performance_testing/CMakeLists.txt
index a75becba04d07..2ec5364059b17 100644
--- a/libc/test/src/math/performance_testing/CMakeLists.txt
+++ b/libc/test/src/math/performance_testing/CMakeLists.txt
@@ -1,5 +1,5 @@
 add_library(
-  libc_diff_test_utils
+  libc_diff_test_utils STATIC
   Timer.cpp
   Timer.h
 )

From 2f5dc596b5719e5ed7f6978dafbce994f425a033 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?kadir=20=C3=A7etinkaya?= <kadircet@google.com>
Date: Mon, 22 Jul 2024 14:58:07 +0200
Subject: [PATCH 406/486] [IncludeCleaner] Also check for spellings of physical
 headers (#99843)

Some physical headers can have "conflicting" spellings, when same
filename exists under two different include search paths. e.g.
<stdlib.h>
can be provided by both standard library and underlying libc headers. In
such scenarios if the usage is from the latter include-search path,
users can't spell it directly.

This patch ensures we also consider spellings of such includes, in
addition to their physical files to prevent conflicting suggestions.
---
 clang-tools-extra/clangd/IncludeCleaner.cpp   | 20 ++++++++++++++
 .../clangd/unittests/IncludeCleanerTests.cpp  | 22 ++++++++++++++++
 .../include-cleaner/lib/Analysis.cpp          | 17 +++++++++---
 .../unittests/AnalysisTest.cpp                | 26 +++++++++++++++++++
 4 files changed, 82 insertions(+), 3 deletions(-)

diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp
index dc5b7ec95db5f..e34706172f0bf 100644
--- a/clang-tools-extra/clangd/IncludeCleaner.cpp
+++ b/clang-tools-extra/clangd/IncludeCleaner.cpp
@@ -401,6 +401,26 @@ computeIncludeCleanerFindings(ParsedAST &AST, bool AnalyzeAngledIncludes) {
             Ref.RT != include_cleaner::RefType::Explicit)
           return;
 
+        // Check if we have any headers with the same spelling, in edge cases
+        // like `#include_next "foo.h"`, the user can't ever include the
+        // physical foo.h, but can have a spelling that refers to it.
+        // We postpone this check because spelling a header for every usage is
+        // expensive.
+        std::string Spelling = include_cleaner::spellHeader(
+            {Providers.front(), AST.getPreprocessor().getHeaderSearchInfo(),
+             MainFile});
+        for (auto *Inc :
+             ConvertedIncludes.match(include_cleaner::Header{Spelling})) {
+          Satisfied = true;
+          auto HeaderID =
+              AST.getIncludeStructure().getID(&Inc->Resolved->getFileEntry());
+          assert(HeaderID.has_value() &&
+                 "ConvertedIncludes only contains resolved includes.");
+          Used.insert(*HeaderID);
+        }
+        if (Satisfied)
+          return;
+
         // We actually always want to map usages to their spellings, but
         // spelling locations can point into preamble section. Using these
         // offsets could lead into crashes in presence of stale preambles. Hence
diff --git a/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp b/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
index 7027232460354..0ee748c1ed2d0 100644
--- a/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
+++ b/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
@@ -644,6 +644,28 @@ TEST(IncludeCleaner, ResourceDirIsIgnored) {
   EXPECT_THAT(Findings.MissingIncludes, IsEmpty());
 }
 
+TEST(IncludeCleaner, DifferentHeaderSameSpelling) {
+  // `foo` is declared in foo_inner/foo.h, but there's no way to spell it
+  // directly. Make sure we don't generate unusued/missing include findings in
+  // such cases.
+  auto TU = TestTU::withCode(R"cpp(
+    #include <foo.h>
+    void baz() {
+      foo();
+    }
+  )cpp");
+  TU.AdditionalFiles["foo/foo.h"] = guard("#include_next <foo.h>");
+  TU.AdditionalFiles["foo_inner/foo.h"] = guard(R"cpp(
+    void foo();
+  )cpp");
+  TU.ExtraArgs.push_back("-Ifoo");
+  TU.ExtraArgs.push_back("-Ifoo_inner");
+
+  auto AST = TU.build();
+  auto Findings = computeIncludeCleanerFindings(AST);
+  EXPECT_THAT(Findings.UnusedIncludes, IsEmpty());
+  EXPECT_THAT(Findings.MissingIncludes, IsEmpty());
+}
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/include-cleaner/lib/Analysis.cpp b/clang-tools-extra/include-cleaner/lib/Analysis.cpp
index f1cd72f877ca2..68fe79d6929f6 100644
--- a/clang-tools-extra/include-cleaner/lib/Analysis.cpp
+++ b/clang-tools-extra/include-cleaner/lib/Analysis.cpp
@@ -105,9 +105,20 @@ analyze(llvm::ArrayRef<Decl *> ASTRoots,
              }
              if (!Satisfied && !Providers.empty() &&
                  Ref.RT == RefType::Explicit &&
-                 !HeaderFilter(Providers.front().resolvedPath()))
-               Missing.insert(spellHeader(
-                   {Providers.front(), PP.getHeaderSearchInfo(), MainFile}));
+                 !HeaderFilter(Providers.front().resolvedPath())) {
+               // Check if we have any headers with the same spelling, in edge
+               // cases like `#include_next "foo.h"`, the user can't ever
+               // include the physical foo.h, but can have a spelling that
+               // refers to it.
+               auto Spelling = spellHeader(
+                   {Providers.front(), PP.getHeaderSearchInfo(), MainFile});
+               for (const Include *I : Inc.match(Header{Spelling})) {
+                 Used.insert(I);
+                 Satisfied = true;
+               }
+               if (!Satisfied)
+                 Missing.insert(std::move(Spelling));
+             }
            });
 
   AnalysisResults Results;
diff --git a/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp b/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp
index 6558b68087684..5696c380758f8 100644
--- a/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp
+++ b/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp
@@ -12,6 +12,7 @@
 #include "clang-include-cleaner/Record.h"
 #include "clang-include-cleaner/Types.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclBase.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
@@ -296,6 +297,31 @@ TEST_F(AnalyzeTest, ResourceDirIsIgnored) {
   EXPECT_THAT(Results.Missing, testing::IsEmpty());
 }
 
+TEST_F(AnalyzeTest, DifferentHeaderSameSpelling) {
+  Inputs.ExtraArgs.push_back("-Ifoo");
+  Inputs.ExtraArgs.push_back("-Ifoo_inner");
+  // `foo` is declared in foo_inner/foo.h, but there's no way to spell it
+  // directly. Make sure we don't generate unusued/missing include findings in
+  // such cases.
+  Inputs.Code = R"cpp(
+    #include <foo.h>
+    void baz() {
+      foo();
+    }
+  )cpp";
+  Inputs.ExtraFiles["foo/foo.h"] = guard("#include_next <foo.h>");
+  Inputs.ExtraFiles["foo_inner/foo.h"] = guard(R"cpp(
+    void foo();
+  )cpp");
+  TestAST AST(Inputs);
+  std::vector<Decl *> DeclsInTU;
+  for (auto *D : AST.context().getTranslationUnitDecl()->decls())
+    DeclsInTU.push_back(D);
+  auto Results = analyze(DeclsInTU, {}, PP.Includes, &PI, AST.preprocessor());
+  EXPECT_THAT(Results.Unused, testing::IsEmpty());
+  EXPECT_THAT(Results.Missing, testing::IsEmpty());
+}
+
 TEST(FixIncludes, Basic) {
   llvm::StringRef Code = R"cpp(#include "d.h"
 #include "a.h"

From a7fb25dd1fcc2e5afcc65cccfa83b7b381b48906 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 22 Jul 2024 15:04:18 +0200
Subject: [PATCH 407/486] [AArch64] Fix broken check lines (NFC)

---
 llvm/test/CodeGen/AArch64/fast-isel-select.ll | 281 ++++++------------
 1 file changed, 88 insertions(+), 193 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/fast-isel-select.ll b/llvm/test/CodeGen/AArch64/fast-isel-select.ll
index 65701343ccc1e..315ca4f285fd7 100644
--- a/llvm/test/CodeGen/AArch64/fast-isel-select.ll
+++ b/llvm/test/CodeGen/AArch64/fast-isel-select.ll
@@ -1,15 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=aarch64-apple-darwin                             -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-apple-darwin                             -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SDAGISEL
 ; RUN: llc -mtriple=aarch64-apple-darwin -fast-isel -fast-isel-abort=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-FASTISEL
 ; RUN: llc -mtriple=aarch64-apple-darwin -global-isel -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GISEL
 
 ; First test the different supported value types for select.
 define zeroext i1 @select_i1(i1 zeroext %c, i1 zeroext %a, i1 zeroext %b) {
-; GISEL-LABEL: select_i1:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    tst w0, #0x1
-; GISEL-NEXT:    csel w0, w1, w2, ne
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_i1:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    cmp w0, #0
+; CHECK-SDAGISEL-NEXT:    csel w0, w1, w2, ne
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_i1:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    tst w0, #0x1
@@ -27,11 +28,12 @@ define zeroext i1 @select_i1(i1 zeroext %c, i1 zeroext %a, i1 zeroext %b) {
 }
 
 define zeroext i8 @select_i8(i1 zeroext %c, i8 zeroext %a, i8 zeroext %b) {
-; GISEL-LABEL: select_i8:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    tst w0, #0x1
-; GISEL-NEXT:    csel w0, w1, w2, ne
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_i8:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    cmp w0, #0
+; CHECK-SDAGISEL-NEXT:    csel w0, w1, w2, ne
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_i8:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    tst w0, #0x1
@@ -49,11 +51,12 @@ define zeroext i8 @select_i8(i1 zeroext %c, i8 zeroext %a, i8 zeroext %b) {
 }
 
 define zeroext i16 @select_i16(i1 zeroext %c, i16 zeroext %a, i16 zeroext %b) {
-; GISEL-LABEL: select_i16:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    tst w0, #0x1
-; GISEL-NEXT:    csel w0, w1, w2, ne
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_i16:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    cmp w0, #0
+; CHECK-SDAGISEL-NEXT:    csel w0, w1, w2, ne
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_i16:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    tst w0, #0x1
@@ -71,11 +74,12 @@ define zeroext i16 @select_i16(i1 zeroext %c, i16 zeroext %a, i16 zeroext %b) {
 }
 
 define i32 @select_i32(i1 zeroext %c, i32 %a, i32 %b) {
-; GISEL-LABEL: select_i32:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    tst w0, #0x1
-; GISEL-NEXT:    csel w0, w1, w2, ne
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_i32:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    cmp w0, #0
+; CHECK-SDAGISEL-NEXT:    csel w0, w1, w2, ne
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_i32:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    tst w0, #0x1
@@ -92,11 +96,12 @@ define i32 @select_i32(i1 zeroext %c, i32 %a, i32 %b) {
 }
 
 define i64 @select_i64(i1 zeroext %c, i64 %a, i64 %b) {
-; GISEL-LABEL: select_i64:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    tst w0, #0x1
-; GISEL-NEXT:    csel x0, x1, x2, ne
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_i64:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    cmp w0, #0
+; CHECK-SDAGISEL-NEXT:    csel x0, x1, x2, ne
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_i64:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    tst w0, #0x1
@@ -113,11 +118,12 @@ define i64 @select_i64(i1 zeroext %c, i64 %a, i64 %b) {
 }
 
 define float @select_f32(i1 zeroext %c, float %a, float %b) {
-; GISEL-LABEL: select_f32:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    tst w0, #0x1
-; GISEL-NEXT:    fcsel s0, s0, s1, ne
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_f32:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    cmp w0, #0
+; CHECK-SDAGISEL-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_f32:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    tst w0, #0x1
@@ -134,11 +140,12 @@ define float @select_f32(i1 zeroext %c, float %a, float %b) {
 }
 
 define double @select_f64(i1 zeroext %c, double %a, double %b) {
-; GISEL-LABEL: select_f64:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    tst w0, #0x1
-; GISEL-NEXT:    fcsel d0, d0, d1, ne
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_f64:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    cmp w0, #0
+; CHECK-SDAGISEL-NEXT:    fcsel d0, d0, d1, ne
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_f64:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    tst w0, #0x1
@@ -156,6 +163,11 @@ define double @select_f64(i1 zeroext %c, double %a, double %b) {
 
 ; Now test the folding of all compares.
 define float @select_fcmp_false(float %x, float %a, float %b) {
+; CHECK-SDAGISEL-LABEL: select_fcmp_false:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    fmov s0, s2
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_fcmp_false:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    fmov s0, s2
@@ -166,11 +178,6 @@ define float @select_fcmp_false(float %x, float %a, float %b) {
 ; CHECK-GISEL-NEXT:    fcmp s0, s0
 ; CHECK-GISEL-NEXT:    fcsel s0, s1, s2, gt
 ; CHECK-GISEL-NEXT:    ret
-; GISEL-LABEL: select_fcmp_false:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s0
-; GISEL-NEXT:    fcsel s0, s1, s2, gt
-; GISEL-NEXT:    ret
   %1 = fcmp ogt float %x, %x
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -182,11 +189,6 @@ define float @select_fcmp_ogt(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, gt
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_ogt:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, gt
-; GISEL-NEXT:    ret
   %1 = fcmp ogt float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -198,11 +200,6 @@ define float @select_fcmp_oge(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, ge
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_oge:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, ge
-; GISEL-NEXT:    ret
   %1 = fcmp oge float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -214,11 +211,6 @@ define float @select_fcmp_olt(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, mi
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_olt:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, mi
-; GISEL-NEXT:    ret
   %1 = fcmp olt float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -230,17 +222,19 @@ define float @select_fcmp_ole(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, ls
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_ole:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, ls
-; GISEL-NEXT:    ret
   %1 = fcmp ole float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
 }
 
 define float @select_fcmp_one(float %x, float %y, float %a, float %b) {
+; CHECK-SDAGISEL-LABEL: select_fcmp_one:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    fcmp s0, s1
+; CHECK-SDAGISEL-NEXT:    fcsel s0, s2, s3, mi
+; CHECK-SDAGISEL-NEXT:    fcsel s0, s2, s0, gt
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_fcmp_one:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    fcmp s0, s1
@@ -257,15 +251,6 @@ define float @select_fcmp_one(float %x, float %y, float %a, float %b) {
 ; CHECK-GISEL-NEXT:    tst w8, #0x1
 ; CHECK-GISEL-NEXT:    fcsel s0, s2, s3, ne
 ; CHECK-GISEL-NEXT:    ret
-; GISEL-LABEL: select_fcmp_one:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    cset w8, mi
-; GISEL-NEXT:    cset w9, gt
-; GISEL-NEXT:    orr w8, w8, w9
-; GISEL-NEXT:    tst w8, #0x1
-; GISEL-NEXT:    fcsel s0, s2, s3, ne
-; GISEL-NEXT:    ret
   %1 = fcmp one float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -277,11 +262,6 @@ define float @select_fcmp_ord(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, vc
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_ord:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, vc
-; GISEL-NEXT:    ret
   %1 = fcmp ord float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -293,17 +273,19 @@ define float @select_fcmp_uno(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, vs
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_uno:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, vs
-; GISEL-NEXT:    ret
   %1 = fcmp uno float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
 }
 
 define float @select_fcmp_ueq(float %x, float %y, float %a, float %b) {
+; CHECK-SDAGISEL-LABEL: select_fcmp_ueq:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    fcmp s0, s1
+; CHECK-SDAGISEL-NEXT:    fcsel s0, s2, s3, eq
+; CHECK-SDAGISEL-NEXT:    fcsel s0, s2, s0, vs
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_fcmp_ueq:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    fcmp s0, s1
@@ -320,15 +302,6 @@ define float @select_fcmp_ueq(float %x, float %y, float %a, float %b) {
 ; CHECK-GISEL-NEXT:    tst w8, #0x1
 ; CHECK-GISEL-NEXT:    fcsel s0, s2, s3, ne
 ; CHECK-GISEL-NEXT:    ret
-; GISEL-LABEL: select_fcmp_ueq:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    cset w8, eq
-; GISEL-NEXT:    cset w9, vs
-; GISEL-NEXT:    orr w8, w8, w9
-; GISEL-NEXT:    tst w8, #0x1
-; GISEL-NEXT:    fcsel s0, s2, s3, ne
-; GISEL-NEXT:    ret
   %1 = fcmp ueq float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -340,11 +313,6 @@ define float @select_fcmp_ugt(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, hi
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_ugt:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, hi
-; GISEL-NEXT:    ret
   %1 = fcmp ugt float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -356,11 +324,6 @@ define float @select_fcmp_uge(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, pl
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_uge:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, pl
-; GISEL-NEXT:    ret
   %1 = fcmp uge float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -372,11 +335,6 @@ define float @select_fcmp_ult(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, lt
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_ult:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, lt
-; GISEL-NEXT:    ret
   %1 = fcmp ult float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -389,11 +347,6 @@ define float @select_fcmp_ule(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, le
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_ule:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, le
-; GISEL-NEXT:    ret
   %1 = fcmp ule float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -405,17 +358,17 @@ define float @select_fcmp_une(float %x, float %y, float %a, float %b) {
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s2, s3, ne
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_fcmp_une:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s1
-; GISEL-NEXT:    fcsel s0, s2, s3, ne
-; GISEL-NEXT:    ret
   %1 = fcmp une float %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
 }
 
 define float @select_fcmp_true(float %x, float %a, float %b) {
+; CHECK-SDAGISEL-LABEL: select_fcmp_true:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    fmov s0, s1
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_fcmp_true:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    fmov s0, s1
@@ -430,15 +383,6 @@ define float @select_fcmp_true(float %x, float %a, float %b) {
 ; CHECK-GISEL-NEXT:    tst w8, #0x1
 ; CHECK-GISEL-NEXT:    fcsel s0, s1, s2, ne
 ; CHECK-GISEL-NEXT:    ret
-; GISEL-LABEL: select_fcmp_true:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    fcmp s0, s0
-; GISEL-NEXT:    cset w8, eq
-; GISEL-NEXT:    cset w9, vs
-; GISEL-NEXT:    orr w8, w8, w9
-; GISEL-NEXT:    tst w8, #0x1
-; GISEL-NEXT:    fcsel s0, s1, s2, ne
-; GISEL-NEXT:    ret
   %1 = fcmp ueq float %x, %x
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -450,11 +394,6 @@ define float @select_icmp_eq(i32 %x, i32 %y, float %a, float %b) {
 ; CHECK-NEXT:    cmp w0, w1
 ; CHECK-NEXT:    fcsel s0, s0, s1, eq
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_icmp_eq:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    cmp w0, w1
-; GISEL-NEXT:    fcsel s0, s0, s1, eq
-; GISEL-NEXT:    ret
   %1 = icmp eq i32 %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -466,11 +405,6 @@ define float @select_icmp_ne(i32 %x, i32 %y, float %a, float %b) {
 ; CHECK-NEXT:    cmp w0, w1
 ; CHECK-NEXT:    fcsel s0, s0, s1, ne
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_icmp_ne:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    cmp w0, w1
-; GISEL-NEXT:    fcsel s0, s0, s1, ne
-; GISEL-NEXT:    ret
   %1 = icmp ne i32 %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -482,11 +416,6 @@ define float @select_icmp_ugt(i32 %x, i32 %y, float %a, float %b) {
 ; CHECK-NEXT:    cmp w0, w1
 ; CHECK-NEXT:    fcsel s0, s0, s1, hi
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_icmp_ugt:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    cmp w0, w1
-; GISEL-NEXT:    fcsel s0, s0, s1, hi
-; GISEL-NEXT:    ret
   %1 = icmp ugt i32 %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -498,11 +427,6 @@ define float @select_icmp_uge(i32 %x, i32 %y, float %a, float %b) {
 ; CHECK-NEXT:    cmp w0, w1
 ; CHECK-NEXT:    fcsel s0, s0, s1, hs
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_icmp_uge:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    cmp w0, w1
-; GISEL-NEXT:    fcsel s0, s0, s1, hs
-; GISEL-NEXT:    ret
   %1 = icmp uge i32 %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -514,11 +438,6 @@ define float @select_icmp_ult(i32 %x, i32 %y, float %a, float %b) {
 ; CHECK-NEXT:    cmp w0, w1
 ; CHECK-NEXT:    fcsel s0, s0, s1, lo
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_icmp_ult:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    cmp w0, w1
-; GISEL-NEXT:    fcsel s0, s0, s1, lo
-; GISEL-NEXT:    ret
   %1 = icmp ult i32 %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -530,11 +449,6 @@ define float @select_icmp_ule(i32 %x, i32 %y, float %a, float %b) {
 ; CHECK-NEXT:    cmp w0, w1
 ; CHECK-NEXT:    fcsel s0, s0, s1, ls
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_icmp_ule:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    cmp w0, w1
-; GISEL-NEXT:    fcsel s0, s0, s1, ls
-; GISEL-NEXT:    ret
   %1 = icmp ule i32 %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -546,11 +460,6 @@ define float @select_icmp_sgt(i32 %x, i32 %y, float %a, float %b) {
 ; CHECK-NEXT:    cmp w0, w1
 ; CHECK-NEXT:    fcsel s0, s0, s1, gt
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_icmp_sgt:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    cmp w0, w1
-; GISEL-NEXT:    fcsel s0, s0, s1, gt
-; GISEL-NEXT:    ret
   %1 = icmp sgt i32 %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -562,11 +471,6 @@ define float @select_icmp_sge(i32 %x, i32 %y, float %a, float %b) {
 ; CHECK-NEXT:    cmp w0, w1
 ; CHECK-NEXT:    fcsel s0, s0, s1, ge
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_icmp_sge:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    cmp w0, w1
-; GISEL-NEXT:    fcsel s0, s0, s1, ge
-; GISEL-NEXT:    ret
   %1 = icmp sge i32 %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -578,11 +482,6 @@ define float @select_icmp_slt(i32 %x, i32 %y, float %a, float %b) {
 ; CHECK-NEXT:    cmp w0, w1
 ; CHECK-NEXT:    fcsel s0, s0, s1, lt
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_icmp_slt:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    cmp w0, w1
-; GISEL-NEXT:    fcsel s0, s0, s1, lt
-; GISEL-NEXT:    ret
   %1 = icmp slt i32 %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -594,11 +493,6 @@ define float @select_icmp_sle(i32 %x, i32 %y, float %a, float %b) {
 ; CHECK-NEXT:    cmp w0, w1
 ; CHECK-NEXT:    fcsel s0, s0, s1, le
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: select_icmp_sle:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    cmp w0, w1
-; GISEL-NEXT:    fcsel s0, s0, s1, le
-; GISEL-NEXT:    ret
   %1 = icmp sle i32 %x, %y
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -606,11 +500,11 @@ define float @select_icmp_sle(i32 %x, i32 %y, float %a, float %b) {
 
 ; Test peephole optimizations for select.
 define zeroext i1 @select_opt1(i1 zeroext %c, i1 zeroext %a) {
-; GISEL-LABEL: select_opt1:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    orr w8, w0, w1
-; GISEL-NEXT:    and w0, w8, #0x1
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_opt1:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    orr w0, w0, w1
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_opt1:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    orr w8, w0, w1
@@ -627,12 +521,12 @@ define zeroext i1 @select_opt1(i1 zeroext %c, i1 zeroext %a) {
 }
 
 define zeroext i1 @select_opt2(i1 zeroext %c, i1 zeroext %a) {
-; GISEL-LABEL: select_opt2:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    eor w8, w0, #0x1
-; GISEL-NEXT:    orr w8, w8, w1
-; GISEL-NEXT:    and w0, w8, #0x1
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_opt2:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    eor w8, w0, #0x1
+; CHECK-SDAGISEL-NEXT:    orr w0, w8, w1
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_opt2:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    eor w8, w0, #0x1
@@ -651,11 +545,11 @@ define zeroext i1 @select_opt2(i1 zeroext %c, i1 zeroext %a) {
 }
 
 define zeroext i1 @select_opt3(i1 zeroext %c, i1 zeroext %a) {
-; GISEL-LABEL: select_opt3:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    eor w8, w0, #0x1
-; GISEL-NEXT:    and w0, w8, w1
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_opt3:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    bic w0, w1, w0
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_opt3:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    bic w8, w1, w0
@@ -672,10 +566,11 @@ define zeroext i1 @select_opt3(i1 zeroext %c, i1 zeroext %a) {
 }
 
 define zeroext i1 @select_opt4(i1 zeroext %c, i1 zeroext %a) {
-; GISEL-LABEL: select_opt4:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    and w0, w0, w1
-; GISEL-NEXT:    ret
+; CHECK-SDAGISEL-LABEL: select_opt4:
+; CHECK-SDAGISEL:       ; %bb.0:
+; CHECK-SDAGISEL-NEXT:    and w0, w0, w1
+; CHECK-SDAGISEL-NEXT:    ret
+;
 ; CHECK-FASTISEL-LABEL: select_opt4:
 ; CHECK-FASTISEL:       ; %bb.0:
 ; CHECK-FASTISEL-NEXT:    and w8, w0, w1

From 0284b4b4b603192ce9eb29d3a34c23ea7378bee1 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Mon, 22 Jul 2024 14:21:31 +0100
Subject: [PATCH 408/486] [Clang][NEON] Add neon target guard to intrinsics
 (#99870)

This patch improves reported error when NEON intrinsics are used without
neon target feature.
---
 clang/include/clang/Basic/arm_neon.td         | 64 +++++++++----------
 clang/include/clang/Basic/arm_neon_incl.td    |  2 +-
 .../aarch64-neon-without-target-feature.cpp   | 36 +++++++++++
 clang/utils/TableGen/NeonEmitter.cpp          |  6 +-
 4 files changed, 72 insertions(+), 36 deletions(-)
 create mode 100644 clang/test/Sema/aarch64-neon-without-target-feature.cpp

diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index 6390ba3f9fe5e..3098fa67e6a51 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -289,7 +289,7 @@ def SPLATQ : WInst<"splat_laneq", ".(!Q)I",
                    "UcUsUicsilPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUlhdQhQdPlQPl"> {
   let isLaneQ = 1;
 }
-let TargetGuard = "bf16" in {
+let TargetGuard = "bf16,neon" in {
   def SPLAT_BF  : WInst<"splat_lane", ".(!q)I", "bQb">;
   def SPLATQ_BF : WInst<"splat_laneq", ".(!Q)I", "bQb"> {
     let isLaneQ = 1;
@@ -323,7 +323,7 @@ def VMLSL    : SOpInst<"vmlsl", "(>Q)(>Q)..", "csiUcUsUi", OP_MLSL>;
 def VQDMULH  : SInst<"vqdmulh", "...", "siQsQi">;
 def VQRDMULH : SInst<"vqrdmulh", "...", "siQsQi">;
 
-let TargetGuard = "v8.1a" in {
+let TargetGuard = "v8.1a,neon" in {
 def VQRDMLAH : SInst<"vqrdmlah", "....", "siQsQi">;
 def VQRDMLSH : SInst<"vqrdmlsh", "....", "siQsQi">;
 }
@@ -614,7 +614,7 @@ def A64_VQDMULH_LANE  : SInst<"vqdmulh_lane", "..(!q)I", "siQsQi">;
 def A64_VQRDMULH_LANE : SInst<"vqrdmulh_lane", "..(!q)I", "siQsQi">;
 }
 
-let TargetGuard = "v8.1a" in {
+let TargetGuard = "v8.1a,neon" in {
 def VQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "...qI", "siQsQi", OP_QRDMLAH_LN>;
 def VQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "...qI", "siQsQi", OP_QRDMLSH_LN>;
 }
@@ -957,7 +957,7 @@ def VQDMLAL_HIGH : SOpInst<"vqdmlal_high", "(>Q)(>Q)QQ", "si", OP_QDMLALHi>;
 def VQDMLAL_HIGH_N : SOpInst<"vqdmlal_high_n", "(>Q)(>Q)Q1", "si", OP_QDMLALHi_N>;
 def VQDMLSL_HIGH : SOpInst<"vqdmlsl_high", "(>Q)(>Q)QQ", "si", OP_QDMLSLHi>;
 def VQDMLSL_HIGH_N : SOpInst<"vqdmlsl_high_n", "(>Q)(>Q)Q1", "si", OP_QDMLSLHi_N>;
-let TargetGuard = "aes" in {
+let TargetGuard = "aes,neon" in {
   def VMULL_P64    : SInst<"vmull", "(1>)11", "Pl">;
   def VMULL_HIGH_P64 : SOpInst<"vmull_high", "(1>)..", "HPl", OP_MULLHi_P64>;
 }
@@ -1091,7 +1091,7 @@ let isLaneQ = 1 in {
 def VQDMULH_LANEQ  : SInst<"vqdmulh_laneq", "..QI", "siQsQi">;
 def VQRDMULH_LANEQ : SInst<"vqrdmulh_laneq", "..QI", "siQsQi">;
 }
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a,neon" in {
 def VQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "...QI", "siQsQi", OP_QRDMLAH_LN> {
   let isLaneQ = 1;
 }
@@ -1122,14 +1122,14 @@ def VEXT_A64 : WInst<"vext", "...I", "dQdPlQPl">;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Crypto
-let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "aes" in {
+let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "aes,neon" in {
 def AESE : SInst<"vaese", "...", "QUc">;
 def AESD : SInst<"vaesd", "...", "QUc">;
 def AESMC : SInst<"vaesmc", "..", "QUc">;
 def AESIMC : SInst<"vaesimc", "..", "QUc">;
 }
 
-let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "sha2" in {
+let ArchGuard = "__ARM_ARCH >= 8", TargetGuard = "sha2,neon" in {
 def SHA1H : SInst<"vsha1h", "11", "Ui">;
 def SHA1SU1 : SInst<"vsha1su1", "...", "QUi">;
 def SHA256SU0 : SInst<"vsha256su0", "...", "QUi">;
@@ -1143,7 +1143,7 @@ def SHA256H2 : SInst<"vsha256h2", "....", "QUi">;
 def SHA256SU1 : SInst<"vsha256su1", "....", "QUi">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3,neon" in {
 def BCAX : SInst<"vbcax", "....", "QUcQUsQUiQUlQcQsQiQl">;
 def EOR3 : SInst<"veor3", "....", "QUcQUsQUiQUlQcQsQiQl">;
 def RAX1 : SInst<"vrax1", "...", "QUl">;
@@ -1153,14 +1153,14 @@ def XAR :  SInst<"vxar", "...I", "QUl">;
 }
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sha3,neon" in {
 def SHA512SU0 : SInst<"vsha512su0", "...", "QUl">;
 def SHA512su1 : SInst<"vsha512su1", "....", "QUl">;
 def SHA512H : SInst<"vsha512h", "....", "QUl">;
 def SHA512H2 : SInst<"vsha512h2", "....", "QUl">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4,neon" in {
 def SM3SS1 : SInst<"vsm3ss1", "....", "QUi">;
 def SM3TT1A : SInst<"vsm3tt1a", "....I", "QUi">;
 def SM3TT1B : SInst<"vsm3tt1b", "....I", "QUi">;
@@ -1170,7 +1170,7 @@ def SM3PARTW1 : SInst<"vsm3partw1", "....", "QUi">;
 def SM3PARTW2 : SInst<"vsm3partw2", "....", "QUi">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "sm4,neon" in {
 def SM4E : SInst<"vsm4e", "...", "QUi">;
 def SM4EKEY : SInst<"vsm4ekey", "...", "QUi">;
 }
@@ -1227,7 +1227,7 @@ def FRINTZ_S64 : SInst<"vrnd", "..", "dQd">;
 def FRINTI_S64 : SInst<"vrndi", "..", "dQd">;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.5a" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.5a,neon" in {
 def FRINT32X_S32 : SInst<"vrnd32x", "..", "fQf">;
 def FRINT32Z_S32 : SInst<"vrnd32z", "..", "fQf">;
 def FRINT64X_S32 : SInst<"vrnd64x", "..", "fQf">;
@@ -1401,7 +1401,7 @@ def SCALAR_SQDMULH : SInst<"vqdmulh", "111", "SsSi">;
 // Scalar Integer Saturating Rounding Doubling Multiply Half High
 def SCALAR_SQRDMULH : SInst<"vqrdmulh", "111", "SsSi">;
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.1a,neon" in {
 ////////////////////////////////////////////////////////////////////////////////
 // Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
 def SCALAR_SQRDMLAH : SInst<"vqrdmlah", "1111", "SsSi">;
@@ -1632,7 +1632,7 @@ def SCALAR_SQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "11QI", "SsSi", OP_SCALAR_
   let isLaneQ = 1;
 }
 
-let TargetGuard = "v8.1a" in {
+let TargetGuard = "v8.1a,neon" in {
 // Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
 def SCALAR_SQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "111.I", "SsSi", OP_SCALAR_QRDMLAH_LN>;
 def SCALAR_SQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "111QI", "SsSi", OP_SCALAR_QRDMLAH_LN> {
@@ -1654,7 +1654,7 @@ def SCALAR_VDUP_LANEQ : IInst<"vdup_laneq", "1QI", "ScSsSiSlSfSdSUcSUsSUiSUlSPcS
 } // ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)"
 
 // ARMv8.2-A FP16 vector intrinsics for A32/A64.
-let TargetGuard = "fullfp16" in {
+let TargetGuard = "fullfp16,neon" in {
 
   // ARMv8.2-A FP16 one-operand vector intrinsics.
 
@@ -1679,7 +1679,7 @@ let TargetGuard = "fullfp16" in {
   def VCVTP_U16    : SInst<"vcvtp_u16", "U.", "hQh">;
 
   // Vector rounding
-  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)", TargetGuard = "fullfp16" in {
+  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)", TargetGuard = "fullfp16,neon" in {
     def FRINTZH      : SInst<"vrnd",  "..", "hQh">;
     def FRINTNH      : SInst<"vrndn", "..", "hQh">;
     def FRINTAH      : SInst<"vrnda", "..", "hQh">;
@@ -1728,7 +1728,7 @@ let TargetGuard = "fullfp16" in {
   // Max/Min
   def VMAXH         : SInst<"vmax", "...", "hQh">;
   def VMINH         : SInst<"vmin", "...", "hQh">;
-  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN)", TargetGuard = "fullfp16" in {
+  let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN)", TargetGuard = "fullfp16,neon" in {
     def FMAXNMH       : SInst<"vmaxnm", "...", "hQh">;
     def FMINNMH       : SInst<"vminnm", "...", "hQh">;
   }
@@ -1775,7 +1775,7 @@ def VEXTH      : WInst<"vext", "...I", "hQh">;
 def VREV64H    : WOpInst<"vrev64", "..", "hQh", OP_REV64>;
 
 // ARMv8.2-A FP16 vector intrinsics for A64 only.
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fullfp16" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fullfp16,neon" in {
 
   // Vector rounding
   def FRINTIH      : SInst<"vrndi", "..", "hQh">;
@@ -1872,11 +1872,11 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)" in {
 }
 
 // v8.2-A dot product instructions.
-let TargetGuard = "dotprod" in {
+let TargetGuard = "dotprod,neon" in {
   def DOT : SInst<"vdot", "..(<<)(<<)", "iQiUiQUi">;
   def DOT_LANE : SOpInst<"vdot_lane", "..(<<)(<<q)I", "iUiQiQUi", OP_DOT_LN>;
 }
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "dotprod" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "dotprod,neon" in {
   // Variants indexing into a 128-bit vector are A64 only.
   def UDOT_LANEQ : SOpInst<"vdot_laneq", "..(<<)(<<Q)I", "iUiQiQUi", OP_DOT_LNQ> {
     let isLaneQ = 1;
@@ -1884,7 +1884,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "d
 }
 
 // v8.2-A FP16 fused multiply-add long instructions.
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fp16fml" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "fp16fml,neon" in {
   def VFMLAL_LOW  : SInst<"vfmlal_low",  ">>..", "hQh">;
   def VFMLSL_LOW  : SInst<"vfmlsl_low",  ">>..", "hQh">;
   def VFMLAL_HIGH : SInst<"vfmlal_high", ">>..", "hQh">;
@@ -1909,7 +1909,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "f
   }
 }
 
-let TargetGuard = "i8mm" in {
+let TargetGuard = "i8mm,neon" in {
   def VMMLA   : SInst<"vmmla", "..(<<)(<<)", "QUiQi">;
   def VUSMMLA : SInst<"vusmmla", "..(<<U)(<<)", "Qi">;
 
@@ -1926,7 +1926,7 @@ let TargetGuard = "i8mm" in {
   }
 }
 
-let TargetGuard = "bf16" in {
+let TargetGuard = "bf16,neon" in {
   def VDOT_BF : SInst<"vbfdot", "..BB", "fQf">;
   def VDOT_LANE_BF : SOpInst<"vbfdot_lane", "..B(Bq)I", "fQf", OP_BFDOT_LN>;
   def VDOT_LANEQ_BF : SOpInst<"vbfdot_laneq", "..B(BQ)I", "fQf", OP_BFDOT_LNQ> {
@@ -1970,7 +1970,7 @@ multiclass VCMLA_ROTS<string type, string lanety, string laneqty> {
 }
 
 // v8.3-A Vector complex addition intrinsics
-let TargetGuard = "v8.3a,fullfp16" in {
+let TargetGuard = "v8.3a,fullfp16,neon" in {
   def VCADD_ROT90_FP16   : SInst<"vcadd_rot90", "...", "h">;
   def VCADD_ROT270_FP16  : SInst<"vcadd_rot270", "...", "h">;
   def VCADDQ_ROT90_FP16  : SInst<"vcaddq_rot90", "QQQ", "h">;
@@ -1978,7 +1978,7 @@ let TargetGuard = "v8.3a,fullfp16" in {
 
   defm VCMLA_FP16  : VCMLA_ROTS<"h", "uint32x2_t", "uint32x4_t">;
 }
-let TargetGuard = "v8.3a" in {
+let TargetGuard = "v8.3a,neon" in {
   def VCADD_ROT90   : SInst<"vcadd_rot90", "...", "f">;
   def VCADD_ROT270  : SInst<"vcadd_rot270", "...", "f">;
   def VCADDQ_ROT90  : SInst<"vcaddq_rot90", "QQQ", "f">;
@@ -1986,7 +1986,7 @@ let TargetGuard = "v8.3a" in {
 
   defm VCMLA_F32        : VCMLA_ROTS<"f", "uint64x1_t", "uint64x2_t">;
 }
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.3a" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v8.3a,neon" in {
   def VCADDQ_ROT90_FP64  : SInst<"vcaddq_rot90", "QQQ", "d">;
   def VCADDQ_ROT270_FP64 : SInst<"vcaddq_rot270", "QQQ", "d">;
 
@@ -1994,7 +1994,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "v
 }
 
 // V8.2-A BFloat intrinsics
-let TargetGuard = "bf16" in {
+let TargetGuard = "bf16,neon" in {
   def VCREATE_BF : NoTestOpInst<"vcreate", ".(IU>)", "b", OP_CAST> {
     let BigEndianSafe = 1;
   }
@@ -2058,14 +2058,14 @@ let TargetGuard = "bf16" in {
   def SCALAR_CVT_F32_BF16 : SOpInst<"vcvtah_f32", "(1F>)(1!)", "b", OP_CVT_F32_BF16>;
 }
 
-let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16" in {
+let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
   def VCVT_BF16_F32_A32_INTERNAL : WInst<"__a32_vcvt_bf16", "BQ", "f">;
   def VCVT_BF16_F32_A32 : SOpInst<"vcvt_bf16", "BQ", "f", OP_VCVT_BF16_F32_A32>;
   def VCVT_LOW_BF16_F32_A32 : SOpInst<"vcvt_low_bf16",  "BQ", "Qf", OP_VCVT_BF16_F32_LO_A32>;
   def VCVT_HIGH_BF16_F32_A32 : SOpInst<"vcvt_high_bf16", "BBQ", "Qf", OP_VCVT_BF16_F32_HI_A32>;
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
   def VCVT_LOW_BF16_F32_A64_INTERNAL : WInst<"__a64_vcvtq_low_bf16", "BQ", "Hf">;
   def VCVT_LOW_BF16_F32_A64 : SOpInst<"vcvt_low_bf16", "BQ", "Qf", OP_VCVT_BF16_F32_LO_A64>;
   def VCVT_HIGH_BF16_F32_A64 : SInst<"vcvt_high_bf16", "BBQ", "Qf">;
@@ -2077,14 +2077,14 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "b
   def COPYQ_LANEQ_BF16 : IOpInst<"vcopy_laneq", "..I.I", "Qb", OP_COPY_LN>;
 }
 
-let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16" in {
+let ArchGuard = "!defined(__aarch64__) && !defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
   let BigEndianSafe = 1 in {
     defm VREINTERPRET_BF : REINTERPRET_CROSS_TYPES<
         "csilUcUsUiUlhfPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQPcQPsQPl", "bQb">;
   }
 }
 
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "bf16,neon" in {
   let BigEndianSafe = 1 in {
     defm VVREINTERPRET_BF : REINTERPRET_CROSS_TYPES<
         "csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk", "bQb">;
@@ -2092,7 +2092,7 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "b
 }
 
 // v8.9a/v9.4a LRCPC3 intrinsics
-let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "rcpc3" in {
+let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)", TargetGuard = "rcpc3,neon" in {
   def VLDAP1_LANE : WInst<"vldap1_lane", ".(c*!).I", "QUlQlUlldQdPlQPl">;
   def VSTL1_LANE  : WInst<"vstl1_lane", "v*(.!)I", "QUlQlUlldQdPlQPl">;
 }
diff --git a/clang/include/clang/Basic/arm_neon_incl.td b/clang/include/clang/Basic/arm_neon_incl.td
index b8155c187d1bc..3b8015daee6d9 100644
--- a/clang/include/clang/Basic/arm_neon_incl.td
+++ b/clang/include/clang/Basic/arm_neon_incl.td
@@ -265,7 +265,7 @@ class Inst <string n, string p, string t, Operation o> {
   string Prototype = p;
   string Types = t;
   string ArchGuard = "";
-  string TargetGuard = "";
+  string TargetGuard = "neon";
 
   Operation Operation = o;
   bit BigEndianSafe = 0;
diff --git a/clang/test/Sema/aarch64-neon-without-target-feature.cpp b/clang/test/Sema/aarch64-neon-without-target-feature.cpp
new file mode 100644
index 0000000000000..0831eb7c754a7
--- /dev/null
+++ b/clang/test/Sema/aarch64-neon-without-target-feature.cpp
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +dotprod  -target-feature +fullfp16 -target-feature +fp16fml -target-feature +i8mm -target-feature +bf16 -verify -emit-llvm -o - %s
+
+// REQUIRES: aarch64-registered-target
+
+// This test is testing the diagnostics that Clang emits when compiling without '+neon'.
+
+#include <arm_neon.h>
+
+void undefined(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t v16i8, uint8x8_t v8i8, float32x2_t v2f32, float32x4_t v4f32, float16x4_t v4f16, float64x2_t v2f64, bfloat16x4_t v4bf16, __bf16 bf16, poly64_t poly64, poly64x2_t poly64x2) {
+  // dotprod
+  vdot_u32(v2i32, v8i8, v8i8); // expected-error {{always_inline function 'vdot_u32' requires target feature 'neon'}}
+  vdot_laneq_u32(v2i32, v8i8, v16i8, 1); // expected-error {{always_inline function 'vdot_u32' requires target feature 'neon'}} expected-error {{'__builtin_neon_splat_laneq_v' needs target feature neon}}
+  // fp16
+  vceqz_f16(v4f16); // expected-error {{always_inline function 'vceqz_f16' requires target feature 'neon'}}
+  vrnd_f16(v4f16); // expected-error {{always_inline function 'vrnd_f16' requires target feature 'neon'}}
+  vmaxnm_f16(v4f16, v4f16); // expected-error {{always_inline function 'vmaxnm_f16' requires target feature 'neon'}}
+  vrndi_f16(v4f16); // expected-error {{always_inline function 'vrndi_f16' requires target feature 'neon'}}
+  // fp16fml depends on fp-armv8
+  vfmlal_low_f16(v2f32, v4f16, v4f16); // expected-error {{always_inline function 'vfmlal_low_f16' requires target feature 'neon'}}
+  // i8mm
+  vmmlaq_s32(v4i32, v8i16, v8i16); // expected-error {{always_inline function 'vmmlaq_s32' requires target feature 'neon'}}
+  vusdot_laneq_s32(v2i32, v8i8, v8i16, 0); // expected-error {{always_inline function 'vusdot_s32' requires target feature 'neon'}} expected-error {{'__builtin_neon_splat_laneq_v' needs target feature neon}}
+  // bf16
+  vbfdot_f32(v2f32, v4bf16, v4bf16); // expected-error {{always_inline function 'vbfdot_f32' requires target feature 'neon'}}
+  vcreate_bf16(10);
+  vdup_lane_bf16(v4bf16, 2); // expected-error {{'__builtin_neon_splat_lane_bf16' needs target feature bf16,neon}}
+  vdup_n_bf16(bf16); // expected-error {{always_inline function 'vdup_n_bf16' requires target feature 'neon'}}
+  vld1_bf16(0); // expected-error {{'__builtin_neon_vld1_bf16' needs target feature bf16,neon}}
+  vcvt_f32_bf16(v4bf16); // expected-error {{always_inline function 'vcvt_f32_bf16' requires target feature 'neon'}}
+  vcvt_bf16_f32(v4f32); // expected-error {{always_inline function 'vcvt_bf16_f32' requires target feature 'neon'}}
+  vmull_p64(poly64, poly64);  // expected-error {{always_inline function 'vmull_p64' requires target feature 'neon'}}
+  vmull_high_p64(poly64x2, poly64x2);  // expected-error {{always_inline function 'vmull_high_p64' requires target feature 'neon'}}
+  vtrn1_s8(v8i8, v8i8); // expected-error {{always_inline function 'vtrn1_s8' requires target feature 'neon'}}
+  vqabsq_s16(v8i16); // expected-error {{always_inline function 'vqabsq_s16' requires target feature 'neon'}}
+  vbslq_s16(v8i16, v8i16, v8i16);// expected-error {{always_inline function 'vbslq_s16' requires target feature 'neon'}}
+}
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 626031d38cf00..30fbb8c5d65e5 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -952,7 +952,7 @@ std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
   char typeCode = '\0';
   bool printNumber = true;
 
-  if (CK == ClassB && TargetGuard == "")
+  if (CK == ClassB && TargetGuard == "neon")
     return "";
 
   if (T.isBFloat16())
@@ -976,7 +976,7 @@ std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
       break;
     }
   }
-  if (CK == ClassB && TargetGuard == "") {
+  if (CK == ClassB && TargetGuard == "neon") {
     typeCode = '\0';
   }
 
@@ -1078,7 +1078,7 @@ std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const {
     S += "_" + getInstTypeCode(InBaseType, LocalCK);
   }
 
-  if (LocalCK == ClassB && TargetGuard == "")
+  if (LocalCK == ClassB && TargetGuard == "neon")
     S += "_v";
 
   // Insert a 'q' before the first '_' character so that it ends up before

From 65e86a8f3fb44dc09dc2e08526d69e3a57f63995 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 22 Jul 2024 13:10:59 +0100
Subject: [PATCH 409/486] [X86] combineConcatVectorOps -
 concat(bitcast(),bitcast()) -> bitcast(concat())

When concatenating subvector ops together, try to use the pre-bitcasted subop to help expose further combines.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |   11 +
 llvm/test/CodeGen/X86/pmul.ll                 |    2 +-
 .../vector-interleaved-store-i16-stride-6.ll  | 3998 +++++-----
 .../vector-interleaved-store-i16-stride-7.ll  | 7086 ++++++++---------
 .../vector-interleaved-store-i8-stride-5.ll   |  516 +-
 .../vector-interleaved-store-i8-stride-6.ll   | 1890 +++--
 .../vector-interleaved-store-i8-stride-7.ll   | 2388 +++---
 .../vector-interleaved-store-i8-stride-8.ll   | 1088 ++-
 llvm/test/CodeGen/X86/widen_bitcnt.ll         |  192 +-
 9 files changed, 8544 insertions(+), 8627 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 2959902c78675..f06b962b74df8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56165,6 +56165,17 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
       SmallVector<SDValue> Subs;
       for (SDValue SubOp : SubOps)
         Subs.push_back(SubOp.getOperand(I));
+      // Attempt to peek through bitcasts and concat the original subvectors.
+      EVT SubVT = peekThroughBitcasts(Subs[0]).getValueType();
+      if (SubVT.isSimple() && SubVT.isVector()) {
+        EVT ConcatVT =
+            EVT::getVectorVT(*DAG.getContext(), SubVT.getScalarType(),
+                             SubVT.getVectorElementCount() * Subs.size());
+        for (SDValue &Sub : Subs)
+          Sub = DAG.getBitcast(SubVT, Sub);
+        return DAG.getBitcast(
+            VT, DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Subs));
+      }
       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
     };
     auto IsConcatFree = [](MVT VT, ArrayRef<SDValue> SubOps, unsigned Op) {
diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll
index fe791e45eff99..2b475644a38cf 100644
--- a/llvm/test/CodeGen/X86/pmul.ll
+++ b/llvm/test/CodeGen/X86/pmul.ll
@@ -964,7 +964,7 @@ define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind  {
 ; AVX512F-LABEL: mul_v64i8:
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512F-NEXT:    vpand %ymm2, %ymm3, %ymm4
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
 ; AVX512F-NEXT:    vpmaddubsw %ymm4, %ymm5, %ymm4
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
index b33cc83ac3f79..c725dcd972cd5 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
@@ -3931,49 +3931,40 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512-LABEL: store_i16_stride6_vf32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm9
-; AVX512-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm9[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm12
-; AVX512-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm12[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm4
+; AVX512-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm8
+; AVX512-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm8[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm12[4],ymm9[4],ymm12[5],ymm9[5],ymm12[6],ymm9[6],ymm12[7],ymm9[7],ymm12[12],ymm9[12],ymm12[13],ymm9[13],ymm12[14],ymm9[14],ymm12[15],ymm9[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm8[4],ymm4[4],ymm8[5],ymm4[5],ymm8[6],ymm4[6],ymm8[7],ymm4[7],ymm8[12],ymm4[12],ymm8[13],ymm4[13],ymm8[14],ymm4[14],ymm8[15],ymm4[15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[1,2,3,3,5,6,7,7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm8
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm8[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm9
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm9[2,1,2,3,6,5,6,7]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm10
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm10[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm11
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[2,1,2,3,6,5,6,7]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm2[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm10[4],ymm8[4],ymm10[5],ymm8[5],ymm10[6],ymm8[6],ymm10[7],ymm8[7],ymm10[12],ymm8[12],ymm10[13],ymm8[13],ymm10[14],ymm8[14],ymm10[15],ymm8[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm11[4],ymm9[4],ymm11[5],ymm9[5],ymm11[6],ymm9[6],ymm11[7],ymm9[7],ymm11[12],ymm9[12],ymm11[13],ymm9[13],ymm11[14],ymm9[14],ymm11[15],ymm9[15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[3,3,3,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm2
 ; AVX512-NEXT:    movw $18724, %ax # imm = 0x4924
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm2, %ymm0
-; AVX512-NEXT:    vmovdqa 32(%r8), %ymm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm1, %ymm3
+; AVX512-NEXT:    vmovdqa 32(%r8), %ymm12
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
+; AVX512-NEXT:    vpshufb %ymm1, %ymm12, %ymm3
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3],ymm3[4],ymm0[5,6],ymm3[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm1[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm12[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm17 = zmm2[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqa 32(%r9), %ymm2
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm18 = ymm0[2,2,2,3]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm2[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm25
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm19 = ymm0[2,1,2,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm16
 ; AVX512-NEXT:    vmovdqa (%rcx), %ymm2
 ; AVX512-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vmovdqa (%rdx), %ymm3
@@ -3983,178 +3974,179 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[1,2,3,3,5,6,7,7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm6
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm5
 ; AVX512-NEXT:    vmovdqa (%rsi), %ymm0
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm0[2,1,2,3,6,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm5[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vmovdqa (%rdi), %ymm5
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm11 = ymm5[2,1,2,3,6,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm11 = ymm11[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm11[0],ymm7[0],ymm11[1],ymm7[1],ymm11[2],ymm7[2],ymm11[3],ymm7[3],ymm11[8],ymm7[8],ymm11[9],ymm7[9],ymm11[10],ymm7[10],ymm11[11],ymm7[11]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,2,3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm11 = ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[3,3,3,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm7, %zmm7
-; AVX512-NEXT:    vmovdqa32 %zmm6, %zmm7 {%k1}
-; AVX512-NEXT:    vextracti64x4 $1, %zmm7, %ymm11
-; AVX512-NEXT:    vmovdqa (%r8), %ymm6
-; AVX512-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,1,2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm11[0],ymm4[1],ymm11[2,3],ymm4[4],ymm11[5,6],ymm4[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm11 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm7 = ymm11[0],ymm7[1,2],ymm11[3],ymm7[4,5],ymm11[6],ymm7[7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm20 = zmm7[0,1,2,3],zmm4[4,5,6,7]
-; AVX512-NEXT:    vmovdqa (%r9), %ymm4
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm4[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm7 = ymm7[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm21 = ymm7[2,2,2,3]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm4[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm7[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm22 = ymm7[2,1,2,3]
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm13
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm14
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm0[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512-NEXT:    vmovdqa (%rdi), %ymm10
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm10[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm7[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[8],ymm6[8],ymm7[9],ymm6[9],ymm7[10],ymm6[10],ymm7[11],ymm6[11]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,1,2,3]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm7 = ymm10[4],ymm0[4],ymm10[5],ymm0[5],ymm10[6],ymm0[6],ymm10[7],ymm0[7],ymm10[12],ymm0[12],ymm10[13],ymm0[13],ymm10[14],ymm0[14],ymm10[15],ymm0[15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[3,3,3,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512-NEXT:    vmovdqa32 %zmm5, %zmm6 {%k1}
+; AVX512-NEXT:    vextracti64x4 $1, %zmm6, %ymm5
+; AVX512-NEXT:    vmovdqa (%r8), %ymm14
+; AVX512-NEXT:    vpshufb %ymm1, %ymm14, %ymm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0],ymm1[1],ymm5[2,3],ymm1[4],ymm5[5,6],ymm1[7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm14[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0],ymm6[1,2],ymm5[3],ymm6[4,5],ymm5[6],ymm6[7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm17
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm5
+; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm13
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm6
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm24 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]
-; AVX512-NEXT:    vpermt2d %zmm7, %zmm24, %zmm2
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm11
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm15
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm15[4],xmm11[4],xmm15[5],xmm11[5],xmm15[6],xmm11[6],xmm15[7],xmm11[7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[1,1,1,1]
-; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm18, %zmm2
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm3
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm7
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,1,1,1]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm10[0],ymm0[0],ymm10[1],ymm0[1],ymm10[2],ymm0[2],ymm10[3],ymm0[3],ymm10[8],ymm0[8],ymm10[9],ymm0[9],ymm10[10],ymm0[10],ymm10[11],ymm0[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm7, %zmm0
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm2, %ymm0
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm6[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,2]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm5[2],ymm0[3,4],ymm5[5],ymm0[6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm5
-; AVX512-NEXT:    vmovdqa (%r8), %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm0, %xmm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm7[1],ymm2[2,3],ymm7[4],ymm2[5,6],ymm7[7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm2[0,1,2,3],zmm5[4,5,6,7]
-; AVX512-NEXT:    vmovdqa (%r9), %xmm2
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,2,2,1,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm16 = ymm5[0,1,0,1]
-; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm12[0],ymm9[0],ymm12[1],ymm9[1],ymm12[2],ymm9[2],ymm12[3],ymm9[3],ymm12[8],ymm9[8],ymm12[9],ymm9[9],ymm12[10],ymm9[10],ymm12[11],ymm9[11]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; AVX512-NEXT:    vpermt2d %zmm7, %zmm24, %zmm9
-; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm12
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm4[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm24 = ymm4[2,2,2,2]
-; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[2],ymm8[2],ymm10[3],ymm8[3],ymm10[8],ymm8[8],ymm10[9],ymm8[9],ymm10[10],ymm8[10],ymm10[11],ymm8[11]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm4[4],xmm12[4],xmm4[5],xmm12[5],xmm4[6],xmm12[6],xmm4[7],xmm12[7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,1,1,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT:    vmovdqa32 %zmm7, %zmm9 {%k1}
-; AVX512-NEXT:    vextracti64x4 $1, %zmm9, %ymm7
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm14[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1],ymm1[2],ymm7[3,4],ymm1[5],ymm7[6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512-NEXT:    vmovdqa 32(%r8), %xmm7
-; AVX512-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0],ymm6[1],ymm9[2,3],ymm6[4],ymm9[5,6],ymm6[7]
-; AVX512-NEXT:    vmovdqa 32(%r9), %xmm10
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm6[0,1,2,3],zmm1[4,5,6,7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]
-; AVX512-NEXT:    vpermt2d %zmm3, %zmm5, %zmm6
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm12[0,1,2,1]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm8 = xmm4[0,1,2,1]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,7,6,5]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm8[4],xmm3[4],xmm8[5],xmm3[5],xmm8[6],xmm3[6],xmm8[7],xmm3[7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm10[2,3,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,1,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm3[0,1,0,1]
-; AVX512-NEXT:    vmovdqa64 %ymm25, %ymm3
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm4[0,1,2,3],zmm9[0,1,0,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm14 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
+; AVX512-NEXT:    vmovdqa (%r8), %xmm10
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm0
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
+; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm0
+; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm2, %zmm2
+; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm14
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm15
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm8[0],ymm4[0],ymm8[1],ymm4[1],ymm8[2],ymm4[2],ymm8[3],ymm4[3],ymm8[8],ymm4[8],ymm8[9],ymm4[9],ymm8[10],ymm4[10],ymm8[11],ymm4[11]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]
+; AVX512-NEXT:    vpermt2d %zmm8, %zmm18, %zmm4
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,1,1,1]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm11[0],ymm9[0],ymm11[1],ymm9[1],ymm11[2],ymm9[2],ymm11[3],ymm9[3],ymm11[8],ymm9[8],ymm11[9],ymm9[9],ymm11[10],ymm9[10],ymm11[11],ymm9[11]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
+; AVX512-NEXT:    vmovdqa32 %zmm8, %zmm4 {%k1}
+; AVX512-NEXT:    vextracti64x4 $1, %zmm4, %ymm8
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm12[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,2]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1],ymm9[2],ymm8[3,4],ymm9[5],ymm8[6,7]
+; AVX512-NEXT:    vmovdqa 32(%r8), %xmm12
+; AVX512-NEXT:    vpshufb %xmm1, %xmm12, %xmm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0],ymm1[1],ymm4[2,3],ymm1[4],ymm4[5,6],ymm1[7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm1, %zmm4
+; AVX512-NEXT:    vmovdqa 32(%r9), %ymm11
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm11[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,2,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm11[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm1[2,1,2,3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm13 = xmm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm18, %zmm1
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm14[0,1,2,1]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm15[0,1,2,1]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,7,6,5]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,2,1]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm13[0,1,2,3],zmm0[0,1,0,1]
 ; AVX512-NEXT:    movw $9362, %ax # imm = 0x2492
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovdqa32 %zmm6, %zmm4 {%k1}
-; AVX512-NEXT:    vextracti64x4 $1, %zmm4, %ymm6
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm7[2,1,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0],ymm6[1,2],ymm9[3],ymm6[4,5],ymm9[6],ymm6[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm6
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero
-; AVX512-NEXT:    vpbroadcastq %xmm7, %ymm7
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm7[2],ymm4[3,4],ymm7[5],ymm4[6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm4[0,1,2,3],zmm6[4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm10[0,0,2,1,4,5,6,7]
-; AVX512-NEXT:    vpbroadcastq %xmm6, %ymm6
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm10[0,2,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,4,4,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,2,1]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm10 = xmm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm12 = xmm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
-; AVX512-NEXT:    vpermt2d %zmm10, %zmm5, %zmm9
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm11[0,1,2,1]
+; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm12[2,1,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,2,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm13[0],ymm1[1,2],ymm13[3],ymm1[4,5],ymm13[6],ymm1[7]
+; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm12 = xmm12[0],zero,xmm12[1],zero,xmm12[2],zero,xmm12[3],zero
+; AVX512-NEXT:    vpbroadcastq %xmm12, %ymm12
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm12[2],ymm0[3,4],ymm12[5],ymm0[6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm12
+; AVX512-NEXT:    vmovdqa (%r9), %ymm0
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm1[2,2,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm14 = ymm1[2,1,2,3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm6 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512-NEXT:    vpermt2d %zmm5, %zmm18, %zmm1
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[0,1,2,1]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,5]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm10 = xmm15[0,1,2,1]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,7,6,5]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm10[0,1,2,3],zmm5[0,1,0,1]
-; AVX512-NEXT:    vmovdqa32 %zmm9, %zmm5 {%k1}
-; AVX512-NEXT:    vextracti64x4 $1, %zmm5, %ymm9
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm0[2,1,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0],ymm9[1,2],ymm10[3],ymm9[4,5],ymm10[6],ymm9[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm9
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-NEXT:    vpbroadcastq %xmm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1],ymm0[2],ymm5[3,4],ymm0[5],ymm5[6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm9[4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm2[0,0,2,1,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm7[0,1,2,1]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,7,6,5]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm5[0,1,0,1]
+; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm3 {%k1}
+; AVX512-NEXT:    vextracti64x4 $1, %zmm3, %ymm1
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm10[2,1,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,2,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0],ymm1[1,2],ymm5[3],ymm1[4,5],ymm5[6],ymm1[7]
+; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm5 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero
 ; AVX512-NEXT:    vpbroadcastq %xmm5, %ymm5
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm5[2],ymm3[3,4],ymm5[5],ymm3[6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; AVX512-NEXT:    vmovdqa (%r9), %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,2,2,1,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,1,0,1]
+; AVX512-NEXT:    vmovdqa 32(%r9), %xmm6
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm7[0,2,2,1,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm11[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm6[0,0,2,1,4,5,6,7]
+; AVX512-NEXT:    vpbroadcastq %xmm11, %ymm11
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,2,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,4,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm3[0,0,2,1,4,5,6,7]
+; AVX512-NEXT:    vpbroadcastq %xmm15, %ymm15
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vinserti64x4 $1, %ymm19, %zmm18, %zmm9
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogd $184, %zmm17, %zmm10, %zmm9
-; AVX512-NEXT:    vinserti64x4 $1, %ymm22, %zmm21, %zmm11
-; AVX512-NEXT:    vpternlogd $184, %zmm20, %zmm10, %zmm11
-; AVX512-NEXT:    vinserti64x4 $1, %ymm24, %zmm16, %zmm10
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd $184, %zmm23, %zmm12, %zmm10
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm8, %zmm3
-; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm12, %zmm3
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm1
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogd $184, %zmm4, %zmm6, %zmm1
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm5, %zmm2
-; AVX512-NEXT:    vpternlogd $184, %zmm0, %zmm6, %zmm2
-; AVX512-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm1, 192(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm3, 256(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm10, 64(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm11, 128(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm9, 320(%rax)
+; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
+; AVX512-NEXT:    vpternlogd $184, %zmm16, %zmm9, %zmm8
+; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm13
+; AVX512-NEXT:    vpternlogd $184, %zmm17, %zmm9, %zmm13
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm5, %zmm0
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogd $184, %zmm2, %zmm5, %zmm0
+; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm7, %zmm2
+; AVX512-NEXT:    vpternlogd $184, %zmm4, %zmm5, %zmm2
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm11, %zmm4
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512-NEXT:    vpternlogd $184, %zmm12, %zmm5, %zmm4
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm15, %zmm3
+; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm5, %zmm3
+; AVX512-NEXT:    vmovdqa64 %zmm3, (%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm4, 192(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm2, 256(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm13, 128(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm8, 320(%rax)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -4168,16 +4160,16 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm4, %ymm3
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[8],ymm1[8],ymm3[9],ymm1[9],ymm3[10],ymm1[10],ymm3[11],ymm1[11]
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[12],ymm2[12],ymm4[13],ymm2[13],ymm4[14],ymm2[14],ymm4[15],ymm2[15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm24
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm25
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm23
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm24
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [2,1,2,3,11,11,11,11]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm6, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm12
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm11
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm13
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [5,6,5,6,5,6,7,7]
 ; AVX512-FCP-NEXT:    vpermd %ymm3, %ymm10, %ymm3
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm4 = ymm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm12[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm4 = ymm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm11[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm5 = ymm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm13[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]
@@ -4185,17 +4177,17 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    movw $18724, %ax # imm = 0x4924
 ; AVX512-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm1 {%k1}
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [8,21,10,11,20,13,14,23]
 ; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm2
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [12,1,2,13,4,5,14,7]
+; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm5
+; AVX512-FCP-NEXT:    vpermt2d %ymm3, %ymm12, %ymm5
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [8,21,10,11,20,13,14,23]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
 ; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm11, %zmm4
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm27
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [12,1,2,13,4,5,14,7]
-; AVX512-FCP-NEXT:    vpermt2d %ymm3, %ymm17, %ymm1
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm16 = zmm1[0,1,2,3],zmm4[0,1,2,3]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm25
+; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm17, %zmm1
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm16
 ; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]
 ; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm3, %ymm1
@@ -4206,13 +4198,13 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [2,2,0,3,10,0,10,11]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm18, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm9
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm9, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm3
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm1
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm2[4],ymm9[4],ymm2[5],ymm9[5],ymm2[6],ymm9[6],ymm2[7],ymm9[7],ymm2[12],ymm9[12],ymm2[13],ymm9[13],ymm2[14],ymm9[14],ymm2[15],ymm9[15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm28
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm27
 ; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm6, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm1
 ; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm6
@@ -4225,14 +4217,14 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm4, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm10
-; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm10, %ymm2
-; AVX512-FCP-NEXT:    vpermi2d %zmm2, %zmm0, %zmm11
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm10[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpermt2d %ymm2, %ymm17, %ymm0
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm17 = zmm0[0,1,2,3],zmm11[0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm11
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm11, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm11, %ymm2
+; AVX512-FCP-NEXT:    vpermi2d %ymm2, %ymm0, %ymm12
+; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm10, %ymm2
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm0
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm12, %zmm17
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm12
+; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm12, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm12, %ymm2
 ; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm14
@@ -4242,120 +4234,120 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm18
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm6
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm20
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm13
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]
 ; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm13, %xmm4
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm3
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm3, %xmm12
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm12[4],xmm4[4],xmm12[5],xmm4[5],xmm12[6],xmm4[6],xmm12[7],xmm4[7]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [0,0,2,1,8,9,8,9]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
-; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm21, %zmm12
-; AVX512-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm23 = [1,0,2,2,1,0,2,2]
-; AVX512-FCP-NEXT:    # ymm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]
-; AVX512-FCP-NEXT:    vpermd %ymm5, %ymm23, %ymm5
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm0
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm5
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm22 = [0,0,2,1,8,9,8,9]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm22, %zmm11
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [1,0,2,2,1,0,2,2]
+; AVX512-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]
+; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm4, %ymm8
 ; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm6 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]
 ; AVX512-FCP-NEXT:    vpbroadcastq %xmm1, %ymm1
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm1
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm8, %zmm1
 ; AVX512-FCP-NEXT:    movw $9362, %ax # imm = 0x2492
 ; AVX512-FCP-NEXT:    kmovw %eax, %k2
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm1, %zmm12 {%k2}
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm5 = [16,9,10,17,12,13,18,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm6
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm6[2,1,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2d %zmm8, %zmm5, %zmm1
-; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm8 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm22 = [0,1,8,3,4,9,6,7]
-; AVX512-FCP-NEXT:    vpermt2d %ymm8, %ymm22, %ymm12
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm19 = zmm12[0,1,2,3],zmm1[0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm8
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm8, %xmm1
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm2
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3]
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm21, %zmm2
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [0,0,0,0,8,8,0,9]
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm12
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
-; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm12, %xmm4
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm12[0,0,2,1,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm21, %zmm1
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
-; AVX512-FCP-NEXT:    vpermd %ymm4, %ymm23, %ymm4
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm14 = xmm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm15 = xmm15[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
-; AVX512-FCP-NEXT:    vpbroadcastq %xmm14, %ymm14
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm4, %zmm2 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm4
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm4[2,1,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermi2d %zmm14, %zmm2, %zmm5
-; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm14 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
-; AVX512-FCP-NEXT:    vpermt2d %ymm14, %ymm22, %ymm2
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm2[0,1,2,3],zmm5[0,1,2,3]
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm1, %zmm11 {%k2}
+; AVX512-FCP-NEXT:    vmovdqa %ymm11, %ymm6
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [16,9,10,17,12,13,18,15]
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm1
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm1[2,1,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm9, %zmm8, %zmm11
+; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm21 = [0,1,8,3,4,9,6,7]
+; AVX512-FCP-NEXT:    vpermt2d %ymm9, %ymm21, %ymm6
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm6, %zmm19
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm9
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm11
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm9, %xmm6
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm22, %zmm5
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512-FCP-NEXT:    vpermd %ymm2, %ymm4, %ymm2
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm6 = xmm15[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; AVX512-FCP-NEXT:    vpbroadcastq %xmm4, %ymm4
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm2, %zmm5 {%k2}
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm2
+; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX512-FCP-NEXT:    vpermi2d %ymm4, %ymm5, %ymm21
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm2[2,1,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm8, %zmm5
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,0,0,0,8,8,0,9]
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm6, %xmm15
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm6[0,0,2,1,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm15, %zmm4, %zmm14
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm21, %zmm21
 ; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm5
-; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm5, %xmm7
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm5[0,0,2,1,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm7, %zmm21, %zmm14
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm28, %ymm2
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm2[0],ymm9[0],ymm2[1],ymm9[1],ymm2[2],ymm9[2],ymm2[3],ymm9[3],ymm2[8],ymm9[8],ymm2[9],ymm9[9],ymm2[10],ymm9[10],ymm2[11],ymm9[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm15 = [1,1,1,1,10,10,10,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm7, %zmm15, %zmm0
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [1,2,1,2,0,0,3,3,13,12,10,10,13,12,14,14]
-; AVX512-FCP-NEXT:    vpermd %zmm18, %zmm7, %zmm18
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm18 {%k1}
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [8,9,20,11,12,21,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm8
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm5[0,0,2,1,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm8, %zmm4, %zmm15
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm27, %ymm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm9 = [1,1,1,1,10,10,10,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm9, %zmm8
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [1,2,1,2,0,0,3,3,13,12,10,10,13,12,14,14]
+; AVX512-FCP-NEXT:    vpermd %zmm18, %zmm4, %zmm11
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm8, %zmm11 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm18 = [0,9,2,3,8,5,6,11]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm11, %ymm22
+; AVX512-FCP-NEXT:    vpermt2d %ymm2, %ymm18, %ymm22
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [8,9,20,11,12,21,14,15]
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm10[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm18, %zmm10
-; AVX512-FCP-NEXT:    vpermt2d %zmm8, %zmm0, %zmm10
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm4, %xmm4
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm21 = [0,9,2,3,8,5,6,11]
-; AVX512-FCP-NEXT:    vpermt2d %ymm4, %ymm21, %ymm18
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm11[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm5, %xmm8
+; AVX512-FCP-NEXT:    vpermt2d %zmm8, %zmm2, %zmm11
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm12[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm5, %xmm8
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [0,0,0,1,0,10,10,0]
-; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm5, %zmm8
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm24, %ymm4
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm25, %ymm9
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm4[0],ymm9[0],ymm4[1],ymm9[1],ymm4[2],ymm9[2],ymm4[3],ymm9[3],ymm4[8],ymm9[8],ymm4[9],ymm9[9],ymm4[10],ymm9[10],ymm4[11],ymm9[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm13[4],xmm3[5],xmm13[5],xmm3[6],xmm13[6],xmm3[7],xmm13[7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm15, %zmm3
-; AVX512-FCP-NEXT:    vpermd %zmm20, %zmm7, %zmm4
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm4 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm27, %ymm3
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpermi2d %zmm3, %zmm4, %zmm0
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
-; AVX512-FCP-NEXT:    vpermt2d %ymm2, %ymm21, %ymm4
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm12, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm26, %ymm3
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm5, %zmm2
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm2
+; AVX512-FCP-NEXT:    vpermt2q %zmm10, %zmm5, %zmm8
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm23, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm24, %ymm7
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm10[0],ymm7[0],ymm10[1],ymm7[1],ymm10[2],ymm7[2],ymm10[3],ymm7[3],ymm10[8],ymm7[8],ymm10[9],ymm7[9],ymm10[10],ymm7[10],ymm10[11],ymm7[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm10, %zmm9, %zmm0
+; AVX512-FCP-NEXT:    vpermd %zmm20, %zmm4, %zmm4
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm4 {%k1}
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm1, %xmm0
+; AVX512-FCP-NEXT:    vpermi2d %ymm0, %ymm4, %ymm18
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm25, %ymm0
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm2, %zmm4
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm6, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm26, %ymm1
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm5, %zmm0
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm18, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm1, %zmm2, %zmm0
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 256(%rax)
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm18[0,1,2,3],zmm10[0,1,2,3]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm8
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 256(%rax)
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm22, %zmm0
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm2, %zmm8
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, 64(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm22, %zmm0, %zmm14
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm14, (%rax)
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm19, %zmm0, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm21, %zmm0, %zmm15
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm15, (%rax)
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm19, %zmm0, %zmm14
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm14, 192(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogd $184, %zmm17, %zmm0, %zmm1
@@ -4370,261 +4362,253 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-LABEL: store_i16_stride6_vf32:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm6
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm1
-; AVX512DQ-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm11
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm2
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm9
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm25
 ; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm26
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]
 ; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm12
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm8
 ; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm4
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm15
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm10
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm5
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm5[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm28
-; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm29
+; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm27
+; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm28
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,0,1]
 ; AVX512DQ-NEXT:    movw $9362, %ax # imm = 0x2492
 ; AVX512DQ-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
-; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm5
+; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm11
 ; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm4
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm4[2,1,3,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0],ymm0[1,2],ymm3[3],ymm0[4,5],ymm3[6],ymm0[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
-; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm31
+; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm29
 ; AVX512DQ-NEXT:    vpbroadcastq %xmm3, %ymm3
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm4
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm1[0,0,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpbroadcastq %xmm0, %ymm26
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm1[0,2,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm30
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm19 = ymm0[0,0,2,1]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm16
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
 ; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm25
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm24
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
 ; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm12[0,1,2,1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm8[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm15[0,1,2,1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm10[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm15[0],xmm12[0],xmm15[1],xmm12[1],xmm15[2],xmm12[2],xmm15[3],xmm12[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm31
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm1[0,1,0,1]
 ; AVX512DQ-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm5[2,1,3,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm11[2,1,3,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm17
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero
 ; AVX512DQ-NEXT:    vpbroadcastq %xmm2, %ymm2
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm4[0,0,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpbroadcastq %xmm0, %ymm22
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm4[0,2,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm16
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm18 = ymm0[0,0,2,1]
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm8
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm8[2,1,2,3,6,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm17
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm13
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm13[2,1,2,3,6,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm7
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm7[2,1,2,3,6,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm14
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm14[2,1,2,3,6,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm14[4],ymm13[4],ymm14[5],ymm13[5],ymm14[6],ymm13[6],ymm14[7],ymm13[7],ymm14[12],ymm13[12],ymm14[13],ymm13[13],ymm14[14],ymm13[14],ymm14[15],ymm13[15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[3,3,3,3]
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm13
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm13[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm14
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm14[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm15
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm15[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm15[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm7
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm14[4],ymm13[4],ymm14[5],ymm13[5],ymm14[6],ymm13[6],ymm14[7],ymm13[7],ymm14[12],ymm13[12],ymm14[13],ymm13[13],ymm14[14],ymm13[14],ymm14[15],ymm13[15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,2,3,3,5,6,7,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm2[2,2,2,2]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm15[4],ymm7[5],ymm15[5],ymm7[6],ymm15[6],ymm7[7],ymm15[7],ymm7[12],ymm15[12],ymm7[13],ymm15[13],ymm7[14],ymm15[14],ymm7[15],ymm15[15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[1,2,3,3,5,6,7,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
 ; AVX512DQ-NEXT:    movw $18724, %ax # imm = 0x4924
 ; AVX512DQ-NEXT:    kmovw %eax, %k1
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm3, %zmm2, %zmm0 {%k1}
+; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm2, %zmm1, %zmm0 {%k1}
 ; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm6
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm6, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm6, %ymm1
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm20 = zmm0[0,1,2,3],zmm1[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm5
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm5[2,1,2,3,6,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm22
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm4
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm4[2,1,2,3,6,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm4
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm4[2,1,2,3,6,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm3
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[2,1,2,3,6,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm24 = ymm0[2,1,2,3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm23 = ymm0[3,3,3,3]
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm3
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm1
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm9 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm9[0],ymm2[0],ymm9[1],ymm2[1],ymm9[2],ymm2[2],ymm9[3],ymm2[3],ymm9[8],ymm2[8],ymm9[9],ymm2[9],ymm9[10],ymm2[10],ymm9[11],ymm2[11]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm9 = ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[1,2,3,3,5,6,7,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm23, %zmm24, %zmm0
-; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm9, %zmm2, %zmm0 {%k1}
-; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm2
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm2, %ymm9
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,1,2,3]
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm10
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5,6],ymm9[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm9
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0],ymm0[1,2],ymm10[3],ymm0[4,5],ymm10[6],ymm0[7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm0[0,1,2,3],zmm9[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm9
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm9[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm18 = ymm0[2,1,2,3]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm0[3,3,3,3]
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm2
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm0
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm8 = ymm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm0[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm8[0],ymm1[0],ymm8[1],ymm1[1],ymm8[2],ymm1[2],ymm8[3],ymm1[3],ymm8[8],ymm1[8],ymm8[9],ymm1[9],ymm8[10],ymm1[10],ymm8[11],ymm1[11]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm8 = ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm8 = ymm8[1,2,3,3,5,6,7,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm18, %zmm12
+; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm8, %zmm1, %zmm12 {%k1}
+; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm1, %ymm5
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,1,2,3]
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm12, %ymm8
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm8[0],ymm5[1],ymm8[2,3],ymm5[4],ymm8[5,6],ymm5[7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm1[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0],ymm12[1,2],ymm8[3],ymm12[4,5],ymm8[6],ymm12[7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm8, %zmm18
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm12
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm12[0,0,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpbroadcastq %xmm5, %ymm19
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm12[0,2,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,4,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm20 = ymm5[0,0,2,1]
+; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm5
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm5[0,0,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpbroadcastq %xmm8, %ymm21
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm5[0,2,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,4,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm23 = ymm8[0,0,2,1]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm8
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm24 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]
+; AVX512DQ-NEXT:    vpermt2d %zmm8, %zmm24, %zmm0
+; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm2
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,1,1]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm3, %zmm2, %zmm0 {%k1}
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm11, %xmm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm1
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm24 = ymm0[2,2,2,3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm0
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm11[4],xmm0[4],xmm11[5],xmm0[5],xmm11[6],xmm0[6],xmm11[7],xmm0[7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm9[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm10[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm25 = ymm10[2,1,2,3]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm10 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]
-; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm10, %zmm3
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm12[4],xmm15[5],xmm12[5],xmm15[6],xmm12[6],xmm15[7],xmm12[7]
-; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm11 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm11 = ymm11[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm5[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,1,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm4, %zmm0, %zmm3 {%k1}
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm2[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm3, %ymm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm2
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm16[2,3,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm1[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm4[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,1,2,3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm8
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm9
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm7[0],ymm15[0],ymm7[1],ymm15[1],ymm7[2],ymm15[2],ymm7[3],ymm15[3],ymm7[8],ymm15[8],ymm7[9],ymm15[9],ymm7[10],ymm15[10],ymm7[11],ymm15[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm8, %zmm24, %zmm7
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm8
+; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm9
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,1,1,1]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm14[0],ymm13[0],ymm14[1],ymm13[1],ymm14[2],ymm13[2],ymm14[3],ymm13[3],ymm14[8],ymm13[8],ymm14[9],ymm13[9],ymm14[10],ymm13[10],ymm14[11],ymm13[11]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm9, %zmm8, %zmm7 {%k1}
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm7, %ymm8
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,2]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1],ymm6[2],ymm8[3,4],ymm6[5],ymm8[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm8
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm8, %xmm3
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm5
-; AVX512DQ-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm4 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm4 = xmm5[4],mem[4],xmm5[5],mem[5],xmm5[6],mem[6],xmm5[7],mem[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm14[0],ymm13[0],ymm14[1],ymm13[1],ymm14[2],ymm13[2],ymm14[3],ymm13[3],ymm14[8],ymm13[8],ymm14[9],ymm13[9],ymm14[10],ymm13[10],ymm14[11],ymm13[11]
-; AVX512DQ-NEXT:    vpermt2d %zmm4, %zmm10, %zmm5
-; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm10
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,1,1,1]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0],ymm3[1],ymm7[2,3],ymm3[4],ymm7[5,6],ymm3[7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm6
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm6[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm7 = ymm7[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm7, %zmm4, %zmm5 {%k1}
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm5, %ymm4
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm8 = ymm6[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm8[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,1,2,3]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,2,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,1,0,1]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,2]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm6[2],ymm4[3,4],ymm6[5],ymm4[6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2,3],ymm0[4],ymm5[5,6],ymm0[7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm4[4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm30[2,3,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,2,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm9[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,2]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm12[2,3,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,2,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm19, %zmm26, %zmm6
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm6 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm18, %zmm22, %zmm8
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm21, %zmm7, %zmm8
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm25, %zmm24, %zmm7
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm20, %zmm9, %zmm7
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm10
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm23, %zmm9, %zmm10
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm2, %zmm3, %zmm1
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm2
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm2
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, 256(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 64(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm8, (%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 192(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm10, 128(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm7, 320(%rax)
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm20, %zmm19, %zmm10
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm16, %zmm11, %zmm10
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm23, %zmm21, %zmm12
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm17, %zmm11, %zmm12
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm22, %zmm4, %zmm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm7, %zmm7
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm18, %zmm4, %zmm7
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm4
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm2, %zmm5, %zmm4
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm9, %zmm1
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm3, %zmm5, %zmm1
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 256(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 64(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm12, (%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm10, 192(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm7, 128(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 320(%rax)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i16_stride6_vf32:
 ; AVX512DQ-FCP:       # %bb.0:
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm14
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm1
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm14[0],ymm1[1],ymm14[1],ymm1[2],ymm14[2],ymm1[3],ymm14[3],ymm1[8],ymm14[8],ymm1[9],ymm14[9],ymm1[10],ymm14[10],ymm1[11],ymm14[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm2
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm31
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm25
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [1,1,1,1,10,10,10,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm20, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm2
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm27
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm26
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [1,1,1,1,10,10,10,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm3
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm26
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm27
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm5
@@ -4633,63 +4617,62 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm6
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [1,2,1,2,0,0,3,3,13,12,10,10,13,12,14,14]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm21, %zmm19
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [1,2,1,2,0,0,3,3,13,12,10,10,13,12,14,14]
+; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm20, %zmm19
 ; AVX512DQ-FCP-NEXT:    movw $18724, %ax # imm = 0x4924
 ; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm1, %zmm19 {%k1}
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [8,9,20,11,12,21,14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm0
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm24
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm19, %zmm23
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm17, %zmm23
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm0, %xmm1
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [0,9,2,3,8,5,6,11]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm19, %ymm23
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm1, %ymm17, %ymm23
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm21 = [8,9,20,11,12,21,14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm9
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm4
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm4[0],ymm9[0],ymm4[1],ymm9[1],ymm4[2],ymm9[2],ymm4[3],ymm9[3],ymm4[8],ymm9[8],ymm4[9],ymm9[9],ymm4[10],ymm9[10],ymm4[11],ymm9[11]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm6, %xmm30
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm5, %xmm29
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm28
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [0,9,2,3,8,5,6,11]
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm1, %ymm6, %ymm19
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm7
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [0,0,0,1,0,10,10,0]
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm7[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm0, %xmm12
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm18, %zmm12
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm8
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm7
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm7[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm21, %zmm19
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm8
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [0,0,0,1,0,10,10,0]
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm8[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm2, %xmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm28
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm6, %zmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm11
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm10
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm11, %zmm20, %zmm0
-; AVX512DQ-FCP-NEXT:    vpermd %zmm3, %zmm21, %zmm20
-; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm0, %zmm20 {%k1}
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm10[4],xmm11[4],xmm10[5],xmm11[5],xmm10[6],xmm11[6],xmm10[7],xmm11[7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm12, %zmm18, %zmm13
+; AVX512DQ-FCP-NEXT:    vpermd %zmm5, %zmm20, %zmm20
+; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm13, %zmm20 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm12
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm12, %xmm3
+; AVX512DQ-FCP-NEXT:    vpermi2d %ymm3, %ymm20, %ymm17
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm3
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm3[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermi2d %zmm0, %zmm20, %zmm17
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm11
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm11, %xmm0
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm0, %ymm6, %ymm20
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm0, %xmm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm31
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm3[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm5, %zmm21, %zmm20
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm5
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm5, %xmm13
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm5, %xmm24
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm5
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm5[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm13
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm6, %zmm13
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm2, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm15
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm15[0],ymm0[0],ymm15[1],ymm0[1],ymm15[2],ymm0[2],ymm15[3],ymm0[3],ymm15[8],ymm0[8],ymm15[9],ymm0[9],ymm15[10],ymm0[10],ymm15[11],ymm0[11]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm14
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm14[0],ymm0[0],ymm14[1],ymm0[1],ymm14[2],ymm0[2],ymm14[3],ymm0[3],ymm14[8],ymm0[8],ymm14[9],ymm0[9],ymm14[10],ymm0[10],ymm14[11],ymm0[11]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [2,1,2,3,11,11,11,11]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
@@ -4701,126 +4684,126 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm4[0],ymm9[0],ymm4[1],ymm9[1],ymm4[2],ymm9[2],ymm4[3],ymm9[3],ymm4[8],ymm9[8],ymm4[9],ymm9[9],ymm4[10],ymm9[10],ymm4[11],ymm9[11]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm1, %zmm4, %zmm0 {%k1}
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [8,21,10,11,20,13,14,23]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, %zmm9
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [12,1,2,13,4,5,14,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm9
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm1, %ymm4, %ymm9
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm3, %ymm15
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm16
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm15, %zmm4, %zmm9
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm22 = [12,1,2,13,4,5,14,7]
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm3, %ymm22, %ymm0
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm18 = zmm0[0,1,2,3],zmm9[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm14, %ymm0
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm22 = [8,21,10,11,20,13,14,23]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm22, %zmm0
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm9, %zmm18
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm9
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm9, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm9, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm31, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm14[4],ymm9[5],ymm14[5],ymm9[6],ymm14[6],ymm9[7],ymm14[7],ymm9[12],ymm14[12],ymm9[13],ymm14[13],ymm9[14],ymm14[14],ymm9[15],ymm14[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm9[4],ymm1[5],ymm9[5],ymm1[6],ymm9[6],ymm1[7],ymm9[7],ymm1[12],ymm9[12],ymm1[13],ymm9[13],ymm1[14],ymm9[14],ymm1[15],ymm9[15]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm5, %ymm6
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]
-; AVX512DQ-FCP-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm5, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm5, %ymm3
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]
+; AVX512DQ-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm14
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,2,0,3,10,0,10,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm5, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm27, %ymm14
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm5, %zmm14
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm26, %ymm1
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm1[4],ymm14[4],ymm1[5],ymm14[5],ymm1[6],ymm14[6],ymm1[7],ymm14[7],ymm1[12],ymm14[12],ymm1[13],ymm14[13],ymm1[14],ymm14[14],ymm1[15],ymm14[15]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm21, %ymm6
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm15 = ymm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm14[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm14 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm14[0],ymm15[0],ymm14[1],ymm15[1],ymm14[2],ymm15[2],ymm14[3],ymm15[3],ymm14[8],ymm15[8],ymm14[9],ymm15[9],ymm14[10],ymm15[10],ymm14[11],ymm15[11]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,2,2,2]
-; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm6, %zmm14, %zmm0 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm24, %ymm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm27, %ymm9
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm9[4],ymm1[5],ymm9[5],ymm1[6],ymm9[6],ymm1[7],ymm9[7],ymm1[12],ymm9[12],ymm1[13],ymm9[13],ymm1[14],ymm9[14],ymm1[15],ymm9[15]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm3, %ymm21, %ymm3
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm9 = ymm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm9[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm9[0],ymm1[1],ymm9[1],ymm1[2],ymm9[2],ymm1[3],ymm9[3],ymm1[8],ymm9[8],ymm1[9],ymm9[9],ymm1[10],ymm9[10],ymm1[11],ymm9[11]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
+; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm3, %zmm1, %zmm0 {%k1}
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm7[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermi2d %ymm1, %ymm0, %ymm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm16, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermi2d %zmm1, %zmm0, %zmm4
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm1, %ymm22, %ymm0
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[0,1,2,3],zmm4[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm7, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm7, %ymm0
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm5, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm10, %xmm3
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm8, %xmm4
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm7, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm22, %zmm0
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm4, %zmm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm8, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm5, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm10, %xmm4
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [0,0,2,1,8,9,8,9]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm5, %zmm3
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [1,0,2,2,1,0,2,2]
 ; AVX512DQ-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm29, %xmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm30, %xmm9
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm29, %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm30, %xmm8
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm4, %ymm6
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm7 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm8 = xmm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm9 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm8, %ymm8
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm7, %ymm7
 ; AVX512DQ-FCP-NEXT:    movw $9362, %ax # imm = 0x2492
 ; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm8, %zmm6, %zmm3 {%k1}
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [16,9,10,17,12,13,18,15]
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm11[2,1,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm9
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm8, %zmm6, %zmm9
-; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm8 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero
+; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm7, %zmm6, %zmm3 {%k1}
+; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm6 = xmm12[0],zero,xmm12[1],zero,xmm12[2],zero,xmm12[3],zero
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm3, %ymm7
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [16,9,10,17,12,13,18,15]
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm12[2,1,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm9, %zmm8, %zmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [0,1,8,3,4,9,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm6, %ymm9, %ymm7
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm7, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm10, %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm5, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm2
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm2, %xmm5
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,1,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm10 = [0,0,0,0,8,8,0,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm10, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm11, %xmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm14, %xmm7
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm7[4],xmm10[4],xmm7[5],xmm10[5],xmm7[6],xmm10[6],xmm7[7],xmm10[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm7, %zmm5, %zmm10
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm5 = [0,1,8,3,4,9,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm8, %ymm5, %ymm3
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm9[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm4, %ymm4
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm7 = xmm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm8 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
-; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm7, %ymm7
-; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm7, %zmm4, %zmm10 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm31, %xmm7
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm7, %xmm4
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm7[0,0,2,1,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm9 = [0,0,0,0,8,8,0,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm9, %zmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm11
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm11[2,1,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermi2d %zmm4, %zmm10, %zmm6
-; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm28[0],zero,xmm28[1],zero,xmm28[2],zero,xmm28[3],zero
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm4, %ymm5, %ymm10
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm10[0,1,2,3],zmm6[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm6, %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm5, %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm11 = xmm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm11[0],xmm5[0],xmm11[1],xmm5[1],xmm11[2],xmm5[2],xmm11[3],xmm5[3]
+; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm5, %ymm5
+; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm5, %zmm4, %zmm6 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
+; AVX512DQ-FCP-NEXT:    vpermi2d %ymm4, %ymm6, %ymm9
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm5[2,1,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm4, %zmm8, %zmm6
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm9, %zmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm6
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm6, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,0,2,1,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm9, %zmm6
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm10, %zmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm4, %zmm5, %zmm6
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, (%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm5, %zmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 192(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm1, %zmm3, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm18, %zmm3, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 320(%rax)
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm20[0,1,2,3],zmm17[0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm5, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 192(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm2, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 128(%rax)
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm18, %zmm2, %zmm14
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm14, 320(%rax)
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm17, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm1, %zmm13
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, 256(%rax)
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm19[0,1,2,3],zmm23[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm1, %zmm12
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, 64(%rax)
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm23, %zmm0
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm1, %zmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm15, 64(%rax)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
@@ -8334,901 +8317,875 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512-LABEL: store_i16_stride6_vf64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    subq $600, %rsp # imm = 0x258
+; AVX512-NEXT:    subq $392, %rsp # imm = 0x188
 ; AVX512-NEXT:    vmovdqa 96(%rcx), %ymm9
 ; AVX512-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm9[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vmovdqa 96(%rdx), %ymm6
-; AVX512-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm6[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vmovdqa 96(%rdx), %ymm4
+; AVX512-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm6[4],ymm9[4],ymm6[5],ymm9[5],ymm6[6],ymm9[6],ymm6[7],ymm9[7],ymm6[12],ymm9[12],ymm6[13],ymm9[13],ymm6[14],ymm9[14],ymm6[15],ymm9[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm9[4],ymm4[5],ymm9[5],ymm4[6],ymm9[6],ymm4[7],ymm9[7],ymm4[12],ymm9[12],ymm4[13],ymm9[13],ymm4[14],ymm9[14],ymm4[15],ymm9[15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[1,2,3,3,5,6,7,7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqa 96(%rsi), %ymm10
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm10[2,1,2,3,6,5,6,7]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vmovdqa 96(%rdi), %ymm8
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm8[2,1,2,3,6,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm2[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
+; AVX512-NEXT:    vmovdqa 96(%rdi), %ymm6
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm6[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[8],ymm1[8],ymm3[9],ymm1[9],ymm3[10],ymm1[10],ymm3[11],ymm1[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm8[4],ymm10[4],ymm8[5],ymm10[5],ymm8[6],ymm10[6],ymm8[7],ymm10[7],ymm8[12],ymm10[12],ymm8[13],ymm10[13],ymm8[14],ymm10[14],ymm8[15],ymm10[15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[3,3,3,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm6[4],ymm10[4],ymm6[5],ymm10[5],ymm6[6],ymm10[6],ymm6[7],ymm10[7],ymm6[12],ymm10[12],ymm6[13],ymm10[13],ymm6[14],ymm10[14],ymm6[15],ymm10[15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[3,3,3,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
 ; AVX512-NEXT:    movw $18724, %ax # imm = 0x4924
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
-; AVX512-NEXT:    vmovdqa 96(%r8), %ymm5
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
-; AVX512-NEXT:    vpshufb %ymm7, %ymm5, %ymm2
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm5[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 96(%r9), %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512-NEXT:    vmovdqa 96(%r8), %ymm3
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm14 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
+; AVX512-NEXT:    vpshufb %ymm14, %ymm3, %ymm5
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,1,2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2,3],ymm5[4],ymm0[5,6],ymm5[7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0],ymm1[1,2],ymm5[3],ymm1[4,5],ymm5[6],ymm1[7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm1[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
-; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa 64(%rcx), %ymm2
-; AVX512-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vmovdqa 64(%rdx), %ymm3
-; AVX512-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vmovdqa 64(%rcx), %ymm5
+; AVX512-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vmovdqa 64(%rdx), %ymm7
+; AVX512-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15]
-; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm21
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm19
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm7[4],ymm5[4],ymm7[5],ymm5[5],ymm7[6],ymm5[6],ymm7[7],ymm5[7],ymm7[12],ymm5[12],ymm7[13],ymm5[13],ymm7[14],ymm5[14],ymm7[15],ymm5[15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[1,2,3,3,5,6,7,7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT:    vmovdqa 64(%rsi), %ymm3
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vmovdqa 64(%rsi), %ymm7
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm7[2,1,2,3,6,5,6,7]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm4
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm4[2,1,2,3,6,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm2[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
+; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm2
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm5[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512-NEXT:    vmovdqa64 %ymm4, %ymm16
-; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm18
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[3,3,3,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm7[4],ymm2[5],ymm7[5],ymm2[6],ymm7[6],ymm2[7],ymm7[7],ymm2[12],ymm7[12],ymm2[13],ymm7[13],ymm2[14],ymm7[14],ymm2[15],ymm7[15]
+; AVX512-NEXT:    vmovdqa64 %ymm7, %ymm20
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[3,3,3,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
 ; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
-; AVX512-NEXT:    vmovdqa 64(%r8), %ymm11
-; AVX512-NEXT:    vpshufb %ymm7, %ymm11, %ymm2
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm11[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 64(%r9), %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512-NEXT:    vmovdqa 64(%r8), %ymm12
+; AVX512-NEXT:    vpshufb %ymm14, %ymm12, %ymm5
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,1,2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2,3],ymm5[4],ymm0[5,6],ymm5[7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm12[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0],ymm1[1,2],ymm5[3],ymm1[4,5],ymm5[6],ymm1[7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm1[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
-; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm13
-; AVX512-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm13[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm12
-; AVX512-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm12[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm5
+; AVX512-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm7
+; AVX512-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm12[4],ymm13[4],ymm12[5],ymm13[5],ymm12[6],ymm13[6],ymm12[7],ymm13[7],ymm12[12],ymm13[12],ymm12[13],ymm13[13],ymm12[14],ymm13[14],ymm12[15],ymm13[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm7[4],ymm5[4],ymm7[5],ymm5[5],ymm7[6],ymm5[6],ymm7[7],ymm5[7],ymm7[12],ymm5[12],ymm7[13],ymm5[13],ymm7[14],ymm5[14],ymm7[15],ymm5[15]
+; AVX512-NEXT:    vmovdqa64 %ymm7, %ymm29
+; AVX512-NEXT:    vmovdqa64 %ymm5, %ymm18
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[1,2,3,3,5,6,7,7]
-; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm4
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm4[2,1,2,3,6,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm2[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm14
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm14[2,1,2,3,6,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm14[4],ymm4[4],ymm14[5],ymm4[5],ymm14[6],ymm4[6],ymm14[7],ymm4[7],ymm14[12],ymm4[12],ymm14[13],ymm4[13],ymm14[14],ymm4[14],ymm14[15],ymm4[15]
-; AVX512-NEXT:    vmovdqa64 %ymm14, %ymm23
-; AVX512-NEXT:    vmovdqa64 %ymm4, %ymm24
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[3,3,3,3]
+; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm7
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm7[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vmovdqa %ymm7, %ymm8
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm5[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm2
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm2[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm7[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm7[0],ymm5[0],ymm7[1],ymm5[1],ymm7[2],ymm5[2],ymm7[3],ymm5[3],ymm7[8],ymm5[8],ymm7[9],ymm5[9],ymm7[10],ymm5[10],ymm7[11],ymm5[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm5[2,1,2,3]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm8[4],ymm2[5],ymm8[5],ymm2[6],ymm8[6],ymm2[7],ymm8[7],ymm2[12],ymm8[12],ymm2[13],ymm8[13],ymm2[14],ymm8[14],ymm2[15],ymm8[15]
+; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm17
+; AVX512-NEXT:    vmovdqa64 %ymm8, %ymm16
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[3,3,3,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
 ; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT:    vmovdqa 32(%r8), %ymm3
-; AVX512-NEXT:    vpshufb %ymm7, %ymm3, %ymm0
+; AVX512-NEXT:    vmovdqa 32(%r8), %ymm15
+; AVX512-NEXT:    vpshufb %ymm14, %ymm15, %ymm0
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
-; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm17
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%r9), %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2,3],ymm0[4],ymm5[5,6],ymm0[7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm15[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0],ymm1[1,2],ymm5[3],ymm1[4,5],ymm5[6],ymm1[7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm1[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm22 = ymm0[2,1,2,3]
-; AVX512-NEXT:    vmovdqa (%rcx), %ymm2
-; AVX512-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vmovdqa (%rdx), %ymm1
-; AVX512-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm25 = ymm0[2,2,2,2]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm20 = ymm3[1,2,3,3,5,6,7,7]
-; AVX512-NEXT:    vmovdqa (%rsi), %ymm4
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm4[2,1,2,3,6,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm3[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vmovdqa (%rdi), %ymm3
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm15 = ymm3[2,1,2,3,6,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm15 = ymm15[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm15[0],ymm0[0],ymm15[1],ymm0[1],ymm15[2],ymm0[2],ymm15[3],ymm0[3],ymm15[8],ymm0[8],ymm15[9],ymm0[9],ymm15[10],ymm0[10],ymm15[11],ymm0[11]
+; AVX512-NEXT:    vmovdqa (%rcx), %ymm8
+; AVX512-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm8[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vmovdqa (%rdx), %ymm13
+; AVX512-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm13[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm19 = ymm0[2,2,2,2]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm13[4],ymm8[4],ymm13[5],ymm8[5],ymm13[6],ymm8[6],ymm13[7],ymm8[7],ymm13[12],ymm8[12],ymm13[13],ymm8[13],ymm13[14],ymm8[14],ymm13[15],ymm8[15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm1[1,2,3,3,5,6,7,7]
+; AVX512-NEXT:    vmovdqa (%rsi), %ymm2
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm5[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512-NEXT:    vmovdqa (%rdi), %ymm5
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm11 = ymm5[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm11 = ymm11[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm11[0],ymm0[0],ymm11[1],ymm0[1],ymm11[2],ymm0[2],ymm11[3],ymm0[3],ymm11[8],ymm0[8],ymm11[9],ymm0[9],ymm11[10],ymm0[10],ymm11[11],ymm0[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm15 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[3,3,3,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm20 = ymm20[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm20, %zmm25, %zmm20
-; AVX512-NEXT:    vinserti64x4 $1, %ymm15, %zmm0, %zmm15
-; AVX512-NEXT:    vmovdqa32 %zmm20, %zmm15 {%k1}
-; AVX512-NEXT:    vmovdqa (%r8), %ymm0
-; AVX512-NEXT:    vpshufb %ymm7, %ymm0, %ymm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,2,3]
-; AVX512-NEXT:    vextracti64x4 $1, %zmm15, %ymm14
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm7 = ymm14[0],ymm7[1],ymm14[2,3],ymm7[4],ymm14[5,6],ymm7[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm7
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm14 = ymm0[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm11 = ymm5[4],ymm2[4],ymm5[5],ymm2[5],ymm5[6],ymm2[6],ymm5[7],ymm2[7],ymm5[12],ymm2[12],ymm5[13],ymm2[13],ymm5[14],ymm2[14],ymm5[15],ymm2[15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[3,3,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm19, %zmm7
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm0
+; AVX512-NEXT:    vmovdqa32 %zmm7, %zmm0 {%k1}
+; AVX512-NEXT:    vmovdqa (%r8), %ymm7
+; AVX512-NEXT:    vpshufb %ymm14, %ymm7, %ymm11
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,1,2,3]
+; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm14
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm11 = ymm14[0],ymm11[1],ymm14[2,3],ymm11[4],ymm14[5,6],ymm11[7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm14 = ymm7[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,2,2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0],ymm15[1,2],ymm14[3],ymm15[4,5],ymm14[6],ymm15[7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm14[0,1,2,3],zmm7[4,5,6,7]
-; AVX512-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm7
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0],ymm0[1,2],ymm14[3],ymm0[4,5],ymm14[6],ymm0[7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm0
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm13[0],ymm8[0],ymm13[1],ymm8[1],ymm13[2],ymm8[2],ymm13[3],ymm8[3],ymm13[8],ymm8[8],ymm13[9],ymm8[9],ymm13[10],ymm8[10],ymm13[11],ymm8[11]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm23 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm13
 ; AVX512-NEXT:    vmovdqa (%rdx), %xmm14
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm14[4],xmm7[4],xmm14[5],xmm7[5],xmm14[6],xmm7[6],xmm14[7],xmm7[7]
-; AVX512-NEXT:    vmovdqa64 %xmm14, %xmm26
-; AVX512-NEXT:    vpermt2d %zmm2, %zmm29, %zmm1
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm14
-; AVX512-NEXT:    vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm2
-; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm14[4],xmm2[5],xmm14[5],xmm2[6],xmm14[6],xmm2[7],xmm14[7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,1,1]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512-NEXT:    vmovdqa32 %zmm2, %zmm1 {%k1}
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
-; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512-NEXT:    vmovdqa (%r8), %xmm2
-; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm3, %xmm14
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512-NEXT:    vpermt2d %zmm8, %zmm23, %zmm0
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm8
+; AVX512-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm8[4],xmm1[4],xmm8[5],xmm1[5],xmm8[6],xmm1[6],xmm8[7],xmm1[7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,1,1,1]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm5[0],ymm2[0],ymm5[1],ymm2[1],ymm5[2],ymm2[2],ymm5[3],ymm2[3],ymm5[8],ymm2[8],ymm5[9],ymm2[9],ymm5[10],ymm2[10],ymm5[11],ymm2[11]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm8, %zmm1
+; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm7[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
+; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1],ymm1[2],ymm5[3,4],ymm1[5],ymm5[6,7]
+; AVX512-NEXT:    vmovdqa (%r8), %xmm8
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm5
+; AVX512-NEXT:    vmovdqa %xmm2, %xmm11
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,1,0,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2,3],ymm5[4],ymm0[5,6],ymm5[7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[2],ymm9[2],ymm6[3],ymm9[3],ymm6[8],ymm9[8],ymm6[9],ymm9[9],ymm6[10],ymm9[10],ymm6[11],ymm9[11]
-; AVX512-NEXT:    vmovdqa 96(%rcx), %xmm3
-; AVX512-NEXT:    vmovdqa 96(%rdx), %xmm4
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512-NEXT:    vpermt2d %zmm0, %zmm29, %zmm6
-; AVX512-NEXT:    vmovdqa 96(%rsi), %xmm1
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm0
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,1,1]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[2],ymm10[2],ymm8[3],ymm10[3],ymm8[8],ymm10[8],ymm8[9],ymm10[9],ymm8[10],ymm10[10],ymm8[11],ymm10[11]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
-; AVX512-NEXT:    vmovdqa32 %zmm2, %zmm6 {%k1}
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm5[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]
-; AVX512-NEXT:    vextracti64x4 $1, %zmm6, %ymm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2],ymm5[3,4],ymm2[5],ymm5[6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm5
-; AVX512-NEXT:    vmovdqa 96(%r8), %xmm2
-; AVX512-NEXT:    vpshufb %xmm14, %xmm2, %xmm8
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0],ymm8[1],ymm6[2,3],ymm8[4],ymm6[5,6],ymm8[7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm27 = zmm6[0,1,2,3],zmm5[4,5,6,7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm4[0],ymm9[0],ymm4[1],ymm9[1],ymm4[2],ymm9[2],ymm4[3],ymm9[3],ymm4[8],ymm9[8],ymm4[9],ymm9[9],ymm4[10],ymm9[10],ymm4[11],ymm9[11]
+; AVX512-NEXT:    vmovdqa 96(%rcx), %xmm4
+; AVX512-NEXT:    vmovdqa 96(%rdx), %xmm5
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm23, %zmm7
+; AVX512-NEXT:    vmovdqa 96(%rsi), %xmm0
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm2
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,1,1,1]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm6[0],ymm10[0],ymm6[1],ymm10[1],ymm6[2],ymm10[2],ymm6[3],ymm10[3],ymm6[8],ymm10[8],ymm6[9],ymm10[9],ymm6[10],ymm10[10],ymm6[11],ymm10[11]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm1, %zmm1
+; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm7 {%k1}
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm3[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
+; AVX512-NEXT:    vextracti64x4 $1, %zmm7, %ymm3
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm1[2],ymm3[3,4],ymm1[5],ymm3[6,7]
+; AVX512-NEXT:    vmovdqa 96(%r8), %xmm1
+; AVX512-NEXT:    vpshufb %xmm11, %xmm1, %xmm6
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0],ymm6[1],ymm7[2,3],ymm6[4],ymm7[5,6],ymm6[7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm6, %zmm3
+; AVX512-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm31 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]
-; AVX512-NEXT:    vpermt2d %zmm3, %zmm31, %zmm5
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[0,1,2,1]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]
+; AVX512-NEXT:    vpermt2d %zmm4, %zmm31, %zmm3
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[0,1,2,1]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,5]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[0,1,2,1]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,5]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,0,1]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm4[0,1,0,1]
 ; AVX512-NEXT:    movw $9362, %ax # imm = 0x2492
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovdqa32 %zmm5, %zmm0 {%k2}
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm2[2,1,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
+; AVX512-NEXT:    vmovdqa32 %zmm3, %zmm0 {%k2}
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm1[2,1,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm3[1,2],ymm1[3],ymm3[4,5],ymm1[6],ymm3[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX512-NEXT:    vpbroadcastq %xmm2, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm28 = zmm0[0,1,2,3],zmm1[4,5,6,7]
-; AVX512-NEXT:    vmovdqa 64(%rcx), %xmm2
-; AVX512-NEXT:    vmovdqa 64(%rdx), %xmm3
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm3[1,2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7]
+; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX512-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm22
+; AVX512-NEXT:    vmovdqa 64(%rcx), %xmm3
+; AVX512-NEXT:    vmovdqa 64(%rdx), %xmm2
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-NEXT:    vpermt2d %zmm0, %zmm31, %zmm5
-; AVX512-NEXT:    vmovdqa 64(%rsi), %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,1,2,1]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]
-; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm4
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[0,1,2,1]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,7,6,5]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm0[0,1,0,1]
-; AVX512-NEXT:    vmovdqa32 %zmm5, %zmm6 {%k2}
-; AVX512-NEXT:    vmovdqa 64(%r8), %xmm0
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm0[2,1,3,3,4,5,6,7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm31, %zmm1
+; AVX512-NEXT:    vmovdqa 64(%rsi), %xmm6
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm6[0,1,2,1]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,4,7,6,5]
+; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[0,1,2,1]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,5]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,2,1]
-; AVX512-NEXT:    vextracti64x4 $1, %zmm6, %ymm8
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0],ymm8[1,2],ymm5[3],ymm8[4,5],ymm5[6],ymm8[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm5
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm8 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-NEXT:    vpbroadcastq %xmm8, %ymm8
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1],ymm8[2],ymm6[3,4],ymm8[5],ymm6[6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm30 = zmm6[0,1,2,3],zmm5[4,5,6,7]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm3
-; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11]
-; AVX512-NEXT:    vpermt2d %zmm2, %zmm29, %zmm3
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,1,1,1]
-; AVX512-NEXT:    vmovdqa64 %ymm16, %ymm2
-; AVX512-NEXT:    vmovdqa64 %ymm18, %ymm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm5[0,1,2,3],zmm4[0,1,0,1]
+; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm4 {%k2}
+; AVX512-NEXT:    vmovdqa 64(%r8), %xmm1
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm1[2,1,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,2,1]
+; AVX512-NEXT:    vextracti64x4 $1, %zmm4, %ymm7
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0],ymm7[1,2],ymm5[3],ymm7[4,5],ymm5[6],ymm7[7]
+; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm7 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX512-NEXT:    vpbroadcastq %xmm7, %ymm7
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm7[2],ymm4[3,4],ymm7[5],ymm4[6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm28
+; AVX512-NEXT:    vmovdqa 96(%r9), %ymm7
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm7[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm24 = ymm5[2,2,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm7[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm5[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm25 = ymm5[2,1,2,3]
+; AVX512-NEXT:    vmovdqa 64(%r9), %ymm4
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm4[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm26 = ymm5[2,2,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm4[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vmovdqa64 %ymm4, %ymm21
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm5[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm27 = ymm5[2,1,2,3]
+; AVX512-NEXT:    vmovdqa 32(%r9), %ymm5
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm5[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm9 = ymm9[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm30 = ymm9[2,2,2,3]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm3 = ymm3[0],mem[0],ymm3[1],mem[1],ymm3[2],mem[2],ymm3[3],mem[3],ymm3[8],mem[8],ymm3[9],mem[9],ymm3[10],mem[10],ymm3[11],mem[11]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm23, %zmm3
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]
+; AVX512-NEXT:    vmovdqa64 %ymm20, %ymm6
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm6[0],ymm2[1],ymm6[1],ymm2[2],ymm6[2],ymm2[3],ymm6[3],ymm2[8],ymm6[8],ymm2[9],ymm6[9],ymm2[10],ymm6[10],ymm2[11],ymm6[11]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm3 {%k1}
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm11[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm3 {%k1}
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm12[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm3, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
-; AVX512-NEXT:    vmovdqa %xmm14, %xmm4
-; AVX512-NEXT:    vpshufb %xmm14, %xmm0, %xmm0
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0],ymm0[1],ymm3[2,3],ymm0[4],ymm3[5,6],ymm0[7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm25 = zmm0[0,1,2,3],zmm1[4,5,6,7]
-; AVX512-NEXT:    vmovdqa (%r9), %ymm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm12[0],ymm13[0],ymm12[1],ymm13[1],ymm12[2],ymm13[2],ymm12[3],ymm13[3],ymm12[8],ymm13[8],ymm12[9],ymm13[9],ymm12[10],ymm13[10],ymm12[11],ymm13[11]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm20 = ymm1[2,2,2,3]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm16 = ymm1[2,1,2,3]
-; AVX512-NEXT:    vmovdqa (%r9), %xmm11
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm11[2,3,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,1,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[0,1,0,1]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm0[2,2,2,2]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]
+; AVX512-NEXT:    vmovdqa %xmm11, %xmm4
+; AVX512-NEXT:    vpshufb %xmm11, %xmm1, %xmm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6],ymm1[7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm20
 ; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm0
-; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm13
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7]
-; AVX512-NEXT:    vpermt2d %zmm1, %zmm29, %zmm6
-; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm15
-; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm1
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm1[4],xmm15[4],xmm1[5],xmm15[5],xmm1[6],xmm15[6],xmm1[7],xmm15[7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,1,1,1]
-; AVX512-NEXT:    vmovdqa64 %ymm23, %ymm2
-; AVX512-NEXT:    vmovdqa64 %ymm24, %ymm3
+; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm1
+; AVX512-NEXT:    vmovdqa64 %ymm29, %ymm2
+; AVX512-NEXT:    vmovdqa64 %ymm18, %ymm3
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
-; AVX512-NEXT:    vmovdqa32 %zmm8, %zmm6 {%k1}
-; AVX512-NEXT:    vmovdqa64 %ymm17, %ymm2
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm2[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,2]
-; AVX512-NEXT:    vextracti64x4 $1, %zmm6, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm14 = ymm9[0,1],ymm8[2],ymm9[3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT:    vmovdqa 32(%r8), %xmm8
-; AVX512-NEXT:    vpshufb %xmm4, %xmm8, %xmm9
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm6[0],ymm9[1],ymm6[2,3],ymm9[4],ymm6[5,6],ymm9[7]
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 32-byte Folded Reload
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm6 # 64-byte Folded Reload
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm23 # 32-byte Folded Reload
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm23 # 64-byte Folded Reload
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm22, %zmm2, %zmm12
-; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm9[0,1,2,3],zmm14[4,5,6,7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm23, %zmm9
+; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm2
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm3
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[1,1,1,1]
+; AVX512-NEXT:    vmovdqa64 %ymm16, %ymm6
+; AVX512-NEXT:    vmovdqa64 %ymm17, %ymm11
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm11[0],ymm6[0],ymm11[1],ymm6[1],ymm11[2],ymm6[2],ymm11[3],ymm6[3],ymm11[8],ymm6[8],ymm11[9],ymm6[9],ymm11[10],ymm6[10],ymm11[11],ymm6[11]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm11
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm5[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm10[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm17 = ymm10[2,1,2,3]
+; AVX512-NEXT:    vmovdqa32 %zmm11, %zmm9 {%k1}
+; AVX512-NEXT:    vextracti64x4 $1, %zmm9, %ymm11
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm12 = ymm15[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,2]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1],ymm12[2],ymm11[3,4],ymm12[5],ymm11[6,7]
+; AVX512-NEXT:    vmovdqa 32(%r8), %xmm15
+; AVX512-NEXT:    vpshufb %xmm4, %xmm15, %xmm12
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,0,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0],ymm12[1],ymm9[2,3],ymm12[4],ymm9[5,6],ymm12[7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm9, %zmm19
+; AVX512-NEXT:    vmovdqa (%r9), %ymm11
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; AVX512-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm13 = xmm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
-; AVX512-NEXT:    vpermt2d %zmm0, %zmm31, %zmm9
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm15[0,1,2,1]
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm31, %zmm12
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm2[0,1,2,1]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm1[0,1,2,1]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,7,6,5]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7]
-; AVX512-NEXT:    vmovdqa 96(%r9), %xmm13
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm15[0],xmm1[1],xmm15[1],xmm1[2],xmm15[2],xmm1[3],xmm15[3]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[2,3,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm14[0,2,2,1,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm24 = ymm14[0,1,0,1]
-; AVX512-NEXT:    vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm14 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm17 = ymm14[2,2,2,2]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,0,1]
-; AVX512-NEXT:    vmovdqa32 %zmm9, %zmm0 {%k2}
-; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm8[2,1,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm9[0],ymm1[1,2],ymm9[3],ymm1[4,5],ymm9[6],ymm1[7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm13[0,0,2,1,4,5,6,7]
-; AVX512-NEXT:    vpbroadcastq %xmm9, %ymm18
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm13[0,2,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,4,4,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm19 = ymm9[0,0,2,1]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm8 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero
-; AVX512-NEXT:    vpbroadcastq %xmm8, %ymm8
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm8[2],ymm0[3,4],ymm8[5],ymm0[6,7]
-; AVX512-NEXT:    vmovdqa 64(%r9), %xmm8
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm0[0,1,2,3],zmm1[4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm7 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm8[0,0,2,1,4,5,6,7]
-; AVX512-NEXT:    vpbroadcastq %xmm7, %ymm7
-; AVX512-NEXT:    vpermt2d %zmm1, %zmm31, %zmm0
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[0,1,2,1]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm11[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm18 = ymm0[2,2,2,3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm2[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm16 = ymm2[2,1,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,0,1]
+; AVX512-NEXT:    vmovdqa32 %zmm12, %zmm1 {%k2}
+; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm15[2,1,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,2,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm12[0],ymm3[1,2],ymm12[3],ymm3[4,5],ymm12[6],ymm3[7]
+; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm12 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero
+; AVX512-NEXT:    vpbroadcastq %xmm12, %ymm12
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm12[2],ymm1[3,4],ymm12[5],ymm1[6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm15
+; AVX512-NEXT:    vmovdqa (%r9), %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,1,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm23 = ymm1[0,1,0,1]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm11[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm29 = ymm1[2,2,2,2]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm11 = xmm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm12 = xmm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
+; AVX512-NEXT:    vpermt2d %zmm11, %zmm31, %zmm1
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm11 = xmm0[0,1,2,1]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,7,6,5]
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm2[0,1,2,1]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,7,6,5]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm8[0,2,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,4,4,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,3,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,2,2,1,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
-; AVX512-NEXT:    vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm13 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,2,2,2]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-NEXT:    vmovdqa 32(%r9), %xmm15
-; AVX512-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,2,1]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm14[0,1,2,3],zmm1[0,1,0,1]
-; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k2}
-; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm3[2,1,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,2,1]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0],ymm0[1,2],ymm14[3],ymm0[4,5],ymm14[6],ymm0[7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm14 = xmm15[2,3,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm14[0,2,2,1,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,1,0,1]
-; AVX512-NEXT:    vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm2 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm26 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
-; AVX512-NEXT:    vpbroadcastq %xmm26, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm15[0,0,2,1,4,5,6,7]
-; AVX512-NEXT:    vpbroadcastq %xmm3, %ymm3
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[0,2,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,4,4,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,0,2,1]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm11[0,0,2,1,4,5,6,7]
-; AVX512-NEXT:    vpbroadcastq %xmm1, %ymm1
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm11[0,2,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,4,4,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1]
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm12 # 64-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm16, %zmm20, %zmm4
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm4 # 64-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm5, %zmm5
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm5 # 64-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm17, %zmm24, %zmm16
-; AVX512-NEXT:    vpternlogd $184, %zmm27, %zmm10, %zmm16
-; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm8, %zmm8
-; AVX512-NEXT:    vpternlogd $184, %zmm25, %zmm10, %zmm8
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm14, %zmm2
-; AVX512-NEXT:    vpternlogd $184, %zmm22, %zmm10, %zmm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm19, %zmm18, %zmm10
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogd $184, %zmm28, %zmm13, %zmm10
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm7, %zmm7
-; AVX512-NEXT:    vpternlogd $184, %zmm30, %zmm13, %zmm7
-; AVX512-NEXT:    vinserti64x4 $1, %ymm15, %zmm3, %zmm3
-; AVX512-NEXT:    vpternlogd $184, %zmm21, %zmm13, %zmm3
-; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm1, %zmm1
-; AVX512-NEXT:    vpternlogd $184, %zmm0, %zmm13, %zmm1
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm2[0,1,2,1]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,7,6,5]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,2,1]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm11 = zmm12[0,1,2,3],zmm11[0,1,0,1]
+; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm11 {%k2}
+; AVX512-NEXT:    vextracti64x4 $1, %zmm11, %ymm1
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm8[2,1,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,2,1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0],ymm1[1,2],ymm12[3],ymm1[4,5],ymm12[6],ymm1[7]
+; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm8 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero
+; AVX512-NEXT:    vpbroadcastq %xmm8, %ymm8
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm11[0,1],ymm8[2],ymm11[3,4],ymm8[5],ymm11[6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm8, %zmm1
+; AVX512-NEXT:    vmovdqa 96(%r9), %xmm8
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm11 = xmm8[2,3,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm11[0,2,2,1,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,1,0,1]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm7[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm8[0,0,2,1,4,5,6,7]
+; AVX512-NEXT:    vpbroadcastq %xmm12, %ymm12
+; AVX512-NEXT:    vmovdqa 64(%r9), %xmm13
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,2,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,4,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,2,1]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm13[0,0,2,1,4,5,6,7]
+; AVX512-NEXT:    vpbroadcastq %xmm14, %ymm14
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm13[0,2,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,4,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm25, %zmm24, %zmm24
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
+; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm24 # 64-byte Folded Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm27, %zmm26, %zmm26
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,3,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm13[0,2,2,1,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1]
+; AVX512-NEXT:    vmovdqa 32(%r9), %xmm0
+; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm2[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,2]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm10 = xmm0[2,3,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[0,2,2,1,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,0,1]
+; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm26 # 64-byte Folded Reload
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm5[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,2]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm0[0,0,2,1,4,5,6,7]
+; AVX512-NEXT:    vpbroadcastq %xmm9, %ymm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm17, %zmm30, %zmm17
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm3[0,0,2,1,4,5,6,7]
+; AVX512-NEXT:    vpbroadcastq %xmm2, %ymm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
+; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm17 # 64-byte Folded Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm16, %zmm18, %zmm16
+; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm16 # 64-byte Folded Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm29, %zmm23, %zmm18
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm23, %zmm18 # 64-byte Folded Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm11, %zmm4
+; AVX512-NEXT:    vpternlogd $184, (%rsp), %zmm23, %zmm4 # 64-byte Folded Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
+; AVX512-NEXT:    vpternlogd $184, %zmm20, %zmm23, %zmm7
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm10, %zmm5
+; AVX512-NEXT:    vpternlogd $184, %zmm19, %zmm23, %zmm5
+; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm12, %zmm8
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512-NEXT:    vpternlogd $184, %zmm22, %zmm10, %zmm8
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm14, %zmm6
+; AVX512-NEXT:    vpternlogd $184, %zmm28, %zmm10, %zmm6
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm9, %zmm0
+; AVX512-NEXT:    vpternlogd $184, %zmm15, %zmm10, %zmm0
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm10, %zmm2
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vmovdqa64 %zmm1, (%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm3, 192(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm2, 256(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm8, 448(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm7, 384(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm10, 576(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm16, 640(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm5, 64(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm4, 128(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm12, 320(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm23, 512(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm6, 704(%rax)
-; AVX512-NEXT:    addq $600, %rsp # imm = 0x258
+; AVX512-NEXT:    vmovdqa64 %zmm2, (%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm0, 192(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm5, 256(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm7, 448(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm6, 384(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm8, 576(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm4, 640(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm18, 64(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm16, 128(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm17, 320(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm26, 512(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm24, 704(%rax)
+; AVX512-NEXT:    addq $392, %rsp # imm = 0x188
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i16_stride6_vf64:
 ; AVX512-FCP:       # %bb.0:
-; AVX512-FCP-NEXT:    subq $1256, %rsp # imm = 0x4E8
-; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %ymm3
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %ymm4
-; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %ymm2
+; AVX512-FCP-NEXT:    subq $1240, %rsp # imm = 0x4D8
+; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %ymm1
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %ymm2
+; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %ymm6
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %ymm5
 ; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %xmm9
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %xmm11
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm8
-; AVX512-FCP-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm12
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %xmm10
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm7
+; AVX512-FCP-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %xmm13
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm8
+; AVX512-FCP-NEXT:    vmovdqa %xmm8, (%rsp) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm12
 ; AVX512-FCP-NEXT:    vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %xmm6
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm7
-; AVX512-FCP-NEXT:    vmovdqa %xmm7, (%rsp) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm13
-; AVX512-FCP-NEXT:    vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %xmm10
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm1
-; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm0
-; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm5[0],ymm2[0],ymm5[1],ymm2[1],ymm5[2],ymm2[2],ymm5[3],ymm2[3],ymm5[8],ymm2[8],ymm5[9],ymm2[9],ymm5[10],ymm2[10],ymm5[11],ymm2[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm10, %xmm19
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm6, %xmm23
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm7
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm10
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm10[0],ymm7[0],ymm10[1],ymm7[1],ymm10[2],ymm7[2],ymm10[3],ymm7[3],ymm10[8],ymm7[8],ymm10[9],ymm7[9],ymm10[10],ymm7[10],ymm10[11],ymm7[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %xmm11
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm4
+; AVX512-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm3
+; AVX512-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm11[4],xmm13[4],xmm11[5],xmm13[5],xmm11[6],xmm13[6],xmm11[7],xmm13[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm13, %xmm18
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm3
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm7
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm7[0],ymm3[0],ymm7[1],ymm3[1],ymm7[2],ymm3[2],ymm7[3],ymm3[3],ymm7[8],ymm3[8],ymm7[9],ymm3[9],ymm7[10],ymm3[10],ymm7[11],ymm3[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm8, %zmm4
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %ymm12
 ; AVX512-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm12, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %ymm1
-; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm8
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm8[0],ymm0[0],ymm8[1],ymm0[1],ymm8[2],ymm0[2],ymm8[3],ymm0[3],ymm8[8],ymm0[8],ymm8[9],ymm0[9],ymm8[10],ymm0[10],ymm8[11],ymm0[11]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm4
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %ymm0
+; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm8
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm8[0],ymm4[0],ymm8[1],ymm4[1],ymm8[2],ymm4[2],ymm8[3],ymm4[3],ymm8[8],ymm4[8],ymm8[9],ymm4[9],ymm8[10],ymm4[10],ymm8[11],ymm4[11]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [2,1,2,3,11,11,11,11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm8 = ymm1[4],ymm12[4],ymm1[5],ymm12[5],ymm1[6],ymm12[6],ymm1[7],ymm12[7],ymm1[12],ymm12[12],ymm1[13],ymm12[13],ymm1[14],ymm12[14],ymm1[15],ymm12[15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm8, %zmm27, %zmm0
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm26 = [5,6,5,6,5,6,7,7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm8 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm26, %ymm8
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm4 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm3, %zmm3
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm0[4],ymm12[4],ymm0[5],ymm12[5],ymm0[6],ymm12[6],ymm0[7],ymm12[7],ymm0[12],ymm12[12],ymm0[13],ymm12[13],ymm0[14],ymm12[14],ymm0[15],ymm12[15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm27, %zmm8
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [5,6,5,6,5,6,7,7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm12 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]
+; AVX512-FCP-NEXT:    vpermd %ymm12, %ymm4, %ymm12
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    movw $18724, %ax # imm = 0x4924
 ; AVX512-FCP-NEXT:    kmovw %eax, %k1
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm0 {%k1}
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm25 = [8,21,10,11,20,13,14,23]
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %ymm1
-; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
-; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm1, %ymm4
-; AVX512-FCP-NEXT:    vmovdqa %ymm13, %ymm8
-; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm25, %zmm3
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm29 = [12,1,2,13,4,5,14,7]
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm1[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpermt2d %ymm4, %ymm29, %ymm0
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm8 {%k1}
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm25 = [12,1,2,13,4,5,14,7]
+; AVX512-FCP-NEXT:    vmovdqa %ymm8, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %ymm2
+; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermt2d %ymm1, %ymm25, %ymm0
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm28 = [8,21,10,11,20,13,14,23]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
+; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa %ymm14, %ymm12
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm28, %zmm8
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [2,2,0,3,10,0,10,11]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm29 = [2,2,0,3,10,0,10,11]
 ; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %ymm1
 ; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm15 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]
 ; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm1, %ymm0
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]
 ; AVX512-FCP-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm1, %ymm3
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm28, %zmm3
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %ymm1
-; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %ymm4
-; AVX512-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm4, %ymm3
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm1[4],ymm4[5],ymm1[5],ymm4[6],ymm1[6],ymm4[7],ymm1[7],ymm4[12],ymm1[12],ymm4[13],ymm1[13],ymm4[14],ymm1[14],ymm4[15],ymm1[15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm27, %zmm0
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm2[4],ymm5[5],ymm2[5],ymm5[6],ymm2[6],ymm5[7],ymm2[7],ymm5[12],ymm2[12],ymm5[13],ymm2[13],ymm5[14],ymm2[14],ymm5[15],ymm2[15]
-; AVX512-FCP-NEXT:    vpermd %ymm3, %ymm26, %ymm3
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm4 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm29, %zmm1
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %ymm2
+; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm2, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %ymm8
+; AVX512-FCP-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm8, %ymm1
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm8[4],ymm2[4],ymm8[5],ymm2[5],ymm8[6],ymm2[6],ymm8[7],ymm2[7],ymm8[12],ymm2[12],ymm8[13],ymm2[13],ymm8[14],ymm2[14],ymm8[15],ymm2[15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm27, %zmm0
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]
+; AVX512-FCP-NEXT:    vpermd %ymm1, %ymm4, %ymm1
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm6 = ymm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm6[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm5 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm0 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %ymm1
-; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm1, %ymm4
-; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm25, %zmm3
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm1[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpermt2d %ymm4, %ymm29, %ymm0
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,2]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %ymm2
+; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermt2d %ymm5, %ymm25, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm2, %ymm5
+; AVX512-FCP-NEXT:    vpermt2d %zmm5, %zmm28, %zmm0
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %ymm1
 ; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm1, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm1, %ymm3
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm28, %zmm3
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm0, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm1, %xmm4
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [0,0,2,1,8,9,8,9]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm18, %zmm4
-; AVX512-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm31 = [1,0,2,2,1,0,2,2]
-; AVX512-FCP-NEXT:    # ymm31 = mem[0,1,2,3,0,1,2,3]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512-FCP-NEXT:    vpermd %ymm3, %ymm31, %ymm3
+; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm29, %zmm1
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm2
+; AVX512-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm2, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm1
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [0,0,2,1,8,9,8,9]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm16, %zmm1
+; AVX512-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm30 = [1,0,2,2,1,0,2,2]
+; AVX512-FCP-NEXT:    # ymm30 = mem[0,1,2,3,0,1,2,3]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-FCP-NEXT:    vpermd %ymm0, %ymm30, %ymm0
 ; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
 ; AVX512-FCP-NEXT:    vpbroadcastq %xmm2, %ymm2
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    movw $9362, %ax # imm = 0x2492
 ; AVX512-FCP-NEXT:    kmovw %eax, %k2
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm2, %zmm4 {%k2}
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm30 = [16,9,10,17,12,13,18,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm0[2,1,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm30, %zmm2
-; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm16 = [0,1,8,3,4,9,6,7]
-; AVX512-FCP-NEXT:    vpermt2d %ymm3, %ymm16, %ymm4
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm2[0,1,2,3]
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k2}
+; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [16,9,10,17,12,13,18,15]
+; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm5[2,1,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm1
+; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm26 = [0,1,8,3,4,9,6,7]
+; AVX512-FCP-NEXT:    vpermt2d %ymm2, %ymm26, %ymm0
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm17 = [0,0,0,0,8,8,0,9]
-; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm0, %xmm2
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm17, %zmm0
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm0
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm0, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm1
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm31 = [0,0,0,0,8,8,0,9]
+; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm1, %xmm3
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm20
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm18, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm23, %xmm0
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm1
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-FCP-NEXT:    vpermd %ymm2, %ymm31, %ymm2
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512-FCP-NEXT:    vpbroadcastq %xmm4, %ymm4
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm2, %zmm3 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %xmm0
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm0[2,1,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm30, %zmm2
-; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm24
-; AVX512-FCP-NEXT:    vpermt2d %ymm4, %ymm16, %ymm3
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm3[0,1,2,3],zmm2[0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm0, %xmm2
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm17, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm6
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,1,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm31, %zmm1
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm2
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm2, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm5
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm1
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm5, %xmm23
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm24
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm16, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm18, %xmm2
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm11[0],xmm2[0],xmm11[1],xmm2[1],xmm11[2],xmm2[2],xmm11[3],xmm2[3]
+; AVX512-FCP-NEXT:    vpermd %ymm0, %ymm30, %ymm0
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512-FCP-NEXT:    vpbroadcastq %xmm2, %ymm2
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k2}
+; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %xmm5
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm5[2,1,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm1
+; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm5, %xmm19
+; AVX512-FCP-NEXT:    vpermt2d %ymm2, %ymm26, %ymm0
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %xmm1
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm1, %xmm0
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,1,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm31, %zmm1
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm2
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm2, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm5, %ymm1
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm5[4],ymm2[4],ymm5[5],ymm2[5],ymm5[6],ymm2[6],ymm5[7],ymm2[7],ymm5[12],ymm2[12],ymm5[13],ymm2[13],ymm5[14],ymm2[14],ymm5[15],ymm2[15]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm5, %ymm21
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm20
+; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm27, %zmm0
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm7[4],ymm3[4],ymm7[5],ymm3[5],ymm7[6],ymm3[6],ymm7[7],ymm3[7],ymm7[12],ymm3[12],ymm7[13],ymm3[13],ymm7[14],ymm3[14],ymm7[15],ymm3[15]
+; AVX512-FCP-NEXT:    vpermd %ymm1, %ymm4, %ymm1
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm22
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm23
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm27, %zmm2
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm10[4],ymm7[4],ymm10[5],ymm7[5],ymm10[6],ymm7[6],ymm10[7],ymm7[7],ymm10[12],ymm7[12],ymm10[13],ymm7[13],ymm10[14],ymm7[14],ymm10[15],ymm7[15]
-; AVX512-FCP-NEXT:    vpermd %ymm3, %ymm26, %ymm3
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm4 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm5 = ymm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm10[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm2 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm0
-; AVX512-FCP-NEXT:    vmovdqa %ymm8, %ymm5
-; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm0, %ymm4
-; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm25, %zmm3
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm0[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm19
-; AVX512-FCP-NEXT:    vpermt2d %ymm4, %ymm29, %ymm2
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,2,3],zmm3[0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm9
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm9, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm8
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm8, %ymm1
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm8[4],ymm9[4],ymm8[5],ymm9[5],ymm8[6],ymm9[6],ymm8[7],ymm9[7],ymm8[12],ymm9[12],ymm8[13],ymm9[13],ymm8[14],ymm9[14],ymm8[15],ymm9[15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm27, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm2
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm2, %ymm3
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm2, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm21
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm28, %zmm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm3
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermt2d %ymm2, %ymm25, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa %ymm12, %ymm5
+; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm18
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm28, %zmm0
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm12
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm11
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm11, %ymm1
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm11[4],ymm12[4],ymm11[5],ymm12[5],ymm11[6],ymm12[6],ymm11[7],ymm12[7],ymm11[12],ymm12[12],ymm11[13],ymm12[13],ymm11[14],ymm12[14],ymm11[15],ymm12[15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm27, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
+; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm3, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm3, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm22
+; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm29, %zmm2
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
-; AVX512-FCP-NEXT:    vpermd %ymm3, %ymm26, %ymm3
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm4 = ymm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm0[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm6 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm6[0],ymm4[0],ymm6[1],ymm4[1],ymm6[2],ymm4[2],ymm6[3],ymm4[3],ymm6[8],ymm4[8],ymm6[9],ymm4[9],ymm6[10],ymm4[10],ymm6[11],ymm4[11]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm1 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm3
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm3, %ymm4
-; AVX512-FCP-NEXT:    vpermi2d %zmm4, %zmm1, %zmm25
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpermt2d %ymm4, %ymm29, %ymm1
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm25 = zmm1[0,1,2,3],zmm25[0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm5
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm5, %ymm1
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm5, %ymm10
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm28, %zmm10
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm11
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm0, %xmm1
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm11, %xmm6
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm11[0],xmm0[0],xmm11[1],xmm0[1],xmm11[2],xmm0[2],xmm11[3],xmm0[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm29
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm18, %zmm7
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15]
+; AVX512-FCP-NEXT:    vpermd %ymm1, %ymm4, %ymm1
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} ymm4 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm10
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm10[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermi2d %ymm1, %ymm0, %ymm25
+; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm10, %ymm1
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm28, %zmm0
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm25, %zmm25
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm7, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm7, %ymm13
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm29, %zmm13
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm15
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm14
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm15, %xmm0
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm14, %xmm1
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm16, %zmm4
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVX512-FCP-NEXT:    vpermd %ymm1, %ymm31, %ymm1
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm6 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm15 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm15[0],xmm6[0],xmm15[1],xmm6[1],xmm15[2],xmm6[2],xmm15[3],xmm6[3]
-; AVX512-FCP-NEXT:    vpbroadcastq %xmm6, %ymm6
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm1, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm1, %zmm7 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm6
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm6[2,1,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2d %zmm15, %zmm30, %zmm1
-; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm15 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
-; AVX512-FCP-NEXT:    vpermt2d %ymm15, %ymm16, %ymm7
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm26 = zmm7[0,1,2,3],zmm1[0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm15
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm15, %xmm1
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm0, %xmm7
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm18, %zmm12
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm7
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm7, %xmm1
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm7[0,0,2,1,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm17, %zmm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX512-FCP-NEXT:    vpermd %ymm0, %ymm30, %ymm5
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm3
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm6, %xmm9
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm3, %xmm8
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
+; AVX512-FCP-NEXT:    vpermt2q %zmm8, %zmm16, %zmm0
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm8 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm9 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512-FCP-NEXT:    vpbroadcastq %xmm8, %ymm8
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm5, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm5, %zmm4 {%k2}
+; AVX512-FCP-NEXT:    vmovdqa %ymm4, %ymm5
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm8
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm8[2,1,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm9, %zmm17, %zmm4
+; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm9 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero
+; AVX512-FCP-NEXT:    vpermt2d %ymm9, %ymm26, %ymm5
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm16
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqa (%rsp), %xmm2 # 16-byte Reload
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm14[0],xmm2[1],xmm14[1],xmm2[2],xmm14[2],xmm2[3],xmm14[3]
-; AVX512-FCP-NEXT:    vpermd %ymm1, %ymm31, %ymm18
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm14 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm14[0],xmm1[0],xmm14[1],xmm1[1],xmm14[2],xmm1[2],xmm14[3],xmm1[3]
-; AVX512-FCP-NEXT:    vpbroadcastq %xmm1, %ymm1
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm18, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm1, %zmm12 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm14
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm14[2,1,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermi2d %zmm1, %zmm12, %zmm30
-; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm14[0],zero,xmm14[1],zero,xmm14[2],zero,xmm14[3],zero
-; AVX512-FCP-NEXT:    vpermt2d %ymm1, %ymm16, %ymm12
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm16 = zmm12[0,1,2,3],zmm30[0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm1, %xmm2
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm1[0,0,2,1,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm17, %zmm12
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[8],ymm9[8],ymm8[9],ymm9[9],ymm8[10],ymm9[10],ymm8[11],ymm9[11]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm9 = [1,1,1,1,10,10,10,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm9, %zmm0
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [1,2,1,2,0,0,3,3,13,12,10,10,13,12,14,14]
-; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm15, %zmm2 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm3[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm18 = [8,9,20,11,12,21,14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, %zmm17
-; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm18, %zmm17
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm14, %xmm0
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm14 = [0,9,2,3,8,5,6,11]
-; AVX512-FCP-NEXT:    vpermt2d %ymm0, %ymm14, %ymm2
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm5[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm1, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm30
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [0,0,0,1,0,10,10,0]
-; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm27, %zmm3
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX512-FCP-NEXT:    vpermd %ymm4, %ymm30, %ymm4
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm9 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
+; AVX512-FCP-NEXT:    vpbroadcastq %xmm5, %ymm5
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm4, %zmm0 {%k2}
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm1
+; AVX512-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX512-FCP-NEXT:    vpermi2d %ymm4, %ymm0, %ymm26
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm1[2,1,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm17, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm9
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm9, %xmm4
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm9[0,0,2,1,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm31, %zmm5
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm26, %zmm17
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm0
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm2
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm0[0,0,2,1,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm31, %zmm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm11[0],ymm12[0],ymm11[1],ymm12[1],ymm11[2],ymm12[2],ymm11[3],ymm12[3],ymm11[8],ymm12[8],ymm11[9],ymm12[9],ymm11[10],ymm12[10],ymm11[11],ymm12[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm12 = [1,1,1,1,10,10,10,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm12, %zmm3
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm26 = [1,2,1,2,0,0,3,3,13,12,10,10,13,12,14,14]
+; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm2 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm2 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm1, %xmm3
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm27 = [0,9,2,3,8,5,6,11]
+; AVX512-FCP-NEXT:    vmovdqa %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vpermt2d %ymm3, %ymm27, %ymm1
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm10[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm30 = [8,9,20,11,12,21,14,15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm30, %zmm2
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm7[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm0, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm6, %xmm31
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,0,0,1,0,10,10,0]
+; AVX512-FCP-NEXT:    vpermt2q %zmm7, %zmm28, %zmm3
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    # ymm5 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11]
+; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    # ymm7 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11]
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm9, %zmm1
-; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm15, %zmm5 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm1, %zmm5 {%k1}
-; AVX512-FCP-NEXT:    vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    # ymm1 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm28
-; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm28
+; AVX512-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-FCP-NEXT:    # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm7, %zmm12, %zmm0
+; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm7 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm7 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm0, %xmm1
-; AVX512-FCP-NEXT:    vpermt2d %ymm1, %ymm14, %ymm5
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    # ymm1 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm20, %xmm13
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm9, %zmm0
-; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm15, %zmm1 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm7, %ymm29
+; AVX512-FCP-NEXT:    vpermt2d %ymm0, %ymm27, %ymm29
 ; AVX512-FCP-NEXT:    vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    # ymm0 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm20
-; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm18, %zmm20
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm0
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vpermt2d %ymm0, %ymm14, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm22, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm23, %ymm13
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm13[0],ymm0[1],ymm13[1],ymm0[2],ymm13[2],ymm0[3],ymm13[3],ymm0[8],ymm13[8],ymm0[9],ymm13[9],ymm0[10],ymm13[10],ymm0[11],ymm13[11]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm29, %xmm13
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm11[4],xmm13[4],xmm11[5],xmm13[5],xmm11[6],xmm13[6],xmm11[7],xmm13[7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm9, %zmm11
-; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm15, %zmm0 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    vmovdqa32 %zmm11, %zmm0 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm9
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm9[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpermi2d %zmm9, %zmm0, %zmm18
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm6, %xmm6
-; AVX512-FCP-NEXT:    vpermt2d %ymm6, %ymm14, %ymm0
-; AVX512-FCP-NEXT:    vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    # ymm6 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm30, %xmm11
+; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm30, %zmm7
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm23, %xmm10
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm6
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm12, %zmm10
+; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm0 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm10, %zmm0 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm6
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm6, %xmm10
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm19
+; AVX512-FCP-NEXT:    vpermt2d %ymm10, %ymm27, %ymm19
+; AVX512-FCP-NEXT:    vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    # ymm10 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm10, %zmm30, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm21, %ymm6
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm10
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm6[0],ymm10[0],ymm6[1],ymm10[1],ymm6[2],ymm10[2],ymm6[3],ymm10[3],ymm6[8],ymm10[8],ymm6[9],ymm10[9],ymm6[10],ymm10[10],ymm6[11],ymm10[11]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm14[4],xmm15[4],xmm14[5],xmm15[5],xmm14[6],xmm15[6],xmm14[7],xmm15[7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm10, %zmm12, %zmm14
+; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm10 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vmovdqa32 %zmm14, %zmm10 {%k1}
 ; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm8, %xmm8
-; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm27, %zmm8
-; AVX512-FCP-NEXT:    vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    # ymm6 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm9, %xmm9
-; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm27, %zmm9
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm7, %xmm6
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm21, %ymm7
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm7[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm7, %zmm27, %zmm6
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm18[0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm7, %zmm6
+; AVX512-FCP-NEXT:    vpermi2d %ymm8, %ymm10, %ymm27
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm18, %ymm6
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm6[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm8, %zmm30, %zmm10
+; AVX512-FCP-NEXT:    vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    # ymm8 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm31, %xmm14
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm11, %xmm11
+; AVX512-FCP-NEXT:    vpermt2q %zmm8, %zmm28, %zmm11
+; AVX512-FCP-NEXT:    vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    # ymm8 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm6, %xmm12
+; AVX512-FCP-NEXT:    vpermt2q %zmm8, %zmm28, %zmm12
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm9, %xmm6
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm22, %ymm8
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm8[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm8, %zmm28, %zmm6
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm27, %zmm8
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm8, %zmm9, %zmm6
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 256(%rax)
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm20[0,1,2,3]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm7, %zmm9
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm9, 448(%rax)
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm5[0,1,2,3],zmm28[0,1,2,3]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm7, %zmm8
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, 640(%rax)
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,2,3],zmm17[0,1,2,3]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm7, %zmm3
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm19, %zmm0
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm9, %zmm12
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, 448(%rax)
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm29, %zmm0
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm9, %zmm11
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, 640(%rax)
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm0
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm9, %zmm3
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 64(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm16, %zmm0, %zmm12
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, (%rax)
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm26, %zmm0, %zmm4
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 192(%rax)
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm17, %zmm0, %zmm4
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm16, %zmm0, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 192(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm25, %zmm1, %zmm10
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, 128(%rax)
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm25, %zmm1, %zmm13
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, 128(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 320(%rax)
@@ -9244,15 +9201,15 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 704(%rax)
-; AVX512-FCP-NEXT:    addq $1256, %rsp # imm = 0x4E8
+; AVX512-FCP-NEXT:    addq $1240, %rsp # imm = 0x4D8
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i16_stride6_vf64:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    subq $936, %rsp # imm = 0x3A8
+; AVX512DQ-NEXT:    subq $584, %rsp # imm = 0x248
 ; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %xmm0
-; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %xmm2
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
@@ -9262,7 +9219,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]
 ; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm0, %zmm1
 ; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm4, (%rsp) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]
 ; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm5
@@ -9282,22 +9239,12 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm4[2,1,3,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0],ymm1[1,2],ymm3[3],ymm1[4,5],ymm3[6],ymm1[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
 ; AVX512DQ-NEXT:    vpbroadcastq %xmm3, %ymm3
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm1[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 96(%r9), %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm2[0,0,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpbroadcastq %xmm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm2[0,2,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm4
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm5
 ; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %xmm2
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm6
@@ -9308,57 +9255,47 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
 ; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm0, %zmm1
-; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %xmm5
-; AVX512DQ-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm5[0,1,2,1]
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm4
+; AVX512DQ-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %xmm7
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm7[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm7
-; AVX512DQ-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm7[0,1,2,1]
+; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm8
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm8[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm29
+; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm30
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,0,1]
 ; AVX512DQ-NEXT:    vmovdqa32 %zmm1, %zmm2 {%k1}
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqa 64(%r8), %xmm5
-; AVX512DQ-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm5[2,1,3,3,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 64(%r8), %xmm7
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm7[2,1,3,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0],ymm1[1,2],ymm3[3],ymm1[4,5],ymm3[6],ymm1[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero
+; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm31
 ; AVX512DQ-NEXT:    vpbroadcastq %xmm3, %ymm3
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm1[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm7
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 64(%r9), %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm2[0,0,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpbroadcastq %xmm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm2[0,2,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
 ; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm0, %zmm2
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm5
-; AVX512DQ-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm5[0,1,2,1]
+; AVX512DQ-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm7[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,0,1]
 ; AVX512DQ-NEXT:    vmovdqa32 %zmm2, %zmm1 {%k1}
@@ -9368,39 +9305,30 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm3[1,2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
 ; AVX512DQ-NEXT:    vpbroadcastq %xmm3, %ymm3
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm2[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm2[0,0,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm2[0,2,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm3
 ; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm4
 ; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm31
-; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm19
+; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm27
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm28
 ; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm0, %zmm2
 ; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm3
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm3[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm15
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm15[0,1,2,1]
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[0,1,2,1]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm15[0],xmm3[0],xmm15[1],xmm3[1],xmm15[2],xmm3[2],xmm15[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm18
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm20
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm21
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,0,1]
 ; AVX512DQ-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k1}
@@ -9409,12 +9337,11 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
-; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm26
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm18
 ; AVX512DQ-NEXT:    vpbroadcastq %xmm2, %ymm2
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %ymm2
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm2[2,1,2,3,6,5,6,7]
@@ -9425,34 +9352,32 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm28
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm27
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm22
+; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm26
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[3,3,3,3]
-; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %ymm4
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %ymm11
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm11[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %ymm12
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm12[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %ymm9
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm9[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm11[4],ymm4[4],ymm11[5],ymm4[5],ymm11[6],ymm4[6],ymm11[7],ymm4[7],ymm11[12],ymm4[12],ymm11[13],ymm4[13],ymm11[14],ymm4[14],ymm11[15],ymm4[15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm4, %ymm21
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm12[4],ymm9[5],ymm12[5],ymm9[6],ymm12[6],ymm9[7],ymm12[7],ymm9[12],ymm12[12],ymm9[13],ymm12[13],ymm9[14],ymm12[14],ymm9[15],ymm12[15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,2,3,3,5,6,7,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
 ; AVX512DQ-NEXT:    movw $18724, %ax # imm = 0x4924
 ; AVX512DQ-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm3, %zmm2, %zmm0 {%k1}
-; AVX512DQ-NEXT:    vmovdqa 96(%r8), %ymm9
+; AVX512DQ-NEXT:    vmovdqa 96(%r8), %ymm6
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm9, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm6, %ymm1
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm9[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %ymm2
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm2[2,1,2,3,6,5,6,7]
@@ -9463,50 +9388,49 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm22
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm24
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm16
+; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm17
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[3,3,3,3]
-; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %ymm12
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm12[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %ymm10
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm10[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %ymm7
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %ymm5
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm10[4],ymm12[4],ymm10[5],ymm12[5],ymm10[6],ymm12[6],ymm10[7],ymm12[7],ymm10[12],ymm12[12],ymm10[13],ymm12[13],ymm10[14],ymm12[14],ymm10[15],ymm12[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm7[4],ymm5[5],ymm7[5],ymm5[6],ymm7[6],ymm5[7],ymm7[7],ymm5[12],ymm7[12],ymm5[13],ymm7[13],ymm5[14],ymm7[14],ymm5[15],ymm7[15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,2,3,3,5,6,7,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm3, %zmm2, %zmm0 {%k1}
-; AVX512DQ-NEXT:    vmovdqa 64(%r8), %ymm3
-; AVX512DQ-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm3, %ymm1
+; AVX512DQ-NEXT:    vmovdqa 64(%r8), %ymm8
+; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm8, %ymm1
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm8[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm2
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm2[2,1,2,3,6,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm14
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm14[2,1,2,3,6,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm3
+; AVX512DQ-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[2,1,2,3,6,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm14[4],ymm2[4],ymm14[5],ymm2[5],ymm14[6],ymm2[6],ymm14[7],ymm2[7],ymm14[12],ymm2[12],ymm14[13],ymm2[13],ymm14[14],ymm2[14],ymm14[15],ymm2[15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm20
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[3,3,3,3]
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm7
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm6
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm6[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm11
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm11[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm10
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm10[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm6[4],ymm7[4],ymm6[5],ymm7[5],ymm6[6],ymm7[6],ymm6[7],ymm7[7],ymm6[12],ymm7[12],ymm6[13],ymm7[13],ymm6[14],ymm7[14],ymm6[15],ymm7[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm10[4],ymm11[4],ymm10[5],ymm11[5],ymm10[6],ymm11[6],ymm10[7],ymm11[7],ymm10[12],ymm11[12],ymm10[13],ymm11[13],ymm10[14],ymm11[14],ymm10[15],ymm11[15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,2,3,3,5,6,7,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
@@ -9516,252 +9440,257 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm25
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm19
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm5
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm5[2,1,2,3,6,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm4
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm4[2,1,2,3,6,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm3
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[2,1,2,3,6,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm17 = ymm0[2,1,2,3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[12],ymm5[12],ymm3[13],ymm5[13],ymm3[14],ymm5[14],ymm3[15],ymm5[15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm16 = ymm0[3,3,3,3]
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm4
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm2
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm8 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm8[0],ymm1[0],ymm8[1],ymm1[1],ymm8[2],ymm1[2],ymm8[3],ymm1[3],ymm8[8],ymm1[8],ymm8[9],ymm1[9],ymm8[10],ymm1[10],ymm8[11],ymm1[11]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm24 = ymm0[2,1,2,3]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm23 = ymm0[3,3,3,3]
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm2
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm0
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm15 = ymm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm0[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm15[0],ymm1[0],ymm15[1],ymm1[1],ymm15[2],ymm1[2],ymm15[3],ymm1[3],ymm15[8],ymm1[8],ymm15[9],ymm1[9],ymm15[10],ymm1[10],ymm15[11],ymm1[11]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm25 = ymm1[2,2,2,2]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm15 = ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm15 = ymm15[1,2,3,3,5,6,7,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm23, %zmm24, %zmm1
+; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm15, %zmm25, %zmm1 {%k1}
+; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm15
+; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm15, %ymm14
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm13
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0],ymm14[1],ymm13[2,3],ymm14[4],ymm13[5,6],ymm14[7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm14 = ymm15[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,2,2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm14[0],ymm1[1,2],ymm14[3],ymm1[4,5],ymm14[6],ymm1[7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm1, %zmm23
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm13
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm13[4],xmm1[5],xmm13[5],xmm1[6],xmm13[6],xmm1[7],xmm13[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm24 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm24, %zmm0
+; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm2
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,1,1,1]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm2, %zmm1, %zmm0 {%k1}
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm15[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm8 = ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm8 = ymm8[1,2,3,3,5,6,7,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm16, %zmm17, %zmm0
-; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm8, %zmm1, %zmm0 {%k1}
-; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm1
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm1, %ymm8
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,1,2,3]
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm13
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0],ymm8[1],ymm13[2,3],ymm8[4],ymm13[5,6],ymm8[7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm8
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm13 = ymm1[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,2,2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0],ymm0[1,2],ymm13[3],ymm0[4,5],ymm13[6],ymm0[7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm29 = zmm0[0,1,2,3],zmm8[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm8
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm8[0,0,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpbroadcastq %xmm0, %ymm30
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm0
-; AVX512DQ-NEXT:    vmovdqa64 %xmm19, %xmm13
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm8[0,2,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,4,4,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm31 = ymm13[0,0,2,1]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]
-; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm4, %zmm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm0
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
-; AVX512DQ-NEXT:    vmovdqa 96(%r9), %ymm15
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm13 = ymm15[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm13 = ymm13[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm23 = ymm13[2,2,2,3]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm15[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm15, %ymm16
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm5[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm18 = ymm5[2,1,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm3, %zmm0, %zmm2 {%k1}
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm1[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm2, %ymm1
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
-; AVX512DQ-NEXT:    vmovdqa 64(%r9), %ymm0
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm15
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm17 = ymm5[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm3
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm19 = zmm2[0,1,2,3],zmm3[4,5,6,7]
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm2
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vmovdqa %xmm3, %xmm4
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm25
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm21, %ymm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm11[0],ymm0[0],ymm11[1],ymm0[1],ymm11[2],ymm0[2],ymm11[3],ymm0[3],ymm11[8],ymm0[8],ymm11[9],ymm0[9],ymm11[10],ymm0[10],ymm11[11],ymm0[11]
-; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm4, %zmm11
+; AVX512DQ-NEXT:    vpunpckhwd (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm9[0],ymm12[0],ymm9[1],ymm12[1],ymm9[2],ymm12[2],ymm9[3],ymm12[3],ymm9[8],ymm12[8],ymm9[9],ymm12[9],ymm9[10],ymm12[10],ymm9[11],ymm12[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm24, %zmm1
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpunpckhwd (%rsp), %xmm0, %xmm2 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm22, %ymm2
+; AVX512DQ-NEXT:    vmovdqa64 %ymm26, %ymm3
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm2, %zmm0, %zmm1 {%k1}
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm6[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vmovdqa %xmm4, %xmm9
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm26
+; AVX512DQ-NEXT:    vmovdqa 96(%r9), %xmm6
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm6[0,0,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpbroadcastq %xmm0, %ymm0
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm6[0,2,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm5[0],ymm7[0],ymm5[1],ymm7[1],ymm5[2],ymm7[2],ymm5[3],ymm7[3],ymm5[8],ymm7[8],ymm5[9],ymm7[9],ymm5[10],ymm7[10],ymm5[11],ymm7[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm24, %zmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm2
+; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm4
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,1,1]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm28, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm27, %ymm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm3, %zmm2, %zmm11 {%k1}
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm9[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm16, %ymm4
+; AVX512DQ-NEXT:    vmovdqa64 %ymm17, %ymm5
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm16
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm16 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm4[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm1, %zmm2, %zmm3 {%k1}
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm3, %ymm1
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm8[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm11, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm13 = ymm10[0],ymm12[0],ymm10[1],ymm12[1],ymm10[2],ymm12[2],ymm10[3],ymm12[3],ymm10[8],ymm12[8],ymm10[9],ymm12[9],ymm10[10],ymm12[10],ymm10[11],ymm12[11]
-; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm4, %zmm13
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,1,1]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm22, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm24, %ymm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm10 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm10 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm6[0],ymm7[0],ymm6[1],ymm7[1],ymm6[2],ymm7[2],ymm6[3],ymm7[3],ymm6[8],ymm7[8],ymm6[9],ymm7[9],ymm6[10],ymm7[10],ymm6[11],ymm7[11]
-; AVX512DQ-NEXT:    vpermt2d %zmm10, %zmm4, %zmm5
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,1,1,1]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm20, %ymm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm14[0],ymm0[0],ymm14[1],ymm0[1],ymm14[2],ymm0[2],ymm14[3],ymm0[3],ymm14[8],ymm0[8],ymm14[9],ymm0[9],ymm14[10],ymm0[10],ymm14[11],ymm0[11]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm3, %zmm2, %zmm13 {%k1}
-; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm6, %zmm4, %zmm5 {%k1}
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm26 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm2 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0],ymm7[1],ymm11[2,3],ymm7[4],ymm11[5,6],ymm7[7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm7 = ymm15[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vmovdqa %ymm15, %ymm6
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]
+; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm14
+; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm0, %xmm2
+; AVX512DQ-NEXT:    vmovdqa %xmm9, %xmm5
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm8
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm27
+; AVX512DQ-NEXT:    vmovdqa 64(%r9), %xmm9
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm9[0,0,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm9[0,2,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm8[0,0,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpbroadcastq %xmm2, %ymm0
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm8[0,2,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm2[0,0,2,1]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm14[0,0,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpbroadcastq %xmm2, %ymm29
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm3 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm3 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm14[0,2,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm30 = ymm2[0,0,2,1]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm10[0],ymm11[0],ymm10[1],ymm11[1],ymm10[2],ymm11[2],ymm10[3],ymm11[3],ymm10[8],ymm11[8],ymm10[9],ymm11[9],ymm10[10],ymm11[10],ymm10[11],ymm11[11]
+; AVX512DQ-NEXT:    vmovdqa 96(%r9), %ymm11
+; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm24, %zmm2
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm11[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm24 = ymm3[2,2,2,3]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm11[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm4[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm31 = ymm4[2,1,2,3]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX512DQ-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm10 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm10 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]
+; AVX512DQ-NEXT:    vmovdqa 64(%r9), %ymm4
+; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512DQ-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm7 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm7 = ymm3[0],mem[0],ymm3[1],mem[1],ymm3[2],mem[2],ymm3[3],mem[3],ymm3[8],mem[8],ymm3[9],mem[9],ymm3[10],mem[10],ymm3[11],mem[11]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm13 = ymm4[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm13 = ymm13[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm18 = ymm13[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[1,1,1,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm7, %zmm10, %zmm2 {%k1}
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm7 = ymm4[2,3,2,3,6,7,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm7[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm21 = ymm7[2,1,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm9
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm20 = zmm10[0,1,2,3],zmm9[4,5,6,7]
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm13, %ymm10
-; AVX512DQ-NEXT:    vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm11 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm11 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,2,3]
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm2, %ymm10
+; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm3
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm15 = ymm3[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,2,2,2]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm15 = ymm10[0,1],ymm15[2],ymm10[3,4],ymm15[5],ymm10[6,7]
 ; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm10
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm12
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,0,1]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0],ymm12[1],ymm13[2,3],ymm12[4],ymm13[5,6],ymm12[7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm13 = ymm10[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm13 = ymm13[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm24 = ymm13[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm11
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm12[0,1,2,3],zmm11[4,5,6,7]
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm5, %ymm12
-; AVX512DQ-NEXT:    vmovdqa64 %ymm25, %ymm0
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm14 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,2,2,2]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1],ymm14[2],ymm12[3,4],ymm14[5],ymm12[6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm14 = ymm10[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm14 = ymm14[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm1
-; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm9
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0],ymm1[1],ymm5[2,3],ymm1[4],ymm5[5,6],ymm1[7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm9[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm10[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5,6],ymm3[7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm10[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]
+; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm1 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm0, %zmm12
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm12[4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm9[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm12 = ymm12[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,1,2,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm15
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm15[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm13 = ymm15[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm13 = ymm13[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,1,2,3]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm14 = xmm14[2,3,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm14[0,2,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,1,0,1]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm15 = ymm15[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,2,2,2]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,2,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm11 = ymm11[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,3,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,2,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm4[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,3,2,3]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,2,2,1,4,5,6,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm9[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,2]
-; AVX512DQ-NEXT:    vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm15 = mem[2,3,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[0,2,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,1,0,1]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm16, %ymm0
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
-; AVX512DQ-NEXT:    vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm7 = mem[2,3,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm7[0,2,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm11 = ymm6[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]
-; AVX512DQ-NEXT:    vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm13 = mem[2,3,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm13[0,2,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm10[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2]
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm4 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm25
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm25 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm18, %zmm23, %zmm6
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm6 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm21, %zmm17, %zmm16
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm16 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm14, %zmm24, %zmm14
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm14 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm5
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm29, %zmm18, %zmm5
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm19, %zmm9, %zmm8
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm15, %zmm0
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm20, %zmm9, %zmm0
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm7, %zmm7
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm22, %zmm9, %zmm7
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm13, %zmm10
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm9, %zmm10
+; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm12 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm30, %zmm29, %zmm17
+; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm17 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm31, %zmm24, %zmm22
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
+; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm22 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm18, %zmm7
+; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm7 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm5, %zmm3
+; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm3 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm23, %zmm24, %zmm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm14, %zmm5
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm25, %zmm13, %zmm5
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm6, %zmm6
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm26, %zmm13, %zmm6
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm9, %zmm4
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm27, %zmm13, %zmm4
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm8, %zmm8
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm2, %zmm13, %zmm8
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vmovdqa64 %zmm10, 256(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm7, 448(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 640(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm8, 64(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm25, (%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 192(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm3, 384(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, 576(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm5, 128(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm14, 320(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 512(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 704(%rax)
-; AVX512DQ-NEXT:    addq $936, %rsp # imm = 0x3A8
+; AVX512DQ-NEXT:    vmovdqa64 %zmm8, 256(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 448(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 640(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, 64(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm17, (%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm12, 192(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 384(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 576(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm3, 320(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm7, 512(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm22, 704(%rax)
+; AVX512DQ-NEXT:    addq $584, %rsp # imm = 0x248
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i16_stride6_vf64:
 ; AVX512DQ-FCP:       # %bb.0:
-; AVX512DQ-FCP-NEXT:    subq $1224, %rsp # imm = 0x4C8
+; AVX512DQ-FCP-NEXT:    subq $1176, %rsp # imm = 0x498
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm0
@@ -9771,8 +9700,8 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %xmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm5
@@ -9782,95 +9711,95 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %ymm6
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm6, %ymm30
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %ymm7
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm7, %ymm29
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm31
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %ymm8
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm8[0],ymm1[0],ymm8[1],ymm1[1],ymm8[2],ymm1[2],ymm8[3],ymm1[3],ymm8[8],ymm1[8],ymm8[9],ymm1[9],ymm8[10],ymm1[10],ymm8[11],ymm1[11]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm8, %ymm28
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm17
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm8, %ymm30
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm20
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm16
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [1,1,1,1,10,10,10,11]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm17 = [1,1,1,1,10,10,10,11]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm18, %zmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [1,2,1,2,0,0,3,3,13,12,10,10,13,12,14,14]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm20, %zmm22
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm17, %zmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [1,2,1,2,0,0,3,3,13,12,10,10,13,12,14,14]
+; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm19, %zmm21
 ; AVX512DQ-FCP-NEXT:    movw $18724, %ax # imm = 0x4924
 ; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm3, %zmm22 {%k1}
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm25 = [8,9,20,11,12,21,14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm22, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm3, %zmm21 {%k1}
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm24 = [0,9,2,3,8,5,6,11]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm21, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm0, %ymm24, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm18 = [8,9,20,11,12,21,14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm25, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm19 = [0,9,2,3,8,5,6,11]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm0, %ymm19, %ymm22
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [0,0,0,1,0,10,10,0]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm18, %zmm21
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm22 = [0,0,0,1,0,10,10,0]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm23, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm1, %xmm1
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm22, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %ymm9
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %ymm5
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm5[0],ymm9[0],ymm5[1],ymm9[1],ymm5[2],ymm9[2],ymm5[3],ymm9[3],ymm5[8],ymm9[8],ymm5[9],ymm9[9],ymm5[10],ymm9[10],ymm5[11],ymm9[11]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm14
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm14[4],xmm0[4],xmm14[5],xmm0[5],xmm14[6],xmm0[6],xmm14[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm21
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm18, %zmm4
-; AVX512DQ-FCP-NEXT:    vpermd %zmm6, %zmm20, %zmm26
-; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm4, %zmm26 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm15
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm23
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm17, %zmm4
+; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm19, %zmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm4, %zmm25 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm1, %xmm2
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm2, %ymm24, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %ymm10
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm10[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm25, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm0, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm24
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm3, %ymm19, %ymm26
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm18, %zmm25
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %ymm11
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm11[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm23, %zmm0
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm22, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -9880,332 +9809,329 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %ymm12
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm12[0],ymm13[0],ymm12[1],ymm13[1],ymm12[2],ymm13[2],ymm12[3],ymm13[3],ymm12[8],ymm13[8],ymm12[9],ymm13[9],ymm12[10],ymm13[10],ymm12[11],ymm13[11]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm7, %zmm18, %zmm6
-; AVX512DQ-FCP-NEXT:    vpermd %zmm17, %zmm20, %zmm17
-; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm6, %zmm17 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %ymm6
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm6[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm7, %zmm25, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm7, %zmm17, %zmm6
+; AVX512DQ-FCP-NEXT:    vpermd %zmm20, %zmm19, %zmm28
+; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm6, %zmm28 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm20
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm0, %xmm7
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm7, %ymm19, %ymm17
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm0, %xmm6
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm6, %ymm24, %ymm20
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %ymm6
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm6[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm7, %zmm18, %zmm28
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %ymm7
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm7[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm23, %zmm1
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm1, %xmm1
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm22, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, (%rsp) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm18, %zmm0
-; AVX512DQ-FCP-NEXT:    vpermd %zmm16, %zmm20, %zmm20
-; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm0, %zmm20 {%k1}
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm17, %zmm0
+; AVX512DQ-FCP-NEXT:    vpermd %zmm16, %zmm19, %zmm19
+; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vpermi2d %ymm0, %ymm19, %ymm24
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermi2d %zmm0, %zmm20, %zmm25
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm0, %ymm19, %ymm20
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm18, %zmm19
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm0, %xmm1
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm0, %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm23, %zmm1
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm22, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm15 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm9, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm5, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm9, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm5, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm5[4],ymm9[4],ymm5[5],ymm9[5],ymm5[6],ymm9[6],ymm5[7],ymm9[7],ymm5[12],ymm9[12],ymm5[13],ymm9[13],ymm5[14],ymm9[14],ymm5[15],ymm9[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm29 = [2,1,2,3,11,11,11,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm29, %zmm8
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm27 = [5,6,5,6,5,6,7,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm2
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [2,1,2,3,11,11,11,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm27, %zmm8
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm26 = [5,6,5,6,5,6,7,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm31, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm0, %ymm27, %ymm0
+; AVX512DQ-FCP-NEXT:    vpermd %ymm0, %ymm26, %ymm0
 ; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm5 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm0, %zmm1, %zmm8 {%k1}
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm23 = [8,21,10,11,20,13,14,23]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm22 = [12,1,2,13,4,5,14,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm10[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm1, %ymm22, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, %ymm9
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm23, %zmm0
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm10[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm30 = [12,1,2,13,4,5,14,7]
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm1, %ymm30, %ymm8
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm8[0,1,2,3],zmm0[0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, %ymm10
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm29 = [8,21,10,11,20,13,14,23]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm29, %zmm8
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm11, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm8
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm5
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]
 ; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm11, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, %ymm11
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, %ymm8
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm31 = [2,2,0,3,10,0,10,11]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm31, %zmm1
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm12, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm13, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm12, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm12[4],ymm13[4],ymm12[5],ymm13[5],ymm12[6],ymm13[6],ymm12[7],ymm13[7],ymm12[12],ymm13[12],ymm12[13],ymm13[13],ymm12[14],ymm13[14],ymm12[15],ymm13[15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm29, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm4
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm27, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm4
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm2, %ymm27, %ymm2
+; AVX512DQ-FCP-NEXT:    vpermd %ymm2, %ymm26, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm3 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm4 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm2, %zmm3, %zmm0 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm6, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm9, %ymm10
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm23, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm3, %ymm30, %ymm0
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm3, %ymm22, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm6, %ymm3
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm29, %zmm0
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm7, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm7, %ymm2
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm31, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm4
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm4, %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm14, %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm19 = [0,0,2,1,8,9,8,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm19, %zmm3
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [1,0,2,2,1,0,2,2]
-; AVX512DQ-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm7, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm7, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm8, %ymm12
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm31, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm23, %xmm4
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm0
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm15, %xmm2
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm15[0],xmm4[0],xmm15[1],xmm4[1],xmm15[2],xmm4[2],xmm15[3],xmm4[3]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [0,0,2,1,8,9,8,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm2
+; AVX512DQ-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm30 = [1,0,2,2,1,0,2,2]
+; AVX512DQ-FCP-NEXT:    # ymm30 = mem[0,1,2,3,0,1,2,3]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm4, %ymm0, %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm0, %ymm30, %ymm0
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm6 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm5, %ymm5
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm4, %ymm4
 ; AVX512DQ-FCP-NEXT:    movw $9362, %ax # imm = 0x2492
 ; AVX512DQ-FCP-NEXT:    kmovw %eax, %k2
-; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm5, %zmm4, %zmm3 {%k2}
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm28 = [16,9,10,17,12,13,18,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm6
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm6[2,1,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm5, %zmm28, %zmm4
-; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm5 = xmm24[0],zero,xmm24[1],zero,xmm24[2],zero,xmm24[3],zero
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm16 = [0,1,8,3,4,9,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm5, %ymm16, %ymm3
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm4[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm4, %zmm0, %zmm2 {%k2}
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,1,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, %ymm5
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [16,9,10,17,12,13,18,15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm2
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,1,8,3,4,9,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm4, %ymm0, %ymm5
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm5, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm5, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, %xmm8
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm5[0,0,2,1,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [0,0,0,0,8,8,0,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm18, %zmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm5, %xmm3
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm5, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, %xmm9
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm5[0,0,2,1,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [0,0,0,0,8,8,0,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm16, %zmm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm4
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm19, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm6, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm7, %xmm5
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm18, %zmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
 ; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm6 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm3, %ymm0, %ymm3
-; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm5, %ymm5
-; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm5, %zmm3, %zmm4 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm6[2,1,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm5, %zmm28, %zmm3
-; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm5, %ymm16, %ymm4
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm12 = zmm4[0,1,2,3],zmm3[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm8, %xmm21
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,0,2,1,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm18, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm8, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm6, %ymm5
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm8[4],ymm6[5],ymm8[5],ymm6[6],ymm8[6],ymm6[7],ymm8[7],ymm6[12],ymm8[12],ymm6[13],ymm8[13],ymm6[14],ymm8[14],ymm6[15],ymm8[15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm29, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm8[4],ymm6[4],ymm8[5],ymm6[5],ymm8[6],ymm6[6],ymm8[7],ymm6[7],ymm8[12],ymm6[12],ymm8[13],ymm6[13],ymm8[14],ymm6[14],ymm8[15],ymm6[15]
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm6 = ymm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm6[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm8 = ymm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm8[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[2],ymm6[2],ymm8[3],ymm6[3],ymm8[8],ymm6[8],ymm8[9],ymm6[9],ymm8[10],ymm6[10],ymm8[11],ymm6[11]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm5, %ymm27, %ymm5
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,2]
-; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm5, %zmm6, %zmm3 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm6
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm10, %ymm14
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm5
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm5, %zmm23, %zmm6
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm8[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm5, %ymm30, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm13, %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm10, %ymm8
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm8[0],ymm5[0],ymm8[1],ymm5[1],ymm8[2],ymm5[2],ymm8[3],ymm5[3],ymm8[8],ymm5[8],ymm8[9],ymm5[9],ymm8[10],ymm5[10],ymm8[11],ymm5[11]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm8 = ymm10[4],ymm13[4],ymm10[5],ymm13[5],ymm10[6],ymm13[6],ymm10[7],ymm13[7],ymm10[12],ymm13[12],ymm10[13],ymm13[13],ymm10[14],ymm13[14],ymm10[15],ymm13[15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm8, %zmm29, %zmm5
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm29 = zmm3[0,1,2,3],zmm6[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm3, %ymm8
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm11, %ymm15
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm8, %zmm31, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm7 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm2, %ymm30, %ymm2
+; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm6, %ymm6
+; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm6, %zmm2, %zmm5 {%k2}
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm5, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm7[2,1,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm6, %zmm17, %zmm5
+; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm6 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm6, %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm2, %zmm23
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm5, %xmm2
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,0,2,1,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm16, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm9, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm7, %ymm6
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm9[4],ymm7[5],ymm9[5],ymm7[6],ymm9[6],ymm7[7],ymm9[7],ymm7[12],ymm9[12],ymm7[13],ymm9[13],ymm7[14],ymm9[14],ymm7[15],ymm9[15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm27, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm9[4],ymm7[4],ymm9[5],ymm7[5],ymm9[6],ymm7[6],ymm9[7],ymm7[7],ymm9[12],ymm7[12],ymm9[13],ymm7[13],ymm9[14],ymm7[14],ymm9[15],ymm7[15]
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm7 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm9 = ymm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm9[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm9[0],ymm7[0],ymm9[1],ymm7[1],ymm9[2],ymm7[2],ymm9[3],ymm7[3],ymm9[8],ymm7[8],ymm9[9],ymm7[9],ymm9[10],ymm7[10],ymm9[11],ymm7[11]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm2, %ymm26, %ymm2
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,2]
+; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm2, %zmm7, %zmm6 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm6, %ymm7
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm9[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm2, %ymm22, %ymm7
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm10, %ymm13
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm9, %ymm2
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm29, %zmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm8 = ymm11[4],ymm6[4],ymm11[5],ymm6[5],ymm11[6],ymm6[6],ymm11[7],ymm6[7],ymm11[12],ymm6[12],ymm11[13],ymm6[13],ymm11[14],ymm6[14],ymm11[15],ymm6[15]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm27, %ymm8
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm10 = ymm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm6[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm11 = ymm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm11[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[8],ymm10[8],ymm11[9],ymm10[9],ymm11[10],ymm10[10],ymm11[11],ymm10[11]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2]
-; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm8, %zmm10, %zmm5 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm6, %ymm8
-; AVX512DQ-FCP-NEXT:    vpermi2d %zmm8, %zmm5, %zmm23
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm8, %ymm30, %ymm5
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm11, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm10, %ymm9
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm9[0],ymm2[0],ymm9[1],ymm2[1],ymm9[2],ymm2[2],ymm9[3],ymm2[3],ymm9[8],ymm2[8],ymm9[9],ymm2[9],ymm9[10],ymm2[10],ymm9[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm9 = ymm10[4],ymm11[4],ymm10[5],ymm11[5],ymm10[6],ymm11[6],ymm10[7],ymm11[7],ymm10[12],ymm11[12],ymm10[13],ymm11[13],ymm10[14],ymm11[14],ymm10[15],ymm11[15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm9, %zmm27, %zmm2
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm7, %zmm27
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm6, %ymm10
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm6, %ymm8
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm10, %zmm31, %zmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm13, %xmm11
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm6[0],xmm13[1],xmm6[1],xmm13[2],xmm6[2],xmm13[3],xmm6[3]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm11, %zmm19, %zmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm11, %ymm0, %ymm11
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm13 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm14 = xmm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm6, %ymm7
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm12, %ymm15
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm6, %ymm6
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm7, %zmm31, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm7 = ymm10[4],ymm9[4],ymm10[5],ymm9[5],ymm10[6],ymm9[6],ymm10[7],ymm9[7],ymm10[12],ymm9[12],ymm10[13],ymm9[13],ymm10[14],ymm9[14],ymm10[15],ymm9[15]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm26, %ymm7
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm11 = ymm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm9[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} ymm12 = ymm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm10[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm12[0],ymm11[0],ymm12[1],ymm11[1],ymm12[2],ymm11[2],ymm12[3],ymm11[3],ymm12[8],ymm11[8],ymm12[9],ymm11[9],ymm12[10],ymm11[10],ymm12[11],ymm11[11]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]
+; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm7, %zmm11, %zmm2 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm9[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermi2d %ymm7, %ymm2, %ymm22
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm9, %ymm7
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm7, %zmm29, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm7, %ymm11
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm7, %ymm7
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm11, %zmm31, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm9, %xmm11
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsp), %xmm10 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm10, %xmm12
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm12 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm12, %zmm18, %zmm11
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm12, %ymm30, %ymm12
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm13 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm14 = xmm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm13, %ymm13
-; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm13, %zmm11, %zmm10 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, %zmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsp), %xmm15 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm15[2,1,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm13, %zmm28, %zmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm14, %xmm13
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm9
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm9[4],xmm13[4],xmm9[5],xmm13[5],xmm9[6],xmm13[6],xmm9[7],xmm13[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm6[0],xmm14[0],xmm6[1],xmm14[1],xmm6[2],xmm14[2],xmm6[3],xmm14[3]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm9, %zmm19, %zmm13
-; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm9 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm6[0],xmm15[0],xmm6[1],xmm15[1],xmm6[2],xmm15[2],xmm6[3],xmm15[3]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm14, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm14 = xmm15[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm15 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
-; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm14, %ymm14
-; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm14, %zmm0, %zmm13 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm6[2,1,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermi2d %zmm0, %zmm13, %zmm28
-; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm9, %ymm16, %ymm10
-; AVX512DQ-FCP-NEXT:    vpermt2d %ymm0, %ymm16, %ymm13
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm5[0,1,2,3],zmm23[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm10[0,1,2,3],zmm11[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm11
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm6, %xmm9
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm6[0,0,2,1,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm9, %zmm18, %zmm10
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm13[0,1,2,3],zmm28[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm6, %xmm11
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm6[0,0,2,1,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm11, %zmm18, %zmm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm9, %zmm11, %zmm13
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm10, %xmm14
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm9, %xmm8
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm8[4],xmm14[4],xmm8[5],xmm14[5],xmm8[6],xmm14[6],xmm8[7],xmm14[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm14, %zmm18, %zmm8
+; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm13, %zmm12, %zmm11 {%k2}
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm12 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm9[2,1,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm11, %ymm14
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm13, %zmm17, %zmm11
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm13, %ymm30, %ymm13
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm15 = xmm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm10 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm15[0],xmm10[1],xmm15[1],xmm10[2],xmm15[2],xmm10[3],xmm15[3]
+; AVX512DQ-FCP-NEXT:    vpbroadcastq %xmm10, %ymm10
+; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm10, %zmm13, %zmm8 {%k2}
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm12, %ymm0, %ymm14
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpmovzxwd {{.*#+}} xmm10 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero
+; AVX512DQ-FCP-NEXT:    vpermi2d %ymm10, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm9[2,1,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm10, %zmm17, %zmm8
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm22, %zmm2
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm14, %zmm10
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm9, %xmm11
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm9[0,0,2,1,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm11, %zmm16, %zmm12
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm9, %xmm8
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm9[0,0,2,1,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm8, %zmm16, %zmm11
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm8, %zmm11
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, (%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm5, %zmm11, %zmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, 192(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm5, %zmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm8, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm29, %zmm5, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 320(%rax)
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm20[0,1,2,3],zmm25[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 256(%rax)
-; AVX512DQ-FCP-NEXT:    vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm0 = zmm17[0,1,2,3],mem[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 448(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm12, %zmm11, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 384(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm7 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 576(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm2 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 512(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm1 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm10, %zmm8, %zmm12
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, 192(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm0, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 128(%rax)
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm27, %zmm0, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 320(%rax)
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm24, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm6, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 256(%rax)
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm28, %zmm20, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm6, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 448(%rax)
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm23, %zmm8, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 384(%rax)
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm4 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 576(%rax)
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 512(%rax)
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 704(%rax)
-; AVX512DQ-FCP-NEXT:    vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm0 = zmm26[0,1,2,3],mem[0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm25, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm6, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 640(%rax)
-; AVX512DQ-FCP-NEXT:    vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm0 = zmm22[0,1,2,3],mem[0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm6, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 64(%rax)
-; AVX512DQ-FCP-NEXT:    addq $1224, %rsp # imm = 0x4C8
+; AVX512DQ-FCP-NEXT:    addq $1176, %rsp # imm = 0x498
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
index 208ee607909ed..dc362d729fcd3 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
@@ -2850,8 +2850,8 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vporq %ymm1, %ymm4, %ymm17
 ; AVX512-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512-NEXT:    vmovdqa (%rsi), %xmm4
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm5 = ymm3[u,u,u,u,u,u,u,u,u,u],zero,zero,ymm3[14,15,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm3[16,17,u,u,u,u]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,u,u,14,15],zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm2[u,u,u,u]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm5 = ymm3[u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[14,15,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[16,17,u,u],zero,zero
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,12,13,14,15],zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm2[u,u,u,u]
 ; AVX512-NEXT:    vporq %ymm5, %ymm10, %ymm19
 ; AVX512-NEXT:    vpbroadcastd 8(%rax), %ymm5
 ; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm5
@@ -2982,8 +2982,8 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,14,15],zero,zero,ymm4[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm4[u,u,u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vporq %ymm0, %ymm3, %ymm17
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm14
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u],zero,zero,ymm2[14,15,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm2[16,17,u,u,u,u]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,14,15],zero,zero,ymm1[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm1[u,u,u,u]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[14,15,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[16,17,u,u],zero,zero
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,12,13,14,15],zero,zero,ymm1[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm1[u,u,u,u]
 ; AVX512-FCP-NEXT:    vporq %ymm0, %ymm3, %ymm18
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,u,u,u,u,26,27,u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm5[2,2,2,2,6,6,6,6]
@@ -3112,8 +3112,8 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vporq %ymm1, %ymm4, %ymm17
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm4
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm5 = ymm3[u,u,u,u,u,u,u,u,u,u],zero,zero,ymm3[14,15,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm3[16,17,u,u,u,u]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,u,u,14,15],zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm2[u,u,u,u]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm5 = ymm3[u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[14,15,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[16,17,u,u],zero,zero
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,12,13,14,15],zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm2[u,u,u,u]
 ; AVX512DQ-NEXT:    vporq %ymm5, %ymm10, %ymm19
 ; AVX512DQ-NEXT:    vpbroadcastd 8(%rax), %ymm5
 ; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm5
@@ -3244,8 +3244,8 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,14,15],zero,zero,ymm4[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm4[u,u,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm3, %ymm17
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm14
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u],zero,zero,ymm2[14,15,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm2[16,17,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,14,15],zero,zero,ymm1[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm1[u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[14,15,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[16,17,u,u],zero,zero
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,12,13,14,15],zero,zero,ymm1[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm1[u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm3, %ymm18
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,u,u,u,u,26,27,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm5[2,2,2,2,6,6,6,6]
@@ -5893,1237 +5893,1257 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512-LABEL: store_i16_stride7_vf32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    subq $632, %rsp # imm = 0x278
+; AVX512-NEXT:    subq $680, %rsp # imm = 0x2A8
 ; AVX512-NEXT:    vmovdqa (%rcx), %ymm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpshufb %ymm9, %ymm1, %ymm0
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm16
-; AVX512-NEXT:    vmovdqa (%rdx), %ymm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vpshufb %ymm13, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm29
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
+; AVX512-NEXT:    vpshufb %ymm13, %ymm1, %ymm0
+; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm27
+; AVX512-NEXT:    vmovdqa (%rdx), %ymm8
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm14 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vpshufb %ymm14, %ymm8, %ymm1
 ; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa (%rsi), %ymm1
+; AVX512-NEXT:    vmovdqa (%rsi), %ymm9
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpshufb %ymm12, %ymm1, %ymm0
-; AVX512-NEXT:    vmovdqa %ymm1, %ymm15
-; AVX512-NEXT:    vmovdqa (%rdi), %ymm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm14 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
-; AVX512-NEXT:    vpshufb %ymm14, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm17
+; AVX512-NEXT:    vpshufb %ymm12, %ymm9, %ymm0
+; AVX512-NEXT:    vmovdqa (%rdi), %ymm11
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm15 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
+; AVX512-NEXT:    vpshufb %ymm15, %ymm11, %ymm1
 ; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa (%r9), %ymm10
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
-; AVX512-NEXT:    vpshufb %ymm1, %ymm10, %ymm2
-; AVX512-NEXT:    vmovdqa (%r8), %ymm11
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
-; AVX512-NEXT:    vpshufb %ymm0, %ymm11, %ymm3
+; AVX512-NEXT:    vmovdqa (%r9), %ymm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
+; AVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
+; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm16
+; AVX512-NEXT:    vmovdqa (%r8), %ymm4
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
+; AVX512-NEXT:    vpshufb %ymm1, %ymm4, %ymm3
+; AVX512-NEXT:    vmovdqa64 %ymm4, %ymm17
 ; AVX512-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%r9), %xmm3
-; AVX512-NEXT:    vmovdqa 32(%r8), %xmm6
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm10[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm2[2],ymm4[3,4],ymm2[5],ymm4[6,7,8,9],ymm2[10],ymm4[11,12],ymm2[13],ymm4[14,15]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [4,5,4,5,4,5,6,7,16,17,16,17,16,17,17,19]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm2[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpermi2d %zmm5, %zmm4, %zmm7
-; AVX512-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm8
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,6]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512-NEXT:    vmovdqa 32(%r9), %xmm2
+; AVX512-NEXT:    vmovdqa 32(%r8), %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3]
+; AVX512-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,6]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [16,18,19,19,19,19,0,0,0,1,0,1,2,3,2,3]
-; AVX512-NEXT:    vpermi2d %zmm2, %zmm3, %zmm4
+; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
 ; AVX512-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm6
-; AVX512-NEXT:    vpshufb %ymm12, %ymm6, %ymm2
-; AVX512-NEXT:    vpshufb %ymm14, %ymm3, %ymm4
-; AVX512-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm2
+; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm10
+; AVX512-NEXT:    vpshufb %ymm12, %ymm10, %ymm3
+; AVX512-NEXT:    vpshufb %ymm15, %ymm2, %ymm4
+; AVX512-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm12
-; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm7
-; AVX512-NEXT:    vpshufb %ymm9, %ymm12, %ymm2
-; AVX512-NEXT:    vpshufb %ymm13, %ymm7, %ymm4
-; AVX512-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm15
+; AVX512-NEXT:    vpshufb %ymm13, %ymm12, %ymm3
+; AVX512-NEXT:    vpshufb %ymm14, %ymm15, %ymm4
+; AVX512-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqa 32(%r9), %ymm13
 ; AVX512-NEXT:    vmovdqa 32(%r8), %ymm14
-; AVX512-NEXT:    vpshufb %ymm1, %ymm13, %ymm1
-; AVX512-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpshufb %ymm0, %ymm13, %ymm0
+; AVX512-NEXT:    vpshufb %ymm1, %ymm14, %ymm1
+; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm20
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,2]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,16,0,0,17,17,0,0,0,0,0,1,2,0,0,3]
-; AVX512-NEXT:    vpermi2d %zmm2, %zmm1, %zmm4
-; AVX512-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512-NEXT:    vmovdqa (%r8), %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm4[0,1,2,3,4,5,7,6]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm9 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
-; AVX512-NEXT:    vpermi2d %zmm5, %zmm4, %zmm9
-; AVX512-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vprold $16, %ymm10, %ymm4
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm11[1,2,2,3,5,6,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm11[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm10[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm9[0,1,2],ymm5[3],ymm9[4,5],ymm5[6],ymm9[7,8,9,10],ymm5[11],ymm9[12,13],ymm5[14],ymm9[15]
-; AVX512-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [2,2,3,3,10,9,11,10]
-; AVX512-NEXT:    vpermi2q %zmm4, %zmm5, %zmm27
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm0
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm3
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm4
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,2,2]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3],xmm4[4],xmm5[5,6],xmm4[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm26 = [0,16,0,0,17,17,0,0,0,0,0,1,2,0,0,3]
+; AVX512-NEXT:    vpermi2d %zmm5, %zmm4, %zmm26
+; AVX512-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512-NEXT:    vmovdqa (%r8), %xmm5
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm6[0,1,2,3,4,5,7,6]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm25 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
+; AVX512-NEXT:    vpermi2d %zmm7, %zmm6, %zmm25
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm31
-; AVX512-NEXT:    vmovdqa (%rax), %ymm4
-; AVX512-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-NEXT:    vpbroadcastd 8(%rax), %ymm0
 ; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm5 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
-; AVX512-NEXT:    vpshufb %ymm5, %ymm4, %ymm4
-; AVX512-NEXT:    vmovdqa64 %ymm5, %ymm28
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
+; AVX512-NEXT:    vmovdqa (%rax), %ymm3
+; AVX512-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm7 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
+; AVX512-NEXT:    vpshufb %ymm7, %ymm3, %ymm6
+; AVX512-NEXT:    vmovdqa64 %ymm7, %ymm21
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm7[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm30
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm12[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm7[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm23
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm6[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm11 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm6[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm19
-; AVX512-NEXT:    vprold $16, %ymm13, %ymm0
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm14[1,2,2,3,5,6,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
-; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm13[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm14[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
-; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm4 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm15[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm28
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm12[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm15[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6,7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13,14,15]
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm29
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm10[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6,7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13,14,15]
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm30
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm10[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1,2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8,9,10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm31
+; AVX512-NEXT:    vprold $16, %ymm13, %ymm4
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm14[1,2,2,3,5,6,6,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
+; AVX512-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm13[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm14[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
+; AVX512-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqa64 %ymm27, %ymm3
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm3[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm8[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6,7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13,14,15]
+; AVX512-NEXT:    vmovdqu %ymm4, (%rsp) # 32-byte Spill
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm9[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm11[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8,9,10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
+; AVX512-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-NEXT:    vmovdqa64 %ymm16, %ymm4
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm4[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm29[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm5, %ymm18
-; AVX512-NEXT:    vmovdqa %ymm15, %ymm8
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm15[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm17[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
-; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm0
-; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm9
-; AVX512-NEXT:    vprold $16, %xmm9, %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm15 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm15[0,1],xmm1[2],xmm15[3,4],xmm1[5],xmm15[6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm17[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
+; AVX512-NEXT:    vmovdqa64 %ymm4, %ymm19
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm4
+; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm5
+; AVX512-NEXT:    vprold $16, %xmm5, %xmm6
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,3]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm7[0,1],xmm6[2],xmm7[3,4],xmm6[5],xmm7[6,7]
+; AVX512-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm4
+; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm5
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[1,1,2,2]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm7[0],xmm1[1],xmm7[2,3],xmm1[4],xmm7[5,6],xmm1[7]
 ; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
-; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm0
-; AVX512-NEXT:    vpshufb %xmm0, %xmm9, %xmm0
-; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm15
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm10 = xmm15[1,1,2,2]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm10[0],xmm0[1],xmm10[2,3],xmm0[4],xmm10[5,6],xmm0[7]
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm26
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm15[4],xmm9[4],xmm15[5],xmm9[5],xmm15[6],xmm9[6],xmm15[7],xmm9[7]
-; AVX512-NEXT:    vmovdqa64 %xmm9, %xmm25
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,1,1,5,5,5,5]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm6[0,1],ymm3[2],ymm6[3,4],ymm3[5],ymm6[6,7,8,9],ymm3[10],ymm6[11,12],ymm3[13],ymm6[14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm24
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[0,1,1,3,4,5,5,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm12[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm22
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm14[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm13[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm6[0,1,2],ymm3[3],ymm6[4,5],ymm3[6],ymm6[7,8,9,10],ymm3[11],ymm6[12,13],ymm3[14],ymm6[15]
-; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm21
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,5,0,0,0,6,0,0,6,0,0,0,7,0,0,7]
-; AVX512-NEXT:    vmovdqa 32(%rax), %ymm6
-; AVX512-NEXT:    vpermd %zmm6, %zmm3, %zmm3
-; AVX512-NEXT:    vmovdqa64 %ymm28, %ymm7
-; AVX512-NEXT:    vpshufb %ymm7, %ymm6, %ymm10
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,1,1,3,4,5,5,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3]
-; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm6
-; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm10, %zmm6
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512-NEXT:    vpshufb %xmm7, %xmm2, %xmm10
-; AVX512-NEXT:    vpshufb %xmm7, %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm13
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm14
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
-; AVX512-NEXT:    vpshufb %xmm7, %xmm2, %xmm2
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm20
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
-; AVX512-NEXT:    vprold $16, %xmm14, %xmm14
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[1,1,2,3]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm13[0,1],xmm14[2],xmm13[3,4],xmm14[5],xmm13[6,7]
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm16
-; AVX512-NEXT:    vmovdqa %ymm4, %ymm2
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm14 = ymm4[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm14 = ymm14[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vmovdqa64 %ymm29, %ymm12
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm29[0,1,1,3,4,5,5,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm14 = ymm9[0,1],ymm14[2],ymm9[3,4],ymm14[5],ymm9[6,7,8,9],ymm14[10],ymm9[11,12],ymm14[13],ymm9[14,15]
-; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm4
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm4[0,2,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[2,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,5,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,0,1,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm28 = ymm30[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm29 = ymm23[0,2,2,3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm18
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[1,1,1,1,5,5,5,5]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm10[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,0,2,1,4,4,6,5]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7,8,9],ymm1[10],ymm2[11,12],ymm1[13],ymm2[14,15]
+; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm15[0,1,1,3,4,5,5,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm12[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,0,0,0,4,4,4,4]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm14[0,0,2,1,4,4,6,5]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm13[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,0,0,0,4,4,4,4]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8,9,10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
+; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,5,0,0,0,6,0,0,6,0,0,0,7,0,0,7]
+; AVX512-NEXT:    vmovdqa 32(%rax), %ymm2
+; AVX512-NEXT:    vpermd %zmm2, %zmm1, %zmm27
+; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm1
+; AVX512-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,1,1,3,4,5,5,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm20
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm5
+; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm14
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm4
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX512-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm21
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512-NEXT:    vprold $16, %xmm2, %xmm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
+; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm24
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm3[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm8[0,1,1,3,4,5,5,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7,8,9],ymm1[10],ymm2[11,12],ymm1[13],ymm2[14,15]
+; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm23
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm8[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm22
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm9[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,2,1,4,4,6,5]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[1,1,1,1,5,5,5,5]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm13 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm11[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm9[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm11 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6,7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14,15]
+; AVX512-NEXT:    vmovdqa64 %ymm16, %ymm4
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm4[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm17[0,0,2,1,4,4,6,5]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm8 = ymm1[0,1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7,8,9,10],ymm2[11],ymm1[12,13],ymm2[14],ymm1[15]
+; AVX512-NEXT:    vprold $16, %ymm16, %ymm1
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm17[1,2,2,3,5,6,6,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm7 = ymm10[0,1],ymm1[2],ymm10[3,4],ymm1[5],ymm10[6,7,8,9],ymm1[10],ymm10[11,12],ymm1[13],ymm10[14,15]
+; AVX512-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm10 = mem[0,2,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,5,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,0,1,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[0,0,1,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm28 = ymm28[2,2,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm29 = ymm29[0,2,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm30 = ymm30[0,2,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm31 = ymm31[2,1,3,3]
+; AVX512-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm6 = mem[2,1,3,2]
+; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm5 = mem[2,2,2,3]
+; AVX512-NEXT:    vpermq $232, (%rsp), %ymm1 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm1 = mem[0,2,2,3]
+; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm0
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm0[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[0,0,1,1]
+; AVX512-NEXT:    vpermq $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm0 = mem[2,1,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm14[0,0,1,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm19[2,2,2,3]
+; AVX512-NEXT:    vpshuflw $180, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm15 = mem[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[0,1,1,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm29, %zmm28, %zmm28
+; AVX512-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm29
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogq $184, %zmm28, %zmm30, %zmm29
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,1,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm1, %zmm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm9
+; AVX512-NEXT:    vpternlogq $226, %zmm1, %zmm30, %zmm9
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 32-byte Folded Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm1 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogq $226, %zmm0, %zmm2, %zmm1
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm0 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm0
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0
+; AVX512-NEXT:    vpshufd $254, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm1 = mem[2,3,3,3,6,7,7,7]
+; AVX512-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm3 = mem[0,0,2,1]
+; AVX512-NEXT:    vpshuflw $230, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm10 = mem[2,1,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,5,5,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,1,3]
+; AVX512-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm12 = mem[0,0,1,1]
+; AVX512-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm14 = mem[0,2,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,2,1]
+; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm17 = mem[2,2,2,3]
+; AVX512-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm18 = mem[2,1,3,2]
+; AVX512-NEXT:    vpermq $250, {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm19 = mem[2,2,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm21 = ymm21[0,0,1,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm28 = ymm24[0,0,2,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm30 = ymm23[2,1,3,2]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm31 = ymm22[2,2,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,2,2,3]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm19[2,1,3,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm30 = ymm18[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm31 = ymm5[2,1,3,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm12[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm8[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm17[1,1,1,1,5,5,5,5]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm17[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm12 = ymm8[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm12[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6,7,8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14,15]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm29, %zmm28, %zmm12
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm11, %zmm7
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq $184, %zmm12, %zmm11, %zmm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm30, %zmm0
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm31, %zmm1
-; AVX512-NEXT:    vpternlogq $226, %zmm0, %zmm11, %zmm1
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm0 # 32-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm15, %zmm9 # 32-byte Folded Reload
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq $226, %zmm0, %zmm11, %zmm9
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm10[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm10 = mem[2,1,3,2]
-; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm12 = mem[2,2,2,3]
-; AVX512-NEXT:    vpshufd $254, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm15 = mem[2,3,3,3,6,7,7,7]
-; AVX512-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm17 = mem[0,0,2,1]
-; AVX512-NEXT:    vpshuflw $230, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm8 = mem[2,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,5,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,1,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm18 = ymm26[0,0,1,1]
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm13
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm13[0,2,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,2,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm19 = ymm24[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm28 = ymm22[2,1,3,2]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm29 = ymm21[2,2,3,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm30 = ymm20[0,0,1,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm31 = ymm16[0,0,2,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,1,3,2]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3]
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm16, %zmm0
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm0
-; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm9
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm9
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm3
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm15[2,1,3,2]
-; AVX512-NEXT:    vpbroadcastd 32(%rax), %ymm9
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm7, %zmm7
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm7
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm7
-; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm17, %zmm1
-; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm18, %zmm8
-; AVX512-NEXT:    vpternlogq $226, %zmm1, %zmm11, %zmm8
-; AVX512-NEXT:    vpbroadcastd 36(%rax), %ymm1
-; AVX512-NEXT:    vpbroadcastd 40(%rax), %ymm9
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm1, %zmm1
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm1
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm1
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm19, %zmm8, %zmm8
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm28, %zmm9, %zmm9
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm9
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],zmm29[0,1,2,3]
-; AVX512-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm6
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm6
-; AVX512-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm8
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm8
-; AVX512-NEXT:    vpbroadcastd (%rax), %ymm9
-; AVX512-NEXT:    vpbroadcastd 4(%rax), %ymm10
-; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm9
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm9
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm14, %zmm2
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,2]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm29, %zmm5
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm27
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm15[0,0,0,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm4
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
-; AVX512-NEXT:    vpermd (%rax), %zmm2, %zmm2
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm2
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,2]
+; AVX512-NEXT:    vpbroadcastd 32(%rax), %ymm5
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm3, %zmm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm12, %zmm4
+; AVX512-NEXT:    vpternlogq $226, %zmm3, %zmm2, %zmm4
+; AVX512-NEXT:    vpbroadcastd 36(%rax), %ymm2
+; AVX512-NEXT:    vpbroadcastd 40(%rax), %ymm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm2
 ; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm2
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm17, %zmm3, %zmm3
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm18, %zmm4, %zmm4
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm4
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm19[0,1,2,3]
+; AVX512-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm20
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm20
+; AVX512-NEXT:    vinserti64x4 $1, %ymm28, %zmm21, %zmm3
+; AVX512-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm26, %zmm3
+; AVX512-NEXT:    vpbroadcastd (%rax), %ymm4
+; AVX512-NEXT:    vpbroadcastd 4(%rax), %ymm5
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm25, %zmm4
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm4
+; AVX512-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm5
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm3
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
+; AVX512-NEXT:    vpermd (%rax), %zmm6, %zmm6
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm6
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vmovdqa64 %zmm2, 128(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm9, (%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm6, 320(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm1, 256(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm7, 192(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm6, 128(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm20, 320(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm2, 256(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm1, 192(%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm3, 384(%rax)
-; AVX512-NEXT:    addq $632, %rsp # imm = 0x278
+; AVX512-NEXT:    vmovdqa64 %zmm27, 384(%rax)
+; AVX512-NEXT:    addq $680, %rsp # imm = 0x2A8
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i16_stride7_vf32:
 ; AVX512-FCP:       # %bb.0:
-; AVX512-FCP-NEXT:    subq $248, %rsp
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm1
+; AVX512-FCP-NEXT:    subq $264, %rsp # imm = 0x108
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm2
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm10
-; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa %ymm2, %ymm10
+; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm4
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm4, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm4, %ymm3
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm18
 ; AVX512-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpor %ymm2, %ymm3, %ymm2
-; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm4
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm11
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm4, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm13
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm4, %ymm3
 ; AVX512-FCP-NEXT:    vmovdqa %ymm4, %ymm9
-; AVX512-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vmovdqu %ymm4, (%rsp) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm6
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm5 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
 ; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm6, %ymm4
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm6, %ymm16
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm6, %ymm17
 ; AVX512-FCP-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm13
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm13, %ymm6
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm14
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm14, %ymm7
-; AVX512-FCP-NEXT:    vporq %ymm6, %ymm7, %ymm25
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm11, %ymm6
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm8
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm8, %ymm5
-; AVX512-FCP-NEXT:    vpor %ymm6, %ymm5, %ymm3
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm3
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm29
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm12
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm12, %ymm1
+; AVX512-FCP-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm15
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm15, %ymm4
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm11
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
+; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm11, %ymm8
+; AVX512-FCP-NEXT:    vpor %ymm4, %ymm8, %ymm3
+; AVX512-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm13, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm12
+; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm12, %ymm5
+; AVX512-FCP-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm8
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm8, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm14
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm14, %ymm1
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm15
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm15, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm6
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm6, %ymm1
-; AVX512-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm21
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm20
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm2
+; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm16
+; AVX512-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm22
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm11, %ymm1
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm8[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm13, %ymm1
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm12[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8,9,10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm11[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm11, %ymm30
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3],ymm2[4],ymm4[5,6,7,8],ymm2[9],ymm4[10,11],ymm2[12],ymm4[13,14,15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm13[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm12[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm5[0],ymm2[1],ymm5[2,3],ymm2[4],ymm5[5,6,7,8],ymm2[9],ymm5[10,11],ymm2[12],ymm5[13,14,15]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [0,2,2,3,10,0,11,0]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm1, %zmm2, %zmm24
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm10[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm18[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm10
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm5
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [0,2,2,3,8,0,9,0]
-; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm23
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm2
+; AVX512-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm25 = [0,2,2,3,8,0,9,0]
+; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm25
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm9, %ymm0
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm16[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm17[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm2
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm4, %xmm4
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,1,3,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermi2q %zmm4, %zmm0, %zmm20
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm14[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm0[0,1],ymm4[2],ymm0[3,4],ymm4[5],ymm0[6,7,8,9],ymm4[10],ymm0[11,12],ymm4[13],ymm0[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm22 = [2,2,2,3,8,8,8,9]
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm3
-; AVX512-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; AVX512-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm7
-; AVX512-FCP-NEXT:    vpermi2q %zmm7, %zmm4, %zmm22
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-FCP-NEXT:    vprold $16, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm10
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa %xmm3, %xmm4
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [2,1,3,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm21
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7,8,9],ymm2[10],ymm0[11,12],ymm2[13],ymm0[14,15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [2,2,2,3,8,8,8,9]
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm2
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm7
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm6
+; AVX512-FCP-NEXT:    vpermi2q %zmm6, %zmm0, %zmm23
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm10[4],xmm1[4],xmm10[5],xmm1[5],xmm10[6],xmm1[6],xmm10[7],xmm1[7]
+; AVX512-FCP-NEXT:    vprold $16, %xmm10, %xmm6
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm4, %xmm2
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2],xmm1[3,4],xmm6[5],xmm1[6,7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm10 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
+; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm0
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm26 = [0,0,0,1,8,8,9,0]
-; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm26
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm2
+; AVX512-FCP-NEXT:    vpermi2q %zmm0, %zmm1, %zmm26
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm1
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm29 = [0,1,1,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermi2q %zmm0, %zmm1, %zmm29
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm2
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm0
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm2[u,u,4,5,u,u,u,u,6,7,u,u,u,u,8,9]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6],xmm1[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm27, %xmm4
+; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [0,0,1,0,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermi2q %zmm1, %zmm3, %zmm27
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm6
 ; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm9
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [0,0,0,1,8,9,9,11]
-; AVX512-FCP-NEXT:    vpermi2q %zmm4, %zmm0, %zmm27
-; AVX512-FCP-NEXT:    vprold $16, %ymm13, %ymm0
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm14[1,2,2,3,5,6,6,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0,1],ymm0[2],ymm4[3,4],ymm0[5],ymm4[6,7,8,9],ymm0[10],ymm4[11,12],ymm0[13],ymm4[14,15]
-; AVX512-FCP-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm13, %ymm4
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm13 = ymm14[0,0,2,1,4,4,6,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm13[3],ymm4[4,5],ymm13[6],ymm4[7,8,9,10],ymm13[11],ymm4[12,13],ymm13[14],ymm4[15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [2,2,3,3,10,9,11,10]
-; AVX512-FCP-NEXT:    vpermi2q %zmm0, %zmm4, %zmm28
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm14
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm14[4],xmm0[4],xmm14[5],xmm0[5],xmm14[6],xmm0[6],xmm14[7],xmm0[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm4, %xmm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm19
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm25, %zmm0, %zmm4
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm25 = [0,0,1,1,12,13,0,15]
-; AVX512-FCP-NEXT:    vpermi2q %zmm4, %zmm2, %zmm25
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm7
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,0,0,1,8,9,9,11]
+; AVX512-FCP-NEXT:    vpermi2q %zmm7, %zmm3, %zmm28
+; AVX512-FCP-NEXT:    vprold $16, %ymm15, %ymm3
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm11[1,2,2,3,5,6,6,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm3[2],ymm5[3,4],ymm3[5],ymm5[6,7,8,9],ymm3[10],ymm5[11,12],ymm3[13],ymm5[14,15]
+; AVX512-FCP-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm15, %ymm5
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm11[0,0,2,1,4,4,6,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1,2],ymm7[3],ymm5[4,5],ymm7[6],ymm5[7,8,9,10],ymm7[11],ymm5[12,13],ymm7[14],ymm5[15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm31 = [2,2,3,3,10,9,11,10]
+; AVX512-FCP-NEXT:    vpermi2q %zmm3, %zmm5, %zmm31
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm19
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm3
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm18
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT:    vpbroadcastd 8(%rax), %ymm2
-; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm9
+; AVX512-FCP-NEXT:    vpbroadcastd 8(%rax), %ymm0
+; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm10
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm9, %ymm4
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm31
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm30, %ymm2
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[1,1,1,1,5,5,5,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm18
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm29, %ymm7
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm7[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm12[0,1,1,3,4,5,5,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0,1],ymm2[2],ymm4[3,4],ymm2[5],ymm4[6,7,8,9],ymm2[10],ymm4[11,12],ymm2[13],ymm4[14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm17
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm15, %ymm2
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm6[0,0,2,1,4,4,6,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8,9,10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm29 = [0,1,0,3,10,10,11,11]
-; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm21, %zmm29
-; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm8
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm8, %ymm1
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [0,1,4,5,4,5,5,7,12,13,10,10,14,14,14,14]
-; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm21, %ymm2
-; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm16
-; AVX512-FCP-NEXT:    vmovdqa %ymm7, %ymm4
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm7[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm12[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13,14,15]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
-; AVX512-FCP-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm4, %ymm13
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm12[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7,8,9],ymm12[10],ymm13[11,12],ymm12[13],ymm13[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [2,2,2,3,8,10,10,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm30, %zmm12
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm13 = ymm6[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm13[2],ymm3[3,4],ymm13[5],ymm3[6,7,8,9],ymm13[10],ymm3[11,12],ymm13[13],ymm3[14,15]
-; AVX512-FCP-NEXT:    vprold $16, %ymm15, %ymm13
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[1,2,2,3,5,6,6,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm13[2],ymm6[3,4],ymm13[5],ymm6[6,7,8,9],ymm13[10],ymm6[11,12],ymm13[13],ymm6[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm13 = [2,1,3,2,10,10,10,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm13, %zmm6
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm10[4],xmm5[5],xmm10[5],xmm5[6],xmm10[6],xmm5[7],xmm10[7]
-; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm15 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0],xmm10[1],xmm5[2,3],xmm10[4],xmm5[5,6],xmm10[7]
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [0,0,1,1,8,8,10,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm2, %zmm5
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm3 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm3 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm1[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512-FCP-NEXT:    vmovdqa %xmm11, %xmm1
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,1,1,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm4, %zmm10
-; AVX512-FCP-NEXT:    vprold $16, %xmm14, %xmm3
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm0[1,1,2,3]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm11[0,1],xmm3[2],xmm11[3,4],xmm3[5],xmm11[6,7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm11
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm2, %zmm11
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm0
-; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm0, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm3
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm14 = xmm3[1,1,2,2]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm14[0],xmm2[1],xmm14[2,3],xmm2[4],xmm14[5,6],xmm2[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm24, %zmm2, %zmm12
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm6
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm12, %ymm3
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7,8,9],ymm4[10],ymm3[11,12],ymm4[13],ymm3[14,15]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm12[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm1[0,1,1,3,4,5,5,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm7[0,1],ymm4[2],ymm7[3,4],ymm4[5],ymm7[6,7,8,9],ymm4[10],ymm7[11,12],ymm4[13],ymm7[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm13, %zmm4
-; AVX512-FCP-NEXT:    vmovdqa64 (%rax), %zmm3
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm10, %ymm2
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm30
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm12[1,1,1,1,5,5,5,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm6[2],ymm0[3,4],ymm6[5],ymm0[6,7,8,9],ymm6[10],ymm0[11,12],ymm6[13],ymm0[14,15]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm17
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm8[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm14[0,1,1,3,4,5,5,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm11 = ymm6[0,1],ymm0[2],ymm6[3,4],ymm0[5],ymm6[6,7,8,9],ymm0[10],ymm6[11,12],ymm0[13],ymm6[14,15]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm13
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm13, %ymm0
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm16[0,0,2,1,4,4,6,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7,8,9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm9 = [0,1,0,3,10,10,11,11]
+; AVX512-FCP-NEXT:    vpermi2q %zmm0, %zmm22, %zmm9
+; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm2
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm22 = [0,1,4,5,4,5,5,7,12,13,10,10,14,14,14,14]
+; AVX512-FCP-NEXT:    vpermd %ymm2, %ymm22, %ymm4
+; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm1
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm8[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm14[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm12[0],ymm4[1],ymm12[2,3],ymm4[4],ymm12[5,6,7,8],ymm4[9],ymm12[10,11],ymm4[12],ymm12[13,14,15]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
+; AVX512-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm8, %ymm12
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm14[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm8[2],ymm12[3,4],ymm8[5],ymm12[6,7,8,9],ymm8[10],ymm12[11,12],ymm8[13],ymm12[14,15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,2,2,3,8,10,10,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm20, %zmm12
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm16[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm8[2],ymm4[3,4],ymm8[5],ymm4[6,7,8,9],ymm8[10],ymm4[11,12],ymm8[13],ymm4[14,15]
+; AVX512-FCP-NEXT:    vprold $16, %ymm13, %ymm8
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm15 = ymm16[1,2,2,3,5,6,6,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm15 = ymm15[0,1],ymm8[2],ymm15[3,4],ymm8[5],ymm15[6,7,8,9],ymm8[10],ymm15[11,12],ymm8[13],ymm15[14,15]
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm0[u,u,4,5,u,u,u,u,6,7,u,u,u,u,8,9]
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[1,1,2,2]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm8 = xmm14[0],xmm8[1],xmm14[2,3],xmm8[4],xmm14[5,6],xmm8[7]
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm14 = [0,0,1,1,8,8,10,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm14, %zmm8
+; AVX512-FCP-NEXT:    vprold $16, %xmm3, %xmm0
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm5[1,1,2,3]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm13[0,1],xmm0[2],xmm13[3,4],xmm0[5],xmm13[6,7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm7, %xmm0
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpermt2q %zmm13, %zmm14, %zmm3
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,1,3,2,10,10,10,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm5, %zmm15
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm7, %ymm4
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm13[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm6[2],ymm4[3,4],ymm6[5],ymm4[6,7,8,9],ymm6[10],ymm4[11,12],ymm6[13],ymm4[14,15]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm7[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm13[0,1,1,3,4,5,5,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7,8,9],ymm6[10],ymm7[11,12],ymm6[13],ymm7[14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm5, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm24, %zmm16, %zmm12
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm15
+; AVX512-FCP-NEXT:    vmovdqa64 (%rax), %zmm5
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [4,5,2,2,6,6,6,6,30,31,27,27,31,31,30,31]
-; AVX512-FCP-NEXT:    vpermi2d %zmm3, %zmm8, %zmm7
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm7
-; AVX512-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm6
-; AVX512-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm8
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm6
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm6
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm26, %zmm8, %zmm5
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm0
-; AVX512-FCP-NEXT:    vpbroadcastd (%rax), %ymm5
-; AVX512-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm10
+; AVX512-FCP-NEXT:    vpermi2d %zmm5, %zmm2, %zmm7
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm7
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm26, %zmm2, %zmm8
+; AVX512-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm12
+; AVX512-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm13
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm29, %zmm12
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm12
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm27
+; AVX512-FCP-NEXT:    vmovdqu (%rsp), %ymm4 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm4[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm24[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm8[0],ymm3[1],ymm8[2,3],ymm3[4],ymm8[5,6,7,8],ymm3[9],ymm8[10,11],ymm3[12],ymm8[13,14,15]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm8
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,2,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,2,1]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm18[0,0,1,3]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,1]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm17[2,2,2,3]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,1,3,2]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm24[1,1,1,1,5,5,5,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm15[0,1],ymm4[2],ymm15[3,4],ymm4[5],ymm15[6,7,8,9],ymm4[10],ymm15[11,12],ymm4[13],ymm15[14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm4
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm4
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [6,0,0,0,7,0,0,7]
+; AVX512-FCP-NEXT:    vpermd %ymm10, %ymm3, %ymm3
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm5, %zmm5
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm14[0,2,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm19[0,0,1,3]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm18[2,2,2,3]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm17[2,1,3,2]
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm5
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm5
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm14 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm14[0],ymm0[1],ymm14[2,3],ymm0[4],ymm14[5,6,7,8],ymm0[9],ymm14[10,11],ymm0[12],ymm14[13,14,15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm14 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm15 = ymm1[1,1,1,1,5,5,5,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm14[0,1],ymm15[2],ymm14[3,4],ymm15[5],ymm14[6,7,8,9],ymm15[10],ymm14[11,12],ymm15[13],ymm14[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm30, %zmm14
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm14
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [6,0,0,0,7,0,0,7]
-; AVX512-FCP-NEXT:    vpermd %ymm9, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm3
-; AVX512-FCP-NEXT:    vpermd %zmm3, %zmm21, %zmm3
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm3
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm3
-; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm4 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm9 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm8, %zmm9
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm23, %zmm2, %zmm20
-; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm25, %zmm31
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm31
+; AVX512-FCP-NEXT:    vpermd %zmm5, %zmm22, %zmm5
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm31, %zmm5
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm5
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm4 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm6 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm2, %zmm6
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm25, %zmm16, %zmm21
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm30, %zmm0
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm2, %zmm2
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm2, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm4, %zmm4
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm4, %zmm4
 ; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm4
-; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm29, %zmm16
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm16
+; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm1
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1
 ; AVX512-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm2
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm0
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm0
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm23, %zmm2
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm21, %zmm2
+; AVX512-FCP-NEXT:    vpbroadcastd (%rax), %ymm3
+; AVX512-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm4
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm3
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm3
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 128(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, 320(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 256(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 192(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 128(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 320(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, 256(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 192(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, 384(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm31, 64(%rax)
-; AVX512-FCP-NEXT:    addq $248, %rsp
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512-FCP-NEXT:    addq $264, %rsp # imm = 0x108
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i16_stride7_vf32:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    subq $632, %rsp # imm = 0x278
+; AVX512DQ-NEXT:    subq $680, %rsp # imm = 0x2A8
 ; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm1
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm1, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm16
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm2
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm29
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
+; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm1, %ymm0
+; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm27
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm8
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm14 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm8, %ymm1
 ; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm1
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm9
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm1, %ymm0
-; AVX512DQ-NEXT:    vmovdqa %ymm1, %ymm15
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm2
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm14 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
-; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm17
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm9, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm11
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm15 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
+; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm11, %ymm1
 ; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm10
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm10, %ymm2
-; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm11
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm11, %ymm3
+; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
+; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm16
+; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm4
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vmovdqa64 %ymm4, %ymm17
 ; AVX512DQ-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm3
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm6
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm10[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm2[2],ymm4[3,4],ymm2[5],ymm4[6,7,8,9],ymm2[10],ymm4[11,12],ymm2[13],ymm4[14,15]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [4,5,4,5,4,5,6,7,16,17,16,17,16,17,17,19]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm2[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpermi2d %zmm5, %zmm4, %zmm7
-; AVX512DQ-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm8
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,6]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm2
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3]
+; AVX512DQ-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,6]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [16,18,19,19,19,19,0,0,0,1,0,1,2,3,2,3]
-; AVX512DQ-NEXT:    vpermi2d %zmm2, %zmm3, %zmm4
+; AVX512DQ-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm6
-; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm6, %ymm2
-; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm3, %ymm4
-; AVX512DQ-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm2
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm10
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm10, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm2, %ymm4
+; AVX512DQ-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm12
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm7
-; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm12, %ymm2
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm7, %ymm4
-; AVX512DQ-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm15
+; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm12, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm15, %ymm4
+; AVX512DQ-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm13
 ; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm14
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm13, %ymm1
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
-; AVX512DQ-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm13, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm14, %ymm1
+; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm20
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,2]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,16,0,0,17,17,0,0,0,0,0,1,2,0,0,3]
-; AVX512DQ-NEXT:    vpermi2d %zmm2, %zmm1, %zmm4
-; AVX512DQ-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm4[0,1,2,3,4,5,7,6]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm9 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
-; AVX512DQ-NEXT:    vpermi2d %zmm5, %zmm4, %zmm9
-; AVX512DQ-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vprold $16, %ymm10, %ymm4
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm11[1,2,2,3,5,6,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm11[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm10[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm9[0,1,2],ymm5[3],ymm9[4,5],ymm5[6],ymm9[7,8,9,10],ymm5[11],ymm9[12,13],ymm5[14],ymm9[15]
-; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [2,2,3,3,10,9,11,10]
-; AVX512DQ-NEXT:    vpermi2q %zmm4, %zmm5, %zmm27
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm3
+; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm4
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,2,2]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3],xmm4[4],xmm5[5,6],xmm4[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm26 = [0,16,0,0,17,17,0,0,0,0,0,1,2,0,0,3]
+; AVX512DQ-NEXT:    vpermi2d %zmm5, %zmm4, %zmm26
+; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm5
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm6[0,1,2,3,4,5,7,6]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm25 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
+; AVX512DQ-NEXT:    vpermi2d %zmm7, %zmm6, %zmm25
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm31
-; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm4
-; AVX512DQ-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-NEXT:    vpbroadcastd 8(%rax), %ymm0
 ; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm4, %ymm4
-; AVX512DQ-NEXT:    vmovdqa64 %ymm5, %ymm28
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm3
+; AVX512DQ-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm7 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm3, %ymm6
+; AVX512DQ-NEXT:    vmovdqa64 %ymm7, %ymm21
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm7[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm30
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm12[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm7[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm23
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm6[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm11 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm6[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm19
-; AVX512DQ-NEXT:    vprold $16, %ymm13, %ymm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm14[1,2,2,3,5,6,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm13[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm14[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm4 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm15[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm28
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm12[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm15[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6,7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13,14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm29
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm10[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6,7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13,14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm30
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm10[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1,2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8,9,10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm31
+; AVX512DQ-NEXT:    vprold $16, %ymm13, %ymm4
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm14[1,2,2,3,5,6,6,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
+; AVX512DQ-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm13[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm14[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
+; AVX512DQ-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovdqa64 %ymm27, %ymm3
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm3[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm8[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6,7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13,14,15]
+; AVX512DQ-NEXT:    vmovdqu %ymm4, (%rsp) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm9[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm11[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8,9,10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
+; AVX512DQ-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm16, %ymm4
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm4[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm29[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm5, %ymm18
-; AVX512DQ-NEXT:    vmovdqa %ymm15, %ymm8
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm15[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm17[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm0
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm9
-; AVX512DQ-NEXT:    vprold $16, %xmm9, %xmm1
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm15 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm15[0,1],xmm1[2],xmm15[3,4],xmm1[5],xmm15[6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm17[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm4, %ymm19
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm4
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm5
+; AVX512DQ-NEXT:    vprold $16, %xmm5, %xmm6
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,3]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm7[0,1],xmm6[2],xmm7[3,4],xmm6[5],xmm7[6,7]
+; AVX512DQ-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512DQ-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm5
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[1,1,2,2]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm7[0],xmm1[1],xmm7[2,3],xmm1[4],xmm7[5,6],xmm1[7]
 ; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
-; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm9, %xmm0
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm15
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm10 = xmm15[1,1,2,2]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm10[0],xmm0[1],xmm10[2,3],xmm0[4],xmm10[5,6],xmm0[7]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm26
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm15[4],xmm9[4],xmm15[5],xmm9[5],xmm15[6],xmm9[6],xmm15[7],xmm9[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm9, %xmm25
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,1,1,5,5,5,5]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm6[0,1],ymm3[2],ymm6[3,4],ymm3[5],ymm6[6,7,8,9],ymm3[10],ymm6[11,12],ymm3[13],ymm6[14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm24
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[0,1,1,3,4,5,5,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm12[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm22
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm14[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm13[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm6[0,1,2],ymm3[3],ymm6[4,5],ymm3[6],ymm6[7,8,9,10],ymm3[11],ymm6[12,13],ymm3[14],ymm6[15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm21
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,5,0,0,0,6,0,0,6,0,0,0,7,0,0,7]
-; AVX512DQ-NEXT:    vmovdqa 32(%rax), %ymm6
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm3, %zmm3
-; AVX512DQ-NEXT:    vmovdqa64 %ymm28, %ymm7
-; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm6, %ymm10
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,1,1,3,4,5,5,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3]
-; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm6
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm10, %zmm6
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm2, %xmm10
-; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm13
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm14
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
-; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm20
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
-; AVX512DQ-NEXT:    vprold $16, %xmm14, %xmm14
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[1,1,2,3]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm13[0,1],xmm14[2],xmm13[3,4],xmm14[5],xmm13[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm16
-; AVX512DQ-NEXT:    vmovdqa %ymm4, %ymm2
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm14 = ymm4[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm14 = ymm14[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm29, %ymm12
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm29[0,1,1,3,4,5,5,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm14 = ymm9[0,1],ymm14[2],ymm9[3,4],ymm14[5],ymm9[6,7,8,9],ymm14[10],ymm9[11,12],ymm14[13],ymm9[14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm4
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm4[0,2,3,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[2,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,5,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,0,1,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm28 = ymm30[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm29 = ymm23[0,2,2,3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm18
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512DQ-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[1,1,1,1,5,5,5,5]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm10[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,0,2,1,4,4,6,5]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7,8,9],ymm1[10],ymm2[11,12],ymm1[13],ymm2[14,15]
+; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm15[0,1,1,3,4,5,5,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm12[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,0,0,0,4,4,4,4]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm14[0,0,2,1,4,4,6,5]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm13[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,0,0,0,4,4,4,4]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8,9,10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
+; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,5,0,0,0,6,0,0,6,0,0,0,7,0,0,7]
+; AVX512DQ-NEXT:    vmovdqa 32(%rax), %ymm2
+; AVX512DQ-NEXT:    vpermd %zmm2, %zmm1, %zmm27
+; AVX512DQ-NEXT:    vmovdqa64 %ymm21, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,1,1,3,4,5,5,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm20
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm14
+; AVX512DQ-NEXT:    vmovdqa %xmm1, %xmm4
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm1
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm2
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm21
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-NEXT:    vprold $16, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm24
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm3[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm8[0,1,1,3,4,5,5,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7,8,9],ymm1[10],ymm2[11,12],ymm1[13],ymm2[14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm23
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm8[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm22
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm9[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,2,1,4,4,6,5]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[1,1,1,1,5,5,5,5]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm13 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm11[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm9[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm11 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6,7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm16, %ymm4
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm4[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm17[0,0,2,1,4,4,6,5]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm8 = ymm1[0,1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7,8,9,10],ymm2[11],ymm1[12,13],ymm2[14],ymm1[15]
+; AVX512DQ-NEXT:    vprold $16, %ymm16, %ymm1
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm17[1,2,2,3,5,6,6,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm7 = ymm10[0,1],ymm1[2],ymm10[3,4],ymm1[5],ymm10[6,7,8,9],ymm1[10],ymm10[11,12],ymm1[13],ymm10[14,15]
+; AVX512DQ-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm10 = mem[0,2,3,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,5,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,0,1,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[0,0,1,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm28 = ymm28[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm29 = ymm29[0,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm30 = ymm30[0,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm31 = ymm31[2,1,3,3]
+; AVX512DQ-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm6 = mem[2,1,3,2]
+; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm5 = mem[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq $232, (%rsp), %ymm1 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm1 = mem[0,2,2,3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm0
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm0[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[0,0,1,1]
+; AVX512DQ-NEXT:    vpermq $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm0 = mem[2,1,3,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm14[0,0,1,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm19[2,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw $180, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm15 = mem[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[0,1,1,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm29, %zmm28, %zmm28
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm29
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm28, %zmm30, %zmm29
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,1,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm9
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm1, %zmm30, %zmm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm1 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm0, %zmm2, %zmm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm0 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm0
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0
+; AVX512DQ-NEXT:    vpshufd $254, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm1 = mem[2,3,3,3,6,7,7,7]
+; AVX512DQ-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm3 = mem[0,0,2,1]
+; AVX512DQ-NEXT:    vpshuflw $230, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm10 = mem[2,1,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,5,5,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,1,3]
+; AVX512DQ-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm12 = mem[0,0,1,1]
+; AVX512DQ-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm14 = mem[0,2,3,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,2,1]
+; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm17 = mem[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm18 = mem[2,1,3,2]
+; AVX512DQ-NEXT:    vpermq $250, {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm19 = mem[2,2,3,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm21 = ymm21[0,0,1,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm28 = ymm24[0,0,2,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm30 = ymm23[2,1,3,2]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm31 = ymm22[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,2,2,3]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm19[2,1,3,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm30 = ymm18[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm31 = ymm5[2,1,3,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm12[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm8[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm17[1,1,1,1,5,5,5,5]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm17[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm12 = ymm8[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm12[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6,7,8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14,15]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm29, %zmm28, %zmm12
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm11, %zmm7
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq $184, %zmm12, %zmm11, %zmm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm30, %zmm0
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm31, %zmm1
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm0, %zmm11, %zmm1
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm0 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm15, %zmm9 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm0, %zmm11, %zmm9
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm10[0,1,2,3],zmm0[4,5,6,7]
-; AVX512DQ-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm10 = mem[2,1,3,2]
-; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm12 = mem[2,2,2,3]
-; AVX512DQ-NEXT:    vpshufd $254, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm15 = mem[2,3,3,3,6,7,7,7]
-; AVX512DQ-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm17 = mem[0,0,2,1]
-; AVX512DQ-NEXT:    vpshuflw $230, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm8 = mem[2,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,5,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,1,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm18 = ymm26[0,0,1,1]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm13
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm13[0,2,3,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,2,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm19 = ymm24[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm28 = ymm22[2,1,3,2]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm29 = ymm21[2,2,3,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm30 = ymm20[0,0,1,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm31 = ymm16[0,0,2,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,1,3,2]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3]
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm16, %zmm0
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm0
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm9
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm9
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm3
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm15[2,1,3,2]
-; AVX512DQ-NEXT:    vpbroadcastd 32(%rax), %ymm9
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm7, %zmm7
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm7
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm7
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm17, %zmm1
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm18, %zmm8
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm1, %zmm11, %zmm8
-; AVX512DQ-NEXT:    vpbroadcastd 36(%rax), %ymm1
-; AVX512DQ-NEXT:    vpbroadcastd 40(%rax), %ymm9
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm1
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm1
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm19, %zmm8, %zmm8
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm28, %zmm9, %zmm9
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm9
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],zmm29[0,1,2,3]
-; AVX512DQ-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm6
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm6
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm8
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm8
-; AVX512DQ-NEXT:    vpbroadcastd (%rax), %ymm9
-; AVX512DQ-NEXT:    vpbroadcastd 4(%rax), %ymm10
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm9
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm9
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm14, %zmm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,3,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,2]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm29, %zmm5
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm27
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm15[0,0,0,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm4
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
-; AVX512DQ-NEXT:    vpermd (%rax), %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,2]
+; AVX512DQ-NEXT:    vpbroadcastd 32(%rax), %ymm5
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm14, %zmm12, %zmm4
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm3, %zmm2, %zmm4
+; AVX512DQ-NEXT:    vpbroadcastd 36(%rax), %ymm2
+; AVX512DQ-NEXT:    vpbroadcastd 40(%rax), %ymm3
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm2
 ; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm2
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm17, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm18, %zmm4, %zmm4
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm4
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm19[0,1,2,3]
+; AVX512DQ-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm20
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm20
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm28, %zmm21, %zmm3
+; AVX512DQ-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm26, %zmm3
+; AVX512DQ-NEXT:    vpbroadcastd (%rax), %ymm4
+; AVX512DQ-NEXT:    vpbroadcastd 4(%rax), %ymm5
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm25, %zmm4
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm4
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm3
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm5
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm3
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
+; AVX512DQ-NEXT:    vpermd (%rax), %zmm6, %zmm6
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm6
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, 128(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm9, (%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 320(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 256(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm7, 192(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 128(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm20, 320(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, 256(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 192(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm3, 384(%rax)
-; AVX512DQ-NEXT:    addq $632, %rsp # imm = 0x278
+; AVX512DQ-NEXT:    vmovdqa64 %zmm27, 384(%rax)
+; AVX512DQ-NEXT:    addq $680, %rsp # imm = 0x2A8
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i16_stride7_vf32:
 ; AVX512DQ-FCP:       # %bb.0:
-; AVX512DQ-FCP-NEXT:    subq $248, %rsp
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm1
+; AVX512DQ-FCP-NEXT:    subq $264, %rsp # imm = 0x108
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm10
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm4
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm4, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm4, %ymm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm4, %ymm18
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpor %ymm2, %ymm3, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm4
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm11
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm4, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm13
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm4, %ymm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm4, %ymm9
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm4, (%rsp) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm5 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm6, %ymm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm6, %ymm16
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm6, %ymm17
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm13
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm13, %ymm6
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm14
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm14, %ymm7
-; AVX512DQ-FCP-NEXT:    vporq %ymm6, %ymm7, %ymm25
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm11, %ymm6
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm8
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm8, %ymm5
-; AVX512DQ-FCP-NEXT:    vpor %ymm6, %ymm5, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm29
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm12
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm12, %ymm1
+; AVX512DQ-FCP-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm15
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm15, %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm11
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm11, %ymm8
+; AVX512DQ-FCP-NEXT:    vpor %ymm4, %ymm8, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm13, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm12, %ymm5
+; AVX512DQ-FCP-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm8
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm14
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm14, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm15
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm15, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm6, %ymm1
-; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm21
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm20
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm16
+; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm22
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm11, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm8[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm13, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm12[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8,9,10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm11[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm11, %ymm30
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3],ymm2[4],ymm4[5,6,7,8],ymm2[9],ymm4[10,11],ymm2[12],ymm4[13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm13[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm12[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm5[0],ymm2[1],ymm5[2,3],ymm2[4],ymm5[5,6,7,8],ymm2[9],ymm5[10,11],ymm2[12],ymm5[13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [0,2,2,3,10,0,11,0]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm1, %zmm2, %zmm24
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm10[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm18[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm5
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [0,2,2,3,8,0,9,0]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm23
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm25 = [0,2,2,3,8,0,9,0]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm25
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm9, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm16[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm17[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm4, %xmm4
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,1,3,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm4, %zmm0, %zmm20
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm14[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm0[0,1],ymm4[2],ymm0[3,4],ymm4[5],ymm0[6,7,8,9],ymm4[10],ymm0[11,12],ymm4[13],ymm0[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm22 = [2,2,2,3,8,8,8,9]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm7
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm7, %zmm4, %zmm22
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vprold $16, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm10
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, %xmm4
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [2,1,3,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm21
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7,8,9],ymm2[10],ymm0[11,12],ymm2[13],ymm0[14,15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [2,2,2,3,8,8,8,9]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm7
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm6
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm6, %zmm0, %zmm23
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm10[4],xmm1[4],xmm10[5],xmm1[5],xmm10[6],xmm1[6],xmm10[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vprold $16, %xmm10, %xmm6
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm4, %xmm2
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2],xmm1[3,4],xmm6[5],xmm1[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm10 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm0
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm26 = [0,0,0,1,8,8,9,0]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm26
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm2
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm0, %zmm1, %zmm26
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm1
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm29 = [0,1,1,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm0, %zmm1, %zmm29
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm0
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm2[u,u,4,5,u,u,u,u,6,7,u,u,u,u,8,9]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm27, %xmm4
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [0,0,1,0,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm1, %zmm3, %zmm27
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm9
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [0,0,0,1,8,9,9,11]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm4, %zmm0, %zmm27
-; AVX512DQ-FCP-NEXT:    vprold $16, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm14[1,2,2,3,5,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0,1],ymm0[2],ymm4[3,4],ymm0[5],ymm4[6,7,8,9],ymm0[10],ymm4[11,12],ymm0[13],ymm4[14,15]
-; AVX512DQ-FCP-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm13, %ymm4
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm13 = ymm14[0,0,2,1,4,4,6,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm13[3],ymm4[4,5],ymm13[6],ymm4[7,8,9,10],ymm13[11],ymm4[12,13],ymm13[14],ymm4[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [2,2,3,3,10,9,11,10]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm0, %zmm4, %zmm28
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm14
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm14[4],xmm0[4],xmm14[5],xmm0[5],xmm14[6],xmm0[6],xmm14[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm4, %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm19
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm25, %zmm0, %zmm4
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm25 = [0,0,1,1,12,13,0,15]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm4, %zmm2, %zmm25
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm7
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,0,0,1,8,9,9,11]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm7, %zmm3, %zmm28
+; AVX512DQ-FCP-NEXT:    vprold $16, %ymm15, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm11[1,2,2,3,5,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm3[2],ymm5[3,4],ymm3[5],ymm5[6,7,8,9],ymm3[10],ymm5[11,12],ymm3[13],ymm5[14,15]
+; AVX512DQ-FCP-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm15, %ymm5
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm11[0,0,2,1,4,4,6,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1,2],ymm7[3],ymm5[4,5],ymm7[6],ymm5[7,8,9,10],ymm7[11],ymm5[12,13],ymm7[14],ymm5[15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm31 = [2,2,3,3,10,9,11,10]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm3, %zmm5, %zmm31
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm19
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm3
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm18
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 8(%rax), %ymm2
-; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm9
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 8(%rax), %ymm0
+; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm10
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm9, %ymm4
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm31
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[1,1,1,1,5,5,5,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm18
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm7
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm7[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm12[0,1,1,3,4,5,5,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0,1],ymm2[2],ymm4[3,4],ymm2[5],ymm4[6,7,8,9],ymm2[10],ymm4[11,12],ymm2[13],ymm4[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm17
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm15, %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm6[0,0,2,1,4,4,6,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8,9,10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm29 = [0,1,0,3,10,10,11,11]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm21, %zmm29
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm8
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm8, %ymm1
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [0,1,4,5,4,5,5,7,12,13,10,10,14,14,14,14]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm21, %ymm2
-; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm16
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm7, %ymm4
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm7[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm12[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13,14,15]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
-; AVX512DQ-FCP-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm4, %ymm13
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm12[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7,8,9],ymm12[10],ymm13[11,12],ymm12[13],ymm13[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [2,2,2,3,8,10,10,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm30, %zmm12
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm13 = ymm6[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm13[2],ymm3[3,4],ymm13[5],ymm3[6,7,8,9],ymm13[10],ymm3[11,12],ymm13[13],ymm3[14,15]
-; AVX512DQ-FCP-NEXT:    vprold $16, %ymm15, %ymm13
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[1,2,2,3,5,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm13[2],ymm6[3,4],ymm13[5],ymm6[6,7,8,9],ymm13[10],ymm6[11,12],ymm13[13],ymm6[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm13 = [2,1,3,2,10,10,10,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm13, %zmm6
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm10[4],xmm5[5],xmm10[5],xmm5[6],xmm10[6],xmm5[7],xmm10[7]
-; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm15 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0],xmm10[1],xmm5[2,3],xmm10[4],xmm5[5,6],xmm10[7]
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [0,0,1,1,8,8,10,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm2, %zmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm3 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm3 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm1[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm11, %xmm1
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,1,1,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm4, %zmm10
-; AVX512DQ-FCP-NEXT:    vprold $16, %xmm14, %xmm3
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm0[1,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm11[0,1],xmm3[2],xmm11[3,4],xmm3[5],xmm11[6,7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm11
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm2, %zmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm0, %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm3
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm14 = xmm3[1,1,2,2]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm14[0],xmm2[1],xmm14[2,3],xmm2[4],xmm14[5,6],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm24, %zmm2, %zmm12
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm6
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm12, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7,8,9],ymm4[10],ymm3[11,12],ymm4[13],ymm3[14,15]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm12[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm1[0,1,1,3,4,5,5,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm7[0,1],ymm4[2],ymm7[3,4],ymm4[5],ymm7[6,7,8,9],ymm4[10],ymm7[11,12],ymm4[13],ymm7[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm13, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rax), %zmm3
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm10, %ymm2
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm30
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm12[1,1,1,1,5,5,5,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm6[2],ymm0[3,4],ymm6[5],ymm0[6,7,8,9],ymm6[10],ymm0[11,12],ymm6[13],ymm0[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm17
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm8[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm14[0,1,1,3,4,5,5,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm11 = ymm6[0,1],ymm0[2],ymm6[3,4],ymm0[5],ymm6[6,7,8,9],ymm0[10],ymm6[11,12],ymm0[13],ymm6[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm13
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm13, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm16[0,0,2,1,4,4,6,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7,8,9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm9 = [0,1,0,3,10,10,11,11]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm0, %zmm22, %zmm9
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm22 = [0,1,4,5,4,5,5,7,12,13,10,10,14,14,14,14]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm2, %ymm22, %ymm4
+; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm1
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm8[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm14[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm12[0],ymm4[1],ymm12[2,3],ymm4[4],ymm12[5,6,7,8],ymm4[9],ymm12[10,11],ymm4[12],ymm12[13,14,15]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
+; AVX512DQ-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm8, %ymm12
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm14[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm8[2],ymm12[3,4],ymm8[5],ymm12[6,7,8,9],ymm8[10],ymm12[11,12],ymm8[13],ymm12[14,15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,2,2,3,8,10,10,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm20, %zmm12
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm16[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm8[2],ymm4[3,4],ymm8[5],ymm4[6,7,8,9],ymm8[10],ymm4[11,12],ymm8[13],ymm4[14,15]
+; AVX512DQ-FCP-NEXT:    vprold $16, %ymm13, %ymm8
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm15 = ymm16[1,2,2,3,5,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm15 = ymm15[0,1],ymm8[2],ymm15[3,4],ymm8[5],ymm15[6,7,8,9],ymm8[10],ymm15[11,12],ymm8[13],ymm15[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm0[u,u,4,5,u,u,u,u,6,7,u,u,u,u,8,9]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[1,1,2,2]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm8 = xmm14[0],xmm8[1],xmm14[2,3],xmm8[4],xmm14[5,6],xmm8[7]
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm14 = [0,0,1,1,8,8,10,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm14, %zmm8
+; AVX512DQ-FCP-NEXT:    vprold $16, %xmm3, %xmm0
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm5[1,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm13[0,1],xmm0[2],xmm13[3,4],xmm0[5],xmm13[6,7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm7, %xmm0
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm13, %zmm14, %zmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,1,3,2,10,10,10,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm5, %zmm15
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm7, %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm13[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm6[2],ymm4[3,4],ymm6[5],ymm4[6,7,8,9],ymm6[10],ymm4[11,12],ymm6[13],ymm4[14,15]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm7[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm13[0,1,1,3,4,5,5,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7,8,9],ymm6[10],ymm7[11,12],ymm6[13],ymm7[14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm5, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm24, %zmm16, %zmm12
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rax), %zmm5
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [4,5,2,2,6,6,6,6,30,31,27,27,31,31,30,31]
-; AVX512DQ-FCP-NEXT:    vpermi2d %zmm3, %zmm8, %zmm7
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm7
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm6
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm8
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm26, %zmm8, %zmm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm0
-; AVX512DQ-FCP-NEXT:    vpbroadcastd (%rax), %ymm5
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm10
+; AVX512DQ-FCP-NEXT:    vpermi2d %zmm5, %zmm2, %zmm7
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm26, %zmm2, %zmm8
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm12
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm13
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm29, %zmm12
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm12
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm27
+; AVX512DQ-FCP-NEXT:    vmovdqu (%rsp), %ymm4 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm4[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm24[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm8[0],ymm3[1],ymm8[2,3],ymm3[4],ymm8[5,6,7,8],ymm3[9],ymm8[10,11],ymm3[12],ymm8[13,14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm19, %xmm8
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,2,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,2,1]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm18[0,0,1,3]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,1]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm17[2,2,2,3]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,1,3,2]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm24[1,1,1,1,5,5,5,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm15[0,1],ymm4[2],ymm15[3,4],ymm4[5],ymm15[6,7,8,9],ymm4[10],ymm15[11,12],ymm4[13],ymm15[14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm4
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm4
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [6,0,0,0,7,0,0,7]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm10, %ymm3, %ymm3
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm5, %zmm5
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm14[0,2,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm19[0,0,1,3]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm18[2,2,2,3]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm17[2,1,3,2]
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm5
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm14 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm14[0],ymm0[1],ymm14[2,3],ymm0[4],ymm14[5,6,7,8],ymm0[9],ymm14[10,11],ymm0[12],ymm14[13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm14 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm15 = ymm1[1,1,1,1,5,5,5,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm14[0,1],ymm15[2],ymm14[3,4],ymm15[5],ymm14[6,7,8,9],ymm15[10],ymm14[11,12],ymm15[13],ymm14[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm30, %zmm14
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm14
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [6,0,0,0,7,0,0,7]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm9, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm3
-; AVX512DQ-FCP-NEXT:    vpermd %zmm3, %zmm21, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm3
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm4 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm9 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm8, %zmm9
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm23, %zmm2, %zmm20
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm25, %zmm31
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm31
+; AVX512DQ-FCP-NEXT:    vpermd %zmm5, %zmm22, %zmm5
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm31, %zmm5
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm5
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm4 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm6 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm2, %zmm6
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm25, %zmm16, %zmm21
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm30, %zmm0
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm2, %zmm2
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm2, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm4, %zmm4
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm4, %zmm4
 ; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm29, %zmm16
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm16
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm2
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm0
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm23, %zmm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm21, %zmm2
+; AVX512DQ-FCP-NEXT:    vpbroadcastd (%rax), %ymm3
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm4
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm3
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm3
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, 320(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 256(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 192(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 128(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 320(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, 256(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 192(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 384(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm31, 64(%rax)
-; AVX512DQ-FCP-NEXT:    addq $248, %rsp
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512DQ-FCP-NEXT:    addq $264, %rsp # imm = 0x108
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
@@ -12619,2682 +12639,2656 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512-LABEL: store_i16_stride7_vf64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    subq $2168, %rsp # imm = 0x878
-; AVX512-NEXT:    vmovdqa 96(%rcx), %ymm2
-; AVX512-NEXT:    vmovdqa 96(%rdx), %ymm9
+; AVX512-NEXT:    subq $2808, %rsp # imm = 0xAF8
+; AVX512-NEXT:    vmovdqa 96(%rcx), %ymm6
+; AVX512-NEXT:    vmovdqa 96(%rdx), %ymm13
 ; AVX512-NEXT:    vmovdqa 96(%rdi), %ymm7
 ; AVX512-NEXT:    vmovdqa 96(%rsi), %ymm8
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpshufb %ymm0, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm18
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vpshufb %ymm11, %ymm9, %ymm2
-; AVX512-NEXT:    vporq %ymm1, %ymm2, %ymm16
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpshufb %ymm12, %ymm8, %ymm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm13 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
-; AVX512-NEXT:    vpshufb %ymm13, %ymm7, %ymm2
-; AVX512-NEXT:    vporq %ymm1, %ymm2, %ymm17
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
+; AVX512-NEXT:    vpshufb %ymm0, %ymm6, %ymm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vpshufb %ymm1, %ymm13, %ymm3
+; AVX512-NEXT:    vporq %ymm2, %ymm3, %ymm17
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512-NEXT:    vpshufb %ymm10, %ymm8, %ymm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
+; AVX512-NEXT:    vpshufb %ymm11, %ymm7, %ymm3
+; AVX512-NEXT:    vporq %ymm2, %ymm3, %ymm18
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
 ; AVX512-NEXT:    vmovdqa 64(%r9), %ymm2
-; AVX512-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqa %ymm3, %ymm10
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm21
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufb %ymm12, %ymm2, %ymm2
 ; AVX512-NEXT:    vmovdqa 64(%r8), %ymm3
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm15 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
-; AVX512-NEXT:    vpshufb %ymm15, %ymm3, %ymm2
-; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm20
-; AVX512-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa 64(%rcx), %ymm2
-; AVX512-NEXT:    vpshufb %ymm0, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm23
-; AVX512-NEXT:    vmovdqa 64(%rdx), %ymm6
-; AVX512-NEXT:    vpshufb %ymm11, %ymm6, %ymm2
-; AVX512-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa 64(%rsi), %ymm5
-; AVX512-NEXT:    vpshufb %ymm12, %ymm5, %ymm1
+; AVX512-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
+; AVX512-NEXT:    vpshufb %ymm9, %ymm3, %ymm3
+; AVX512-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqa 64(%rcx), %ymm3
+; AVX512-NEXT:    vpshufb %ymm0, %ymm3, %ymm2
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm26
+; AVX512-NEXT:    vmovdqa 64(%rdx), %ymm4
+; AVX512-NEXT:    vpshufb %ymm1, %ymm4, %ymm3
+; AVX512-NEXT:    vmovdqa64 %ymm4, %ymm27
+; AVX512-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqa 64(%rsi), %ymm3
+; AVX512-NEXT:    vpshufb %ymm10, %ymm3, %ymm2
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm23
 ; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm4
-; AVX512-NEXT:    vpshufb %ymm13, %ymm4, %ymm2
-; AVX512-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufb %ymm10, %ymm1, %ymm1
-; AVX512-NEXT:    vmovdqa %ymm10, %ymm3
-; AVX512-NEXT:    vmovdqa (%r8), %ymm2
+; AVX512-NEXT:    vpshufb %ymm11, %ymm4, %ymm3
+; AVX512-NEXT:    vmovdqa64 %ymm4, %ymm22
+; AVX512-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufb %ymm15, %ymm2, %ymm2
-; AVX512-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa (%rcx), %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm1
-; AVX512-NEXT:    vmovdqa (%rdx), %ymm2
+; AVX512-NEXT:    vmovdqa (%r9), %ymm2
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufb %ymm12, %ymm2, %ymm2
+; AVX512-NEXT:    vmovdqa (%r8), %ymm3
+; AVX512-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufb %ymm9, %ymm3, %ymm3
+; AVX512-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqa (%rcx), %ymm2
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufb %ymm0, %ymm2, %ymm2
+; AVX512-NEXT:    vmovdqa (%rdx), %ymm3
+; AVX512-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
+; AVX512-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufb %ymm11, %ymm2, %ymm2
-; AVX512-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-NEXT:    vmovdqa (%rsi), %ymm2
-; AVX512-NEXT:    vpshufb %ymm12, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512-NEXT:    vmovdqa (%rdi), %ymm10
-; AVX512-NEXT:    vpshufb %ymm13, %ymm10, %ymm2
-; AVX512-NEXT:    vmovdqa64 %ymm10, %ymm22
-; AVX512-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm2
-; AVX512-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm1
-; AVX512-NEXT:    vpshufb %ymm11, %ymm1, %ymm11
-; AVX512-NEXT:    vpor %ymm0, %ymm11, %ymm0
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufb %ymm10, %ymm2, %ymm2
+; AVX512-NEXT:    vmovdqa (%rdi), %ymm3
+; AVX512-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm4
+; AVX512-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm2
+; AVX512-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm0
-; AVX512-NEXT:    vpshufb %ymm12, %ymm0, %ymm12
-; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm11
-; AVX512-NEXT:    vpshufb %ymm13, %ymm11, %ymm13
-; AVX512-NEXT:    vpor %ymm12, %ymm13, %ymm10
-; AVX512-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%r8), %ymm13
-; AVX512-NEXT:    vpshufb %ymm15, %ymm13, %ymm12
-; AVX512-NEXT:    vmovdqa 32(%r9), %ymm15
-; AVX512-NEXT:    vpshufb %ymm3, %ymm15, %ymm14
-; AVX512-NEXT:    vpor %ymm14, %ymm12, %ymm10
-; AVX512-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vprold $16, %ymm15, %ymm12
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm14 = ymm13[1,2,2,3,5,6,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm12 = ymm14[0,1],ymm12[2],ymm14[3,4],ymm12[5],ymm14[6,7,8,9],ymm12[10],ymm14[11,12],ymm12[13],ymm14[14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm14 = ymm15[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm14 = ymm14[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm5
+; AVX512-NEXT:    vpshufb %ymm10, %ymm5, %ymm0
+; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm3
+; AVX512-NEXT:    vpshufb %ymm11, %ymm3, %ymm1
+; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 32(%r8), %ymm1
+; AVX512-NEXT:    vpshufb %ymm9, %ymm1, %ymm9
+; AVX512-NEXT:    vmovdqa 32(%r9), %ymm0
+; AVX512-NEXT:    vpshufb %ymm12, %ymm0, %ymm10
+; AVX512-NEXT:    vpor %ymm10, %ymm9, %ymm9
+; AVX512-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
+; AVX512-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm10, %ymm6, %ymm9
+; AVX512-NEXT:    vmovdqa64 %ymm10, %ymm31
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm13[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm9 = ymm9[0,1],ymm10[2],ymm9[3,4],ymm10[5],ymm9[6,7,8,9],ymm10[10],ymm9[11,12],ymm10[13],ymm9[14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm9 = ymm6[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[2,2,2,2,6,6,6,6]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm13[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm14[0,1],ymm10[2],ymm14[3,4],ymm10[5],ymm14[6,7,8,9],ymm10[10],ymm14[11,12],ymm10[13],ymm14[14,15]
-; AVX512-NEXT:    vpmovsxbq {{.*#+}} zmm14 = [2,1,3,2,10,10,10,11]
-; AVX512-NEXT:    vpermi2q %zmm10, %zmm12, %zmm14
-; AVX512-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
-; AVX512-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512-NEXT:    vmovdqa64 %ymm18, %ymm14
-; AVX512-NEXT:    vpshufb %ymm12, %ymm14, %ymm10
-; AVX512-NEXT:    vmovdqa64 %ymm12, %ymm19
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm9[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7,8,9],ymm12[10],ymm10[11,12],ymm12[13],ymm10[14,15]
-; AVX512-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm14[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm10[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm9[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0],ymm10[1],ymm12[2,3],ymm10[4],ymm12[5,6,7,8],ymm10[9],ymm12[10,11],ymm10[12],ymm12[13,14,15]
-; AVX512-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm8[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm10[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm7[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0],ymm10[1],ymm12[2,3],ymm10[4],ymm12[5,6,7,8],ymm10[9],ymm12[10,11],ymm10[12],ymm12[13,14,15]
-; AVX512-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm8[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm10[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm7[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0,1,2],ymm10[3],ymm12[4,5],ymm10[6],ymm12[7,8,9,10],ymm10[11],ymm12[12,13],ymm10[14],ymm12[15]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5,6,7,8],ymm9[9],ymm10[10,11],ymm9[12],ymm10[13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,2,3]
+; AVX512-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm9 = ymm8[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm7[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5,6,7,8],ymm9[9],ymm10[10,11],ymm9[12],ymm10[13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,2,3]
+; AVX512-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm9 = ymm8[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm7[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3],ymm10[4,5],ymm9[6],ymm10[7,8,9,10],ymm9[11],ymm10[12,13],ymm9[14],ymm10[15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,1,3,3]
+; AVX512-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqa 96(%r9), %ymm9
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm9[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[3,3,3,3]
 ; AVX512-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[0,1,1,3,4,5,5,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm14[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm10[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm9 = ymm9[0,1],ymm10[2],ymm9[3,4],ymm10[5],ymm9[6,7,8,9],ymm10[10],ymm9[11,12],ymm10[13],ymm9[14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,1,3,2]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm9[2,1,2,3,6,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm10[0,0,3,3,4,5,6,7,8,8,11,11,12,13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm30 = ymm10[2,2,2,2]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm13[0,1,1,3,4,5,5,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,0,0,0,4,4,4,4]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm10[0,1],ymm6[2],ymm10[3,4],ymm6[5],ymm10[6,7,8,9],ymm6[10],ymm10[11,12],ymm6[13],ymm10[14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,1,3,2]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[1,1,1,1,5,5,5,5]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm8[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm8 = ymm8[0,0,2,1,4,4,6,5]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm7 = ymm8[0,1],ymm7[2],ymm8[3,4],ymm7[5],ymm8[6,7,8,9],ymm7[10],ymm8[11,12],ymm7[13],ymm8[14,15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm16, %zmm8
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm17, %zmm7
-; AVX512-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm7
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm26 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vmovdqa 96(%r8), %ymm10
+; AVX512-NEXT:    vprold $16, %ymm9, %ymm8
+; AVX512-NEXT:    vpshufb %ymm12, %ymm9, %ymm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm17, %zmm6
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm18, %zmm7
+; AVX512-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm7
+; AVX512-NEXT:    vmovdqa 96(%r8), %ymm6
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm6[1,2,2,3,5,6,6,7]
 ; AVX512-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = ymm10[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm10[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm10[16,17,u,u,u,u],zero,zero
-; AVX512-NEXT:    vpternlogq $248, %ymm26, %ymm7, %ymm8
-; AVX512-NEXT:    vmovdqa 96(%r9), %ymm12
-; AVX512-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufb %ymm3, %ymm12, %ymm9
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq $248, %ymm14, %ymm8, %ymm9
-; AVX512-NEXT:    vextracti64x4 $1, %zmm7, %ymm7
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm8 = ymm10[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,1,3,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm6[0,0,2,1,4,4,6,5]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[16,17,u,u,u,u],zero,zero
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogq $248, %ymm11, %ymm7, %ymm6
+; AVX512-NEXT:    vmovdqa64 %zmm11, %zmm12
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogq $248, %ymm11, %ymm6, %ymm9
+; AVX512-NEXT:    vextracti64x4 $1, %zmm7, %ymm6
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm10[2,1,3,3]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogq $184, %ymm7, %ymm10, %ymm8
-; AVX512-NEXT:    vprold $16, %ymm12, %ymm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,2]
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq $184, %ymm8, %ymm14, %ymm7
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm7
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm9[0,1,2,3],zmm7[4,5,6,7]
-; AVX512-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpternlogq $184, %ymm6, %ymm10, %ymm7
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm8[2,2,2,2]
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogq $184, %ymm7, %ymm8, %ymm6
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm6
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm9[0,1,2,3],zmm6[4,5,6,7]
+; AVX512-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,5,0,0,0,6,0,0,6,0,0,0,7,0,0,7]
-; AVX512-NEXT:    vmovdqa 96(%rax), %ymm7
-; AVX512-NEXT:    vpermd %zmm7, %zmm18, %zmm9
-; AVX512-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm7[0,1,1,3,4,5,5,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm10 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
-; AVX512-NEXT:    vpshufb %ymm10, %ymm7, %ymm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,2,3]
-; AVX512-NEXT:    vpandnq %ymm9, %ymm26, %ymm9
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm7, %zmm7
+; AVX512-NEXT:    vmovdqa 96(%rax), %ymm6
+; AVX512-NEXT:    vpermd %zmm6, %zmm18, %zmm7
 ; AVX512-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512-NEXT:    vpbroadcastd 72(%rax), %ymm7
-; AVX512-NEXT:    vpandnq %ymm7, %ymm16, %ymm9
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm6[0,1,1,3,4,5,5,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
+; AVX512-NEXT:    vpshufb %ymm11, %ymm6, %ymm6
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,2,2,3]
+; AVX512-NEXT:    vpandn %ymm7, %ymm12, %ymm7
+; AVX512-NEXT:    vmovdqa64 %zmm12, %zmm19
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
+; AVX512-NEXT:    vpbroadcastd 72(%rax), %ymm6
+; AVX512-NEXT:    vpandnq %ymm6, %ymm28, %ymm6
 ; AVX512-NEXT:    vmovdqa 64(%rax), %ymm7
-; AVX512-NEXT:    vpshufb %ymm10, %ymm7, %ymm12
-; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm9, %zmm9
-; AVX512-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpbroadcastd 8(%rax), %ymm9
-; AVX512-NEXT:    vpandnq %ymm9, %ymm16, %ymm9
-; AVX512-NEXT:    vmovdqa (%rax), %ymm8
-; AVX512-NEXT:    vpshufb %ymm10, %ymm8, %ymm12
-; AVX512-NEXT:    vmovdqa64 %ymm8, %ymm24
-; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm9, %zmm9
-; AVX512-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm2[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[0,1,1,3,4,5,5,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm9 = ymm12[0,1],ymm9[2],ymm12[3,4],ymm9[5],ymm12[6,7,8,9],ymm9[10],ymm12[11,12],ymm9[13],ymm12[14,15]
-; AVX512-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm0[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm11[1,1,1,1,5,5,5,5]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm9 = ymm9[0,1],ymm12[2],ymm9[3,4],ymm12[5],ymm9[6,7,8,9],ymm12[10],ymm9[11,12],ymm12[13],ymm9[14,15]
-; AVX512-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%rax), %ymm9
-; AVX512-NEXT:    vpshufb %ymm10, %ymm9, %ymm10
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm9[0,1,1,3,4,5,5,7]
+; AVX512-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufb %ymm11, %ymm7, %ymm7
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 64(%r9), %xmm7
+; AVX512-NEXT:    vmovdqa 64(%r8), %xmm8
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm17
+; AVX512-NEXT:    vmovdqa64 %xmm7, %xmm20
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512-NEXT:    vpshufb %xmm7, %xmm6, %xmm6
+; AVX512-NEXT:    vmovdqa %xmm7, %xmm15
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,1,1]
+; AVX512-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 64(%rcx), %xmm9
+; AVX512-NEXT:    vmovdqa 64(%rdx), %xmm7
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm7[4],xmm9[4],xmm7[5],xmm9[5],xmm7[6],xmm9[6],xmm7[7],xmm9[7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,2,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,2,1]
+; AVX512-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm14
+; AVX512-NEXT:    vmovdqa 64(%rsi), %xmm10
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm10[4],xmm14[4],xmm10[5],xmm14[5],xmm10[6],xmm14[6],xmm10[7],xmm14[7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[2,1,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,5,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,1,3]
+; AVX512-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpbroadcastd 8(%rax), %ymm8
+; AVX512-NEXT:    vpandnq %ymm8, %ymm28, %ymm8
+; AVX512-NEXT:    vmovdqa (%rax), %ymm12
+; AVX512-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufb %ymm11, %ymm12, %ymm13
+; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm8, %zmm8
+; AVX512-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa (%r9), %xmm6
+; AVX512-NEXT:    vmovdqa (%r8), %xmm12
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm12[4],xmm6[4],xmm12[5],xmm6[5],xmm12[6],xmm6[6],xmm12[7],xmm6[7]
+; AVX512-NEXT:    vmovdqa64 %xmm12, %xmm29
+; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm24
+; AVX512-NEXT:    vpshufb %xmm15, %xmm8, %xmm8
+; AVX512-NEXT:    vmovdqa64 %xmm15, %xmm25
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,1,1]
+; AVX512-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm8
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm13
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm13[4],xmm8[4],xmm13[5],xmm8[5],xmm13[6],xmm8[6],xmm13[7],xmm8[7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[0,2,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm15[0,0,2,1]
+; AVX512-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm6
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm12
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm12[4],xmm6[4],xmm12[5],xmm6[5],xmm12[6],xmm6[6],xmm12[7],xmm6[7]
+; AVX512-NEXT:    vmovdqa64 %xmm12, %xmm21
+; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm16
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[2,1,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,5,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm15[0,0,1,3]
+; AVX512-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm15 = ymm4[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm15 = ymm15[0,0,0,0,4,4,4,4]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm2[0,1,1,3,4,5,5,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm15[2],ymm12[3,4],ymm15[5],ymm12[6,7,8,9],ymm15[10],ymm12[11,12],ymm15[13],ymm12[14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,1,3,2]
+; AVX512-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm12 = ymm5[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm12[0,0,2,1,4,4,6,5]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm15 = ymm3[1,1,1,1,5,5,5,5]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm15[2],ymm12[3,4],ymm15[5],ymm12[6,7,8,9],ymm15[10],ymm12[11,12],ymm15[13],ymm12[14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
+; AVX512-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqa 32(%rax), %ymm15
+; AVX512-NEXT:    vpshufb %ymm11, %ymm15, %ymm11
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm15[0,1,1,3,4,5,5,7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,2,3]
-; AVX512-NEXT:    vpandnq %ymm12, %ymm26, %ymm12
-; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
-; AVX512-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm13[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm12 = ymm15[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm12[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm8 = ymm12[0,1,2],ymm10[3],ymm12[4,5],ymm10[6],ymm12[7,8,9,10],ymm10[11],ymm12[12,13],ymm10[14],ymm12[15]
-; AVX512-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm13
-; AVX512-NEXT:    vpshufb %ymm13, %ymm2, %ymm10
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm8 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7,8,9],ymm12[10],ymm10[11,12],ymm12[13],ymm10[14,15]
-; AVX512-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpandnq %ymm12, %ymm19, %ymm12
+; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
+; AVX512-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm11 = ymm0[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm11 = ymm11[0,0,0,0,4,4,4,4]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[0,0,2,1,4,4,6,5]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm11 = ymm11[0,1,2],ymm12[3],ymm11[4,5],ymm12[6],ymm11[7,8,9,10],ymm12[11],ymm11[12,13],ymm12[14],ymm11[15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,3,3]
+; AVX512-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa64 %ymm31, %ymm6
+; AVX512-NEXT:    vpshufb %ymm6, %ymm4, %ymm11
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm11 = ymm11[0,1],ymm12[2],ymm11[3,4],ymm12[5],ymm11[6,7,8,9],ymm12[10],ymm11[11,12],ymm12[13],ymm11[14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]
+; AVX512-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2,3],ymm4[4],ymm2[5,6,7,8],ymm4[9],ymm2[10,11],ymm4[12],ymm2[13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm5[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6,7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14,15]
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm0[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm31
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm11[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3],ymm2[4],ymm4[5,6,7,8],ymm2[9],ymm4[10,11],ymm2[12],ymm4[13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm3[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm5[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8,9,10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,3]
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vprold $16, %ymm0, %ymm2
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,2,2,3,5,6,6,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7,8,9],ymm2[10],ymm3[11,12],ymm2[13],ymm3[14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,2]
+; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
 ; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpermd %zmm9, %zmm18, %zmm0
+; AVX512-NEXT:    vpermd %zmm15, %zmm18, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm5[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
+; AVX512-NEXT:    vmovdqa64 %ymm23, %ymm12
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm12[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm4[1,1,1,1,5,5,5,5]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm22[1,1,1,1,5,5,5,5]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
-; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm5[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm12[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm22[2,2,2,2,6,6,6,6]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
 ; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa64 %ymm23, %ymm2
+; AVX512-NEXT:    vmovdqa64 %ymm26, %ymm2
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm2[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm6[0,1,1,3,4,5,5,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm27[0,1,1,3,4,5,5,7]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm30
-; AVX512-NEXT:    vpshufb %ymm13, %ymm2, %ymm0
-; AVX512-NEXT:    vmovdqa64 %ymm23, %ymm3
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm6[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,2]
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshufb %ymm6, %ymm2, %ymm0
+; AVX512-NEXT:    vmovdqa64 %ymm26, %ymm3
+; AVX512-NEXT:    vmovdqa64 %ymm31, %ymm11
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm27[2,2,2,2,6,6,6,6]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
 ; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm8
-; AVX512-NEXT:    vprold $16, %ymm21, %ymm0
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm20[1,2,2,3,5,6,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm8[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm20[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7,8,9,10],ymm2[11],ymm1[12,13],ymm2[14],ymm1[15]
-; AVX512-NEXT:    vpmovsxbq {{.*#+}} zmm10 = [2,2,3,3,10,9,11,10]
-; AVX512-NEXT:    vpermt2q %zmm0, %zmm10, %zmm1
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm11 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
-; AVX512-NEXT:    vpermd 64(%rax), %zmm11, %zmm0
-; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm26, %zmm0
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm5[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[0,0,2,1,4,4,6,5]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7,8,9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,3,3]
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm6[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vprold $16, %ymm5, %ymm0
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[1,2,2,3,5,6,6,7]
+; AVX512-NEXT:    vmovdqa %ymm2, %ymm6
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,2]
+; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm27[3,3,3,3,7,7,7,7]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm3[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6,7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14,15]
-; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm4[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm5[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7,8,9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
-; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm20[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm8[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm12 = [4,5,4,5,4,5,6,7,16,17,16,17,16,17,17,19]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 96(%rcx), %xmm0
+; AVX512-NEXT:    vmovdqa 96(%rdx), %xmm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm22[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm12[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8,9,10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,3]
+; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 96(%rsi), %xmm2
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm3
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm12
+; AVX512-NEXT:    vpshufb %xmm12, %xmm4, %xmm4
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,1,1]
+; AVX512-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm6[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
+; AVX512-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vprold $16, %xmm2, %xmm4
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,2,3]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]
+; AVX512-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} xmm3 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm2
+; AVX512-NEXT:    vmovdqa %xmm3, %xmm4
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,2]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2,3],xmm2[4],xmm3[5,6],xmm2[7]
+; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vmovdqa 96(%r9), %xmm0
 ; AVX512-NEXT:    vmovdqa 96(%r8), %xmm1
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm2[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpermt2d %zmm4, %zmm12, %zmm3
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm7[2,3,3,3,6,7,7,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,1,3,2]
-; AVX512-NEXT:    vpbroadcastd 96(%rax), %ymm5
-; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogd $184, %zmm3, %zmm5, %zmm4
-; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm15
-; AVX512-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 96(%rsi), %xmm3
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm4
-; AVX512-NEXT:    vprold $16, %xmm3, %xmm5
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,3]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2],xmm6[3,4],xmm5[5],xmm6[6,7]
-; AVX512-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm23
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; AVX512-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa 96(%rcx), %xmm3
-; AVX512-NEXT:    vmovdqa 96(%rdx), %xmm4
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} xmm6 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm5
-; AVX512-NEXT:    vmovdqa %xmm6, %xmm7
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,2]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3],xmm5[4],xmm6[5,6],xmm5[7]
-; AVX512-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,6]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [16,18,19,19,19,19,0,0,0,1,0,1,2,3,2,3]
-; AVX512-NEXT:    vpermt2d %zmm2, %zmm29, %zmm1
-; AVX512-NEXT:    vpbroadcastd 100(%rax), %ymm2
-; AVX512-NEXT:    vpbroadcastd 104(%rax), %ymm3
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm14, %zmm2
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 64(%rcx), %xmm1
-; AVX512-NEXT:    vmovdqa 64(%rdx), %xmm2
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm27
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vpshufb %xmm7, %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa %xmm7, %xmm8
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,2]
+; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-NEXT:    vpshufb %xmm12, %xmm0, %xmm0
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [16,18,19,19,19,19,0,0,0,1,0,1,2,3,2,3]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5,7,6]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm18, %zmm0
+; AVX512-NEXT:    vpbroadcastd 100(%rax), %ymm1
+; AVX512-NEXT:    vpbroadcastd 104(%rax), %ymm2
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm31
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm31
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
+; AVX512-NEXT:    vpshufb %xmm4, %xmm9, %xmm1
+; AVX512-NEXT:    vmovdqa %xmm4, %xmm6
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm7[1,1,2,2]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm3[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [16,16,17,17,17,17,0,0,0,1,0,1,2,3,2,3]
-; AVX512-NEXT:    vpermt2d %zmm2, %zmm3, %zmm1
-; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm2
-; AVX512-NEXT:    vmovdqa 64(%rsi), %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
-; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,1,1]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
-; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm25
-; AVX512-NEXT:    vprold $16, %xmm4, %xmm4
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2],xmm2[3,4],xmm4[5],xmm2[6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm5, %zmm2
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
-; AVX512-NEXT:    vpternlogq $226, %zmm1, %zmm4, %zmm2
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 64(%r9), %xmm1
-; AVX512-NEXT:    vmovdqa 64(%r8), %xmm2
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm17
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,5,7,6]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
-; AVX512-NEXT:    vpermt2d %zmm2, %zmm6, %zmm1
-; AVX512-NEXT:    vpbroadcastd 64(%rax), %ymm2
-; AVX512-NEXT:    vpbroadcastd 68(%rax), %ymm5
-; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm2, %zmm2
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm9, %zmm2
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm1
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm5
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm20
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512-NEXT:    vpshufb %xmm8, %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa %xmm8, %xmm14
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3],xmm1[4],xmm5[5,6],xmm1[7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm7[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpermt2d %zmm5, %zmm3, %zmm1
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm3
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,1,1]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm19
-; AVX512-NEXT:    vprold $16, %xmm5, %xmm5
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm5[2],xmm3[3,4],xmm5[5],xmm3[6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [16,16,17,17,17,17,0,0,0,1,0,1,2,3,2,3]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm2, %zmm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
+; AVX512-NEXT:    vpshufb %xmm12, %xmm0, %xmm0
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,1]
+; AVX512-NEXT:    vprold $16, %xmm10, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm14[1,1,2,3]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm7, %zmm2
-; AVX512-NEXT:    vpternlogq $226, %zmm1, %zmm4, %zmm2
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512-NEXT:    vmovdqa (%r8), %xmm3
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm14
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
+; AVX512-NEXT:    vpternlogq $226, %zmm1, %zmm0, %zmm14
+; AVX512-NEXT:    vmovdqa64 %xmm17, %xmm1
+; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm3
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,5,7,6]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpermt2d %zmm3, %zmm6, %zmm1
-; AVX512-NEXT:    vpbroadcastd (%rax), %ymm3
-; AVX512-NEXT:    vpbroadcastd 4(%rax), %ymm4
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm2
-; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm9, %zmm2
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa64 %ymm28, %ymm7
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm7[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vmovdqa64 %ymm22, %ymm2
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm22[1,1,1,1,5,5,5,5]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm22
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm7[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm2[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vmovdqa %ymm2, %ymm9
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm4[0],ymm1[1],ymm4[2,3],ymm1[4],ymm4[5,6,7,8],ymm1[9],ymm4[10,11],ymm1[12],ymm4[13,14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm21
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
+; AVX512-NEXT:    vpermt2d %zmm3, %zmm4, %zmm1
+; AVX512-NEXT:    vpbroadcastd 64(%rax), %ymm3
+; AVX512-NEXT:    vpbroadcastd 68(%rax), %ymm5
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm25
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm19, %zmm25
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm8, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm13[1,1,2,2]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0],xmm3[1],xmm5[2,3],xmm3[4],xmm5[5,6],xmm3[7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm2, %zmm3
+; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512-NEXT:    vpshufb %xmm12, %xmm1, %xmm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,1]
+; AVX512-NEXT:    vprold $16, %xmm21, %xmm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm16[1,1,2,3]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2],xmm5[3,4],xmm2[5],xmm5[6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm5
+; AVX512-NEXT:    vpternlogq $226, %zmm3, %zmm0, %zmm5
+; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5,7,6]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm4, %zmm0
+; AVX512-NEXT:    vpbroadcastd (%rax), %ymm1
+; AVX512-NEXT:    vpbroadcastd 4(%rax), %ymm2
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm20
+; AVX512-NEXT:    vpternlogd $184, %zmm0, %zmm19, %zmm20
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm6[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5]
 ; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512-NEXT:    vprold $16, %ymm2, %ymm1
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[1,1,1,1,5,5,5,5]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm6[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vmovdqa %ymm2, %ymm10
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm2[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,2,2,3,5,6,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm4[0,1],ymm1[2],ymm4[3,4],ymm1[5],ymm4[6,7,8,9],ymm1[10],ymm4[11,12],ymm1[13],ymm4[14,15]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm2[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm3[0,0,2,1,4,4,6,5]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm6[3],ymm4[4,5],ymm6[6],ymm4[7,8,9,10],ymm6[11],ymm4[12,13],ymm6[14],ymm4[15]
-; AVX512-NEXT:    vpermt2q %zmm1, %zmm10, %zmm4
-; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm8[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
-; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm10[0,1,1,3,4,5,5,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm6[0,1],ymm1[2],ymm6[3,4],ymm1[5],ymm6[6,7,8,9],ymm1[10],ymm6[11,12],ymm1[13],ymm6[14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm16
-; AVX512-NEXT:    vpshufb %ymm13, %ymm8, %ymm1
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm10[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm6[2],ymm1[3,4],ymm6[5],ymm1[6,7,8,9],ymm6[10],ymm1[11,12],ymm6[13],ymm1[14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm18
-; AVX512-NEXT:    vpermd (%rax), %zmm11, %zmm1
-; AVX512-NEXT:    vpternlogd $184, %zmm4, %zmm26, %zmm1
-; AVX512-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm10[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm8[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm4[1],ymm1[2,3],ymm4[4],ymm1[5,6,7,8],ymm4[9],ymm1[10,11],ymm4[12],ymm1[13,14,15]
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm9[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm7[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1,2],ymm4[3],ymm1[4,5],ymm4[6],ymm1[7,8,9,10],ymm4[11],ymm1[12,13],ymm4[14],ymm1[15]
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[3,3,3,3,7,7,7,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm2[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm4[0,1],ymm1[2],ymm4[3,4],ymm1[5],ymm4[6,7,8,9],ymm1[10],ymm4[11,12],ymm1[13],ymm4[14,15]
-; AVX512-NEXT:    vmovdqa 32(%r9), %xmm7
-; AVX512-NEXT:    vmovdqa 32(%r8), %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm6[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpermt2d %zmm11, %zmm12, %zmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm11 = ymm24[2,3,3,3,6,7,7,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,1,3,2]
-; AVX512-NEXT:    vpbroadcastd 32(%rax), %ymm12
-; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm26
-; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm15, %zmm26
-; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm1
-; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm12
-; AVX512-NEXT:    vprold $16, %xmm12, %xmm15
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm1[1,1,2,3]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm13[0,1],xmm15[2],xmm13[3,4],xmm15[5],xmm13[6,7]
-; AVX512-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm1[4],xmm12[5],xmm1[5],xmm12[6],xmm1[6],xmm12[7],xmm1[7]
-; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]
-; AVX512-NEXT:    vmovdqa64 %xmm17, %xmm1
-; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm1
-; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm1
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm24 = mem[2,2,2,3]
-; AVX512-NEXT:    vpermq $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm23 = mem[0,2,2,3]
-; AVX512-NEXT:    vpermq $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm7 = mem[0,2,2,3]
-; AVX512-NEXT:    vpermq $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm17 = mem[2,1,3,3]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm6[0,1,2,3,4,5,7,6]
-; AVX512-NEXT:    vpermt2d %zmm0, %zmm29, %zmm4
-; AVX512-NEXT:    vpbroadcastd 36(%rax), %ymm0
-; AVX512-NEXT:    vpbroadcastd 40(%rax), %ymm6
-; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm28
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm28
-; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm6
-; AVX512-NEXT:    vpshufb %xmm14, %xmm6, %xmm4
-; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm0[1,1,2,2]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm12[0],xmm4[1],xmm12[2,3],xmm4[4],xmm12[5,6],xmm4[7]
-; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
-; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
-; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm1[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm29 = ymm0[3,3,3,3]
-; AVX512-NEXT:    vpshufd $233, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm0 = mem[1,2,2,3,5,6,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm14 = ymm1[2,1,2,3,6,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm14 = ymm14[0,0,3,3,4,5,6,7,8,8,11,11,12,13,14,15]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,2,2,2]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,1,1]
-; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm1
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm1[0,2,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm1
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm1[2,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,5,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,1,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,1]
-; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm1
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm1[0,2,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,0,2,1]
-; AVX512-NEXT:    vmovdqa64 %xmm19, %xmm1
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm1[2,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,5,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,1,3]
-; AVX512-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm4 = mem[2,1,3,2]
-; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm5 = mem[2,2,2,3]
-; AVX512-NEXT:    vpermq $250, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm6 = mem[2,2,3,3]
-; AVX512-NEXT:    vpermpd $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm1 = mem[2,2,2,3]
-; AVX512-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpermpd $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm1 = mem[0,2,2,3]
-; AVX512-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vpermq {{.*#+}} ymm31 = ymm31[0,2,2,3]
-; AVX512-NEXT:    vpermq $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm27 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm27 = mem[2,1,3,3]
-; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm20 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm20 = mem[2,2,2,3]
-; AVX512-NEXT:    vpermq $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm11 = mem[0,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm19 = ymm30[2,1,3,2]
-; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm1 = mem[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm25 = ymm22[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm21[0,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm16 = ymm16[2,1,3,2]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm18[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm23, %zmm24, %zmm23
-; AVX512-NEXT:    vinserti64x4 $1, %ymm17, %zmm7, %zmm24
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq $184, %zmm23, %zmm30, %zmm24
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[0,1,1,3,4,5,5,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,2]
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogq $184, %ymm24, %ymm7, %ymm0
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512-NEXT:    vpternlogq $184, %ymm0, %ymm18, %ymm14
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm2 # 64-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm3 # 64-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 32-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 32-byte Folded Reload
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq $226, %zmm10, %zmm23, %zmm8
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm17, %zmm8
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm15, %zmm2 # 32-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm10 # 32-byte Folded Reload
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm23, %zmm10
-; AVX512-NEXT:    vpternlogq $226, %zmm3, %zmm17, %zmm10
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshufb %ymm11, %ymm2, %ymm0
+; AVX512-NEXT:    vmovdqa %ymm2, %ymm9
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vmovdqa %ymm3, %ymm11
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm7[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm4[0,0,2,1,4,4,6,5]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm0[0,1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7,8,9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
+; AVX512-NEXT:    vprold $16, %ymm7, %ymm0
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm4[1,2,2,3,5,6,6,7]
+; AVX512-NEXT:    vmovdqa %ymm4, %ymm8
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7,8,9],ymm0[10],ymm2[11,12],ymm0[13],ymm2[14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm11[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm9[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6,7,8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm10[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm6[3],ymm0[4,5],ymm6[6],ymm0[7,8,9,10],ymm6[11],ymm0[12,13],ymm6[14],ymm0[15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm8[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm7 = ymm7[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7,8,9],ymm6[10],ymm7[11,12],ymm6[13],ymm7[14,15]
+; AVX512-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpermq {{.*#+}} ymm22 = ymm1[2,2,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm26 = ymm3[2,2,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm24 = ymm4[2,1,3,2]
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm9
+; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm10
+; AVX512-NEXT:    vprold $16, %xmm10, %xmm1
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm9[1,1,2,3]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]
+; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpermq {{.*#+}} ymm21 = ymm2[0,2,2,3]
+; AVX512-NEXT:    vmovdqa 32(%r9), %xmm1
+; AVX512-NEXT:    vmovdqa 32(%r8), %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
+; AVX512-NEXT:    vpshufb %xmm12, %xmm2, %xmm3
+; AVX512-NEXT:    vpshufb %xmm12, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm6[0,1,2,3,4,5,7,6]
+; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm27
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm18, %zmm1
+; AVX512-NEXT:    vpbroadcastd 36(%rax), %ymm2
+; AVX512-NEXT:    vpbroadcastd 40(%rax), %ymm4
+; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm13
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm13
+; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm7
+; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm6
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm4[0,0,1,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm0[2,1,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm3[0,0,1,1]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm2
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm21
-; AVX512-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm21
-; AVX512-NEXT:    vextracti64x4 $1, %zmm24, %ymm2
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd $226, 124(%r8){1to8}, %ymm0, %ymm2
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq $184, %ymm2, %ymm3, %ymm29
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm22 # 64-byte Folded Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogq $184, %zmm3, %zmm4, %zmm1
+; AVX512-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm3 = mem[2,1,3,2]
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512-NEXT:    vpternlogq $184, %ymm1, %ymm29, %ymm3
+; AVX512-NEXT:    vpternlogq $184, %ymm3, %ymm28, %ymm30
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm24 = zmm0[0,1,2,3],zmm6[0,1,2,3]
-; AVX512-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm24 # 64-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm29, %zmm0, %zmm2
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm14[0,1,2,3],zmm2[4,5,6,7]
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogd $184, %zmm2, %zmm0, %zmm29
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm7 # 64-byte Folded Reload
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm18 # 64-byte Folded Reload
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm17 # 64-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm20, %zmm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm19, %zmm3
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm4, %zmm3
-; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm25, %zmm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm16, %zmm5
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm4, %zmm5
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq $184, %zmm3, %zmm2, %zmm20
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq $184, %zmm5, %zmm2, %zmm25
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 32-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm3 # 64-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 32-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm27, %zmm31, %zmm3
-; AVX512-NEXT:    vpternlogq $184, %zmm2, %zmm30, %zmm3
-; AVX512-NEXT:    vpermq $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm2 = mem[0,2,2,3]
-; AVX512-NEXT:    vpshuflw $180, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm4 = mem[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512-NEXT:    vpermq $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm4 = mem[2,1,3,3]
-; AVX512-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm5 = mem[0,0,1,1]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm5 = mem[0,0,2,1]
-; AVX512-NEXT:    vpshuflw $230, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm6 = mem[2,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5,5,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,1,3]
-; AVX512-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm19 = mem[0,0,1,1]
-; AVX512-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm9 = mem[0,2,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]
-; AVX512-NEXT:    vpermq $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm11 = mem[0,2,2,3]
-; AVX512-NEXT:    vpshuflw $180, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm12 = mem[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[0,0,1,1]
-; AVX512-NEXT:    vpermq $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm13 = mem[2,1,3,3]
-; AVX512-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm14 = mem[0,0,1,1]
-; AVX512-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm15 = mem[0,0,2,1]
-; AVX512-NEXT:    vpshuflw $230, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm0 = mem[2,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,5,4]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,3]
-; AVX512-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm16 = mem[0,0,1,1]
-; AVX512-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm1 = mem[0,2,3,3,4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm30, %zmm4
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm12[0,1,1,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm11, %zmm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm11
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm30, %zmm11
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq $184, %zmm4, %zmm2, %zmm12
-; AVX512-NEXT:    vpternlogq $184, %zmm11, %zmm2, %zmm26
-; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm19, %zmm4
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm23, %zmm4
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm15, %zmm0
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm16, %zmm1
-; AVX512-NEXT:    vpternlogq $226, %zmm0, %zmm23, %zmm1
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm0 # 64-byte Folded Reload
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm23 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm18 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogq $226, %zmm23, %zmm28, %zmm18
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
+; AVX512-NEXT:    vpternlogq $226, %zmm3, %zmm2, %zmm18
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm23 # 32-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $226, %zmm3, %zmm28, %zmm23
+; AVX512-NEXT:    vpternlogq $226, %zmm0, %zmm2, %zmm23
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm17 # 32-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm17
+; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
+; AVX512-NEXT:    vpternlogd $226, 124(%r8){1to8}, %ymm19, %ymm0
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-NEXT:    vpternlogq $184, %ymm0, %ymm1, %ymm2
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm19 # 64-byte Folded Reload
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[0,1,2,3]
+; AVX512-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm1
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm30[0,1,2,3],zmm1[4,5,6,7]
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
+; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm2, %zmm30
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm2, %zmm11
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogq $184, %zmm14, %zmm1, %zmm25
+; AVX512-NEXT:    vpternlogq $184, %zmm5, %zmm1, %zmm20
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
+; AVX512-NEXT:    vpternlogq $226, %zmm1, %zmm5, %zmm2
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm22, %zmm3, %zmm22
+; AVX512-NEXT:    vpternlogq $226, %zmm1, %zmm5, %zmm22
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm14 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
+; AVX512-NEXT:    vpermd 64(%rax), %zmm14, %zmm5
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
+; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm3, %zmm5
+; AVX512-NEXT:    vinserti64x4 $1, %ymm24, %zmm26, %zmm1
+; AVX512-NEXT:    vpermd (%rax), %zmm14, %zmm14
+; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm3, %zmm14
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0]
+; AVX512-NEXT:    vpternlogq $184, %zmm2, %zmm1, %zmm5
+; AVX512-NEXT:    vpternlogq $184, %zmm22, %zmm1, %zmm14
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq $184, %zmm4, %zmm0, %zmm2
-; AVX512-NEXT:    vpternlogq $184, %zmm1, %zmm0, %zmm28
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm21, %zmm24
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm7
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 32-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $184, %zmm1, %zmm4, %zmm2
+; AVX512-NEXT:    vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm1 = mem[0,1,1,3]
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm22 # 32-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $226, %zmm1, %zmm4, %zmm22
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm12[0,1,1,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm21, %zmm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm15, %zmm8
+; AVX512-NEXT:    vpternlogq $226, %zmm1, %zmm4, %zmm8
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm7[u,u,4,5,u,u,u,u,6,7,u,u,u,u,8,9]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm6[1,1,2,2]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm9[0],xmm4[1],xmm9[2,3],xmm4[4],xmm9[5,6],xmm4[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]
+; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm7 = mem[2,2,2,3]
+; AVX512-NEXT:    vpshuflw $180, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm9 = mem[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[0,1,1,3]
+; AVX512-NEXT:    vpshufd $254, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm10 = mem[2,3,3,3,6,7,7,7]
+; AVX512-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm16 = mem[0,0,2,1]
+; AVX512-NEXT:    vpshuflw $230, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm12 = mem[2,1,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,5,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,1,3]
+; AVX512-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm21 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm21 = mem[0,0,1,1]
+; AVX512-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm15 = mem[0,2,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,0,2,1]
+; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm24 = mem[2,2,2,3]
+; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm3
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,2,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,1,1,3]
+; AVX512-NEXT:    vpshufd $254, {{[-0-9]+}}(%r{{[sb]}}p), %ymm26 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm26 = mem[2,3,3,3,6,7,7,7]
+; AVX512-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm27 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm27 = mem[0,0,2,1]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,4]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,1,1]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,2,3,3,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,0,1]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm7, %zmm7
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm10[2,1,3,2]
+; AVX512-NEXT:    vpbroadcastd 96(%rax), %ymm10
+; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
+; AVX512-NEXT:    vpternlogd $184, %zmm7, %zmm29, %zmm9
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,0,1]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm24, %zmm3
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm26[2,1,3,2]
+; AVX512-NEXT:    vpbroadcastd 32(%rax), %ymm10
+; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm7, %zmm7
+; AVX512-NEXT:    vpternlogd $184, %zmm3, %zmm29, %zmm7
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
+; AVX512-NEXT:    vpternlogq $184, %zmm22, %zmm3, %zmm9
+; AVX512-NEXT:    vpternlogq $184, %zmm8, %zmm3, %zmm7
+; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm16, %zmm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm15, %zmm21, %zmm8
+; AVX512-NEXT:    vpternlogq $226, %zmm3, %zmm28, %zmm8
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm27, %zmm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm3
+; AVX512-NEXT:    vpternlogq $226, %zmm1, %zmm28, %zmm3
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
+; AVX512-NEXT:    vpternlogq $184, %zmm8, %zmm1, %zmm31
+; AVX512-NEXT:    vpternlogq $184, %zmm3, %zmm1, %zmm13
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm17, %zmm0
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm11
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vmovdqa64 %zmm24, 320(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm28, 256(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm26, 192(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm25, 128(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm10, 64(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm17, (%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm18, 448(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm2, 704(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm12, 640(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm20, 576(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm8, 512(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm7, 384(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm22, 768(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm29, 832(%rax)
-; AVX512-NEXT:    addq $2168, %rsp # imm = 0x878
+; AVX512-NEXT:    vmovdqa64 %zmm0, 320(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm13, 256(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm7, 192(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm14, 128(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm23, 64(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm20, (%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm25, 448(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm31, 704(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm9, 640(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm5, 576(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm18, 512(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm11, 384(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm19, 768(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm30, 832(%rax)
+; AVX512-NEXT:    addq $2808, %rsp # imm = 0xAF8
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i16_stride7_vf64:
 ; AVX512-FCP:       # %bb.0:
 ; AVX512-FCP-NEXT:    subq $1496, %rsp # imm = 0x5D8
-; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %ymm0
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm15 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm20
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %ymm9
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm9, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm16
-; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %ymm6
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm6, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %ymm7
+; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %ymm3
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm3, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %ymm1
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm1, %ymm2
+; AVX512-FCP-NEXT:    vporq %ymm0, %ymm2, %ymm16
+; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %ymm4
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm4, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %ymm5
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm7, %ymm3
-; AVX512-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm17
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
-; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm2
+; AVX512-FCP-NEXT:    vpor %ymm0, %ymm2, %ymm7
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
+; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %ymm2
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm2, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm26
+; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %ymm12
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm12, %ymm28
+; AVX512-FCP-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %ymm0
-; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm5 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm0, %ymm4
-; AVX512-FCP-NEXT:    vporq %ymm2, %ymm4, %ymm18
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %ymm0
-; AVX512-FCP-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm0, %ymm8
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm0, %ymm14
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm31
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %ymm0
-; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm12
-; AVX512-FCP-NEXT:    vpor %ymm8, %ymm12, %ymm0
-; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm0, %ymm15
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm21
+; AVX512-FCP-NEXT:    vpor %ymm14, %ymm15, %ymm12
+; AVX512-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %ymm0
-; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %ymm1
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm1, %ymm13
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm28
-; AVX512-FCP-NEXT:    vpor %ymm0, %ymm13, %ymm0
-; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm14
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm29
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm15
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm17
+; AVX512-FCP-NEXT:    vpor %ymm14, %ymm15, %ymm12
+; AVX512-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm1
-; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm1, %ymm13
-; AVX512-FCP-NEXT:    vporq %ymm0, %ymm13, %ymm19
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm14
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm0
+; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm15
+; AVX512-FCP-NEXT:    vpor %ymm14, %ymm15, %ymm12
+; AVX512-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm1
-; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa %ymm10, %ymm2
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm13
-; AVX512-FCP-NEXT:    vpor %ymm0, %ymm13, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm0, %ymm14
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm0, %ymm12
+; AVX512-FCP-NEXT:    vpor %ymm14, %ymm12, %ymm12
+; AVX512-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm1
-; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm1, %ymm13
-; AVX512-FCP-NEXT:    vpor %ymm0, %ymm13, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm12
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm10
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm10, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm14
+; AVX512-FCP-NEXT:    vpor %ymm12, %ymm14, %ymm12
+; AVX512-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm14
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm14, %ymm8
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm15
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm15, %ymm13
-; AVX512-FCP-NEXT:    vpor %ymm0, %ymm13, %ymm0
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm8
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm8, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm4
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm11
-; AVX512-FCP-NEXT:    vpor %ymm0, %ymm11, %ymm0
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm13
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm13, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm14
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm14, %ymm5
-; AVX512-FCP-NEXT:    vmovdqa %ymm3, %ymm12
-; AVX512-FCP-NEXT:    vporq %ymm5, %ymm0, %ymm21
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm2
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm11 = ymm9[0,1,1,3,4,5,5,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm11[0,1],ymm0[2],ymm11[3,4],ymm0[5],ymm11[6,7,8,9],ymm0[10],ymm11[11,12],ymm0[13],ymm11[14,15]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,2]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[1,1,1,1,5,5,5,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm11[0,1],ymm3[2],ymm11[3,4],ymm3[5],ymm11[6,7,8,9],ymm3[10],ymm11[11,12],ymm3[13],ymm11[14,15]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm16, %zmm0
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm17, %zmm3
-; AVX512-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %ymm11
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm11[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm11[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm11[16,17,u,u,u,u],zero,zero
-; AVX512-FCP-NEXT:    vpternlogq $248, %ymm25, %ymm3, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %ymm5
-; AVX512-FCP-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm5, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq $248, %ymm12, %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm3, %ymm0
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [0,4,0,0,0,5,0,0]
-; AVX512-FCP-NEXT:    vpermd %ymm11, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm0, %ymm17, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm17, %zmm22
-; AVX512-FCP-NEXT:    vprold $16, %ymm5, %ymm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm3, %ymm5, %ymm0
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm15, %ymm9
+; AVX512-FCP-NEXT:    vpor %ymm8, %ymm9, %ymm8
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm8
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm2
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm2, %ymm9
+; AVX512-FCP-NEXT:    vpor %ymm8, %ymm9, %ymm8
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm8
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm8, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm9
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm9, %ymm11
+; AVX512-FCP-NEXT:    vporq %ymm11, %ymm10, %ymm22
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm3[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[0,1,1,3,4,5,5,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm11 = ymm12[0,1],ymm11[2],ymm12[3,4],ymm11[5],ymm12[6,7,8,9],ymm11[10],ymm12[11,12],ymm11[13],ymm12[14,15]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,1,3,2]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
+; AVX512-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm4, %ymm12
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm13, %ymm30
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm13 = ymm5[1,1,1,1,5,5,5,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm13[2],ymm12[3,4],ymm13[5],ymm12[6,7,8,9],ymm13[10],ymm12[11,12],ymm13[13],ymm12[14,15]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm16, %zmm11
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm7, %zmm12
+; AVX512-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm12
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %ymm7
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm7[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[16,17,u,u,u,u],zero,zero
+; AVX512-FCP-NEXT:    vpternlogq $248, %ymm13, %ymm12, %ymm11
+; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %ymm10
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm10, %ymm6
+; AVX512-FCP-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $248, %ymm16, %ymm11, %ymm6
+; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm12, %ymm11
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [0,4,0,0,0,5,0,0]
+; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm12, %ymm12
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
+; AVX512-FCP-NEXT:    vpternlogq $184, %ymm11, %ymm16, %ymm12
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, %zmm19
+; AVX512-FCP-NEXT:    vprold $16, %ymm10, %ymm11
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $184, %ymm12, %ymm10, %ymm11
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm11
+; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm11[4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,1,4,5,4,5,5,7]
-; AVX512-FCP-NEXT:    vmovdqa 96(%rax), %ymm0
-; AVX512-FCP-NEXT:    vpermd %ymm0, %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vpandnq %ymm1, %ymm25, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm5 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm0, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm5, %ymm20
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [0,1,4,5,4,5,5,7]
+; AVX512-FCP-NEXT:    vmovdqa 96(%rax), %ymm6
+; AVX512-FCP-NEXT:    vpermd %ymm6, %ymm11, %ymm11
+; AVX512-FCP-NEXT:    vpandn %ymm11, %ymm13, %ymm11
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, %zmm18
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm6, %ymm12
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm11
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm4, %ymm11
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm5[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3],ymm12[4,5],ymm11[6],ymm12[7,8,9,10],ymm11[11],ymm12[12,13],ymm11[14],ymm12[15]
+; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm12 = [151522058,0,421010202,421010202]
+; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm4, %ymm4
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm12, %ymm23
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6,7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13,14,15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [0,2,2,3,10,9,11,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm11, %zmm5, %zmm4
+; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm12 = [218894094,0,488382238,488382238]
+; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm11
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm12, %ymm16
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm11 = ymm12[0],ymm11[1],ymm12[2,3],ymm11[4],ymm12[5,6,7,8],ymm11[9],ymm12[10,11],ymm11[12],ymm12[13,14,15]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
+; AVX512-FCP-NEXT:    # ymm12 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm3
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0,1],ymm1[2],ymm3[3,4],ymm1[5],ymm3[6,7,8,9],ymm1[10],ymm3[11,12],ymm1[13],ymm3[14,15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,2,2,3,8,10,10,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm11, %zmm20, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm27, %zmm1
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm6, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm26
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5],ymm1[6],ymm3[7,8,9,10],ymm1[11],ymm3[12,13],ymm1[14],ymm3[15]
-; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm5 = [151522058,0,421010202,421010202]
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm6, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm5, %ymm29
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm7[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm6[0],ymm3[1],ymm6[2,3],ymm3[4],ymm6[5,6,7,8],ymm3[9],ymm6[10,11],ymm3[12],ymm6[13,14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [0,2,2,3,10,9,11,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm6, %zmm3
-; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm7 = [218894094,0,488382238,488382238]
-; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm7, %ymm27
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm9[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm7[0],ymm1[1],ymm7[2,3],ymm1[4],ymm7[5,6,7,8],ymm1[9],ymm7[10,11],ymm1[12],ymm7[13,14,15]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
-; AVX512-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm2, %ymm7
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm5, %ymm16
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1],ymm9[2],ymm7[3,4],ymm9[5],ymm7[6,7,8,9],ymm9[10],ymm7[11,12],ymm9[13],ymm7[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm17 = [2,2,2,3,8,10,10,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm17, %zmm7
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm1, %zmm7
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm24
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [5,0,0,0,6,0,0,6]
-; AVX512-FCP-NEXT:    vpermd %ymm11, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm1, %ymm1
 ; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa64 64(%rax), %zmm12
+; AVX512-FCP-NEXT:    vmovdqa64 64(%rax), %zmm1
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,5,0,0,0,6,0,0,30,0,0,0,31,0,0,31]
-; AVX512-FCP-NEXT:    vpermi2d %zmm12, %zmm0, %zmm3
+; AVX512-FCP-NEXT:    vpermi2d %zmm1, %zmm6, %zmm3
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vpbroadcastd 72(%rax), %ymm0
-; AVX512-FCP-NEXT:    vpandnq %ymm0, %ymm23, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
+; AVX512-FCP-NEXT:    vpbroadcastd 72(%rax), %ymm3
+; AVX512-FCP-NEXT:    vpandn %ymm3, %ymm6, %ymm3
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rax), %ymm7
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm5
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm7, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm7, %ymm30
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm11
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %xmm2
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm20
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [0,0,1,1,12,13,14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm3
-; AVX512-FCP-NEXT:    vpternlogq $248, %zmm23, %zmm3, %zmm11
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpbroadcastd 8(%rax), %ymm0
-; AVX512-FCP-NEXT:    vpandnq %ymm0, %ymm23, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm1
-; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm1, %ymm3
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm5
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm2
-; AVX512-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm3
-; AVX512-FCP-NEXT:    vpternlogq $248, %zmm23, %zmm3, %zmm5
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
-; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm14, %ymm0
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm13[0,0,2,1,4,4,6,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5],ymm3[6],ymm0[7,8,9,10],ymm3[11],ymm0[12,13],ymm3[14],ymm0[15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,1,0,3,10,10,11,11]
-; AVX512-FCP-NEXT:    vpermi2q %zmm0, %zmm21, %zmm3
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm7, %ymm4
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm7, %ymm25
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm26, %ymm9
-; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm8, %ymm0
+; AVX512-FCP-NEXT:    vpbroadcastd 8(%rax), %ymm3
+; AVX512-FCP-NEXT:    vpandn %ymm3, %ymm6, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm4
 ; AVX512-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm4[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3],ymm3[4,5],ymm0[6],ymm3[7,8,9,10],ymm0[11],ymm3[12,13],ymm0[14],ymm3[15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm29, %ymm1
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm8, %ymm3
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm4[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3],ymm3[4],ymm5[5,6,7,8],ymm3[9],ymm5[10,11],ymm3[12],ymm5[13,14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm6, %zmm3
-; AVX512-FCP-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm27, %ymm11
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm4, %ymm4
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm9, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm24
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[0,0,2,1,4,4,6,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8,9,10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,1,0,3,10,10,11,11]
+; AVX512-FCP-NEXT:    vpermi2q %zmm3, %zmm22, %zmm4
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm3
+; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm2[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5],ymm3[6],ymm4[7,8,9,10],ymm3[11],ymm4[12,13],ymm3[14],ymm4[15]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm23, %ymm7
+; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm0, %ymm4
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm6[0],ymm4[1],ymm6[2,3],ymm4[4],ymm6[5,6,7,8],ymm4[9],ymm6[10,11],ymm4[12],ymm6[13,14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm5, %zmm4
+; AVX512-FCP-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm16, %ymm2
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm14, %ymm3
 ; AVX512-FCP-NEXT:    vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm15[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2,3],ymm0[4],ymm5[5,6,7,8],ymm0[9],ymm5[10,11],ymm0[12],ymm5[13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm16, %ymm4
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm10, %ymm5
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3],ymm3[4],ymm5[5,6,7,8],ymm3[9],ymm5[10,11],ymm3[12],ymm5[13,14,15]
+; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm14, %ymm5
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm15[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7,8,9],ymm6[10],ymm5[11,12],ymm6[13],ymm5[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm17, %zmm2
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm24, %zmm16
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm24, %zmm2
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm14, %ymm0
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm13[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7,8,9],ymm3[10],ymm0[11,12],ymm3[13],ymm0[14,15]
-; AVX512-FCP-NEXT:    vprold $16, %ymm14, %ymm3
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm13[1,2,2,3,5,6,6,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm3[2],ymm5[3,4],ymm3[5],ymm5[6,7,8,9],ymm3[10],ymm5[11,12],ymm3[13],ymm5[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [2,1,3,2,10,10,10,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa64 (%rax), %zmm19
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,21,0,0,0,22,0,0,14,0,0,0,15,0,0,15]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7,8,9],ymm6[10],ymm5[11,12],ymm6[13],ymm5[14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm0
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm27, %zmm0
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7,8,9],ymm4[10],ymm3[11,12],ymm4[13],ymm3[14,15]
+; AVX512-FCP-NEXT:    vprold $16, %ymm9, %ymm4
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm8[1,2,2,3,5,6,6,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm11 = [2,1,3,2,10,10,10,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm11, %zmm4
+; AVX512-FCP-NEXT:    vmovdqa64 (%rax), %zmm16
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,21,0,0,0,22,0,0,14,0,0,0,15,0,0,15]
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpermi2d %zmm0, %zmm19, %zmm2
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm22, %zmm2
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm13, %ymm0
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm28[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1],ymm3[2,3],ymm0[4],ymm3[5,6,7,8],ymm0[9],ymm3[10,11],ymm0[12],ymm3[13,14,15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm28[1,1,1,1,5,5,5,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm5[2],ymm3[3,4],ymm5[5],ymm3[6,7,8,9],ymm5[10],ymm3[11,12],ymm5[13],ymm3[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm17, %zmm3
-; AVX512-FCP-NEXT:    vmovdqu (%rsp), %ymm1 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm0
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm5[2],ymm0[3,4],ymm5[5],ymm0[6,7,8,9],ymm5[10],ymm0[11,12],ymm5[13],ymm0[14,15]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm1[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm2[0,1,1,3,4,5,5,7]
-; AVX512-FCP-NEXT:    vmovdqa %ymm2, %ymm4
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7,8,9],ymm5[10],ymm6[11,12],ymm5[13],ymm6[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm5
-; AVX512-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512-FCP-NEXT:    vprold $16, %ymm8, %ymm0
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm2[1,2,2,3,5,6,6,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1],ymm0[2],ymm3[3,4],ymm0[5],ymm3[6,7,8,9],ymm0[10],ymm3[11,12],ymm0[13],ymm3[14,15]
-; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm8, %ymm3
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm2[0,0,2,1,4,4,6,5]
-; AVX512-FCP-NEXT:    vmovdqa %ymm2, %ymm14
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm6[3],ymm3[4,5],ymm6[6],ymm3[7,8,9,10],ymm6[11],ymm3[12,13],ymm6[14],ymm3[15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [2,2,3,3,10,9,11,10]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm2, %zmm3
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm30, %zmm12, %zmm0
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm2, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm25, %zmm0
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm0
+; AVX512-FCP-NEXT:    vpermi2d %zmm0, %zmm16, %zmm3
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm4, %zmm19, %zmm3
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm29, %ymm10
+; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm10, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm17, %ymm19
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm17[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6,7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13,14,15]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm30, %ymm9
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm10, %ymm4
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm17[1,1,1,1,5,5,5,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm31, %ymm8
+; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm8, %ymm3
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm21[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7,8,9],ymm4[10],ymm3[11,12],ymm4[13],ymm3[14,15]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm8[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm21[0,1,1,3,4,5,5,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm4[2],ymm6[3,4],ymm4[5],ymm6[6,7,8,9],ymm4[10],ymm6[11,12],ymm4[13],ymm6[14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm11, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm5, %zmm27, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm26, %ymm12
+; AVX512-FCP-NEXT:    vprold $16, %ymm26, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm28, %ymm22
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm28[1,2,2,3,5,6,6,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm3[2],ymm5[3,4],ymm3[5],ymm5[6,7,8,9],ymm3[10],ymm5[11,12],ymm3[13],ymm5[14,15]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm24, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm12, %ymm5
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm26, %ymm30
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm28[0,0,2,1,4,4,6,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1,2],ymm7[3],ymm5[4,5],ymm7[6],ymm5[7,8,9,10],ymm7[11],ymm5[12,13],ymm7[14],ymm5[15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [2,2,3,3,10,9,11,10]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm23, %zmm5
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm25, %zmm1, %zmm1
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
+; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm5, %zmm18, %zmm0
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm1, %ymm0
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm4[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm0
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm21[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %xmm3
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm29
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm7
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm31 = [0,2,2,3,8,9,9,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm31, %zmm1
-; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm13, %ymm0
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm28[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3],ymm2[4,5],ymm0[6],ymm2[7,8,9,10],ymm0[11],ymm2[12,13],ymm0[14],ymm2[15]
-; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm3
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm4, %xmm26
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm24
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [2,1,3,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm3, %zmm0
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm1, %zmm16, %zmm0
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm8, %ymm1
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm14[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm13 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %xmm2
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %xmm1
-; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm4 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm2, %xmm5
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, %xmm8
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm1[1,1,2,2]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3],xmm5[4],xmm6[5,6],xmm5[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX512-FCP-NEXT:    vmovdqa %xmm7, %xmm14
-; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm1, %xmm2
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,1,1,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm1, %zmm2
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm11
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, (%rsp) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %xmm2
+; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm5 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa %xmm5, %xmm14
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[1,1,2,2]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0],xmm3[1],xmm5[2,3],xmm3[4],xmm5[5,6],xmm3[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512-FCP-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm0 = [0,1,1,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm6
 ; AVX512-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vprold $16, %xmm6, %xmm1
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,2,3]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm1[2],xmm5[3,4],xmm1[5],xmm5[6,7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm9, %xmm3
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [0,0,1,1,8,8,10,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm27, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm5, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm20, %xmm2
-; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3]
+; AVX512-FCP-NEXT:    vprold $16, %xmm6, %xmm3
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,2,3]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2],xmm5[3,4],xmm3[5],xmm5[6,7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm5, %xmm5
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,0,1,1,8,8,10,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm28, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm3, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %xmm2
+; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %xmm6
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm10 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm2, %xmm6
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm2, %xmm6
 ; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [0,0,0,1,8,9,9,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm16, %zmm2
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm9
-; AVX512-FCP-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm6
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm9[1,1,2,2]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm8[0],xmm6[1],xmm8[2,3],xmm6[4],xmm8[5,6],xmm6[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm8, %xmm8
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm14, %xmm21
-; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm11, %zmm8
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm11
-; AVX512-FCP-NEXT:    vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vprold $16, %xmm11, %xmm6
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm4[1,1,2,3]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm9[0,1],xmm6[2],xmm9[3,4],xmm6[5],xmm9[6,7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm9, %xmm9
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm28
-; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm27, %zmm9
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm8, %zmm5, %zmm9
-; AVX512-FCP-NEXT:    vpbroadcastd 64(%rax), %ymm5
-; AVX512-FCP-NEXT:    vpbroadcastd 68(%rax), %ymm6
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm23
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm4, %zmm23
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm26 = [0,0,0,1,8,9,9,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm26, %zmm2
+; AVX512-FCP-NEXT:    vpbroadcastd 64(%rax), %ymm6
+; AVX512-FCP-NEXT:    vpbroadcastd 68(%rax), %ymm8
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm31
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm21, %zmm31
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq $184, %zmm1, %zmm2, %zmm23
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm1, %xmm5
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm10, %xmm22
-; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm1, %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm7, %xmm3
-; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm16, %zmm1
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm5, %zmm2, %zmm31
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm8
+; AVX512-FCP-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm0, %xmm5
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm14, %xmm29
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm8[1,1,2,2]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3],xmm5[4],xmm6[5,6],xmm5[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm6, %xmm6
+; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm13, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm13
+; AVX512-FCP-NEXT:    vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vprold $16, %xmm13, %xmm5
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[1,1,2,3]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm8[0,1],xmm5[2],xmm8[3,4],xmm5[5],xmm8[6,7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm8, %xmm8
+; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm28, %zmm8
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm6, %zmm3, %zmm8
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm5
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm3, %xmm5
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm7, %xmm17
+; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm26, %zmm3
 ; AVX512-FCP-NEXT:    vpbroadcastd (%rax), %ymm5
 ; AVX512-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm6
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm20
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm1, %zmm4, %zmm20
-; AVX512-FCP-NEXT:    vpternlogq $184, %zmm9, %zmm2, %zmm20
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm4[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm17[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm17[1,1,1,1,5,5,5,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm5[2],ymm2[3,4],ymm5[5],ymm2[6,7,8,9],ymm5[10],ymm2[11,12],ymm5[13],ymm2[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,2,2,3,8,10,10,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm5, %zmm2
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm26
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm21, %zmm26
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm8, %zmm2, %zmm26
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm14[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm6[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13,14,15]
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm14, %ymm3
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm6[1,1,1,1,5,5,5,5]
+; AVX512-FCP-NEXT:    vmovdqa %ymm6, %ymm9
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm5[2],ymm3[3,4],ymm5[5],ymm3[6,7,8,9],ymm5[10],ymm3[11,12],ymm5[13],ymm3[14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm20, %zmm3
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm7[2,2,2,2,6,6,6,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm5[2],ymm1[3,4],ymm5[5],ymm1[6,7,8,9],ymm5[10],ymm1[11,12],ymm5[13],ymm1[14,15]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm9[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm5[2],ymm2[3,4],ymm5[5],ymm2[6,7,8,9],ymm5[10],ymm2[11,12],ymm5[13],ymm2[14,15]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm7[0,1,1,3,4,5,5,7]
-; AVX512-FCP-NEXT:    vmovdqa %ymm7, %ymm15
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7,8,9],ymm5[10],ymm6[11,12],ymm5[13],ymm6[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm18, %zmm8
-; AVX512-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm8
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX512-FCP-NEXT:    vprold $16, %ymm5, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm7, %ymm18
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7,8,9],ymm5[10],ymm6[11,12],ymm5[13],ymm6[14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm11, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %xmm11
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %xmm7
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [0,2,2,3,8,9,9,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm6, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm10, %ymm2
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm19[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm8[0,1,2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7,8,9,10],ymm2[11],ymm8[12,13],ymm2[14],ymm8[15]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm27, %zmm5
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512-FCP-NEXT:    vprold $16, %ymm4, %ymm2
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm19[1,2,2,3,5,6,6,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7,8,9],ymm2[10],ymm3[11,12],ymm2[13],ymm3[14,15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm24
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm19[0,0,2,1,4,4,6,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm3[0,1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8,9,10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm23, %zmm8
+; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,1,3,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm20, %zmm10
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm1, %zmm21, %zmm10
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm30, %ymm1
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm22[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Reload
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm23, %zmm16, %zmm1
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
+; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm16, %zmm22
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm22
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm22
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm1
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm1, %xmm1
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm18[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2,3],ymm0[4],ymm5[5,6,7,8],ymm0[9],ymm5[10,11],ymm0[12],ymm5[13,14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm6, %zmm0
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [2,2,2,3,8,8,8,9]
+; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %xmm5
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm17, %xmm1
+; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm8, %xmm1
+; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm30, %zmm2
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm14, %ymm1
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm9[0,1,2],ymm1[3],ymm9[4,5],ymm1[6],ymm9[7,8,9,10],ymm1[11],ymm9[12,13],ymm1[14],ymm9[15]
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm15
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm14, %xmm14
+; AVX512-FCP-NEXT:    vpermt2q %zmm14, %zmm20, %zmm1
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [6,7,3,3,7,7,6,7]
+; AVX512-FCP-NEXT:    vpermd %ymm25, %ymm17, %ymm16
+; AVX512-FCP-NEXT:    vpbroadcastd 96(%rax), %ymm20
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm16, %zmm25
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm18, %zmm25
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm10, %zmm27, %zmm25
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; AVX512-FCP-NEXT:    vprold $16, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm21, %zmm1
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [0,0,2,1,8,8,9,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm21, %zmm3
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm29, %xmm2
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,2,2]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0],xmm2[1],xmm7[2,3],xmm2[4],xmm7[5,6],xmm2[7]
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm28, %zmm7
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm28, %zmm7
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm16[1,2,2,3,5,6,6,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7,8,9],ymm1[10],ymm2[11,12],ymm1[13],ymm2[14,15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm5, %ymm18
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm16[0,0,2,1,4,4,6,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm5[3],ymm2[4,5],ymm5[6],ymm2[7,8,9,10],ymm5[11],ymm2[12,13],ymm5[14],ymm2[15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,2,3,3,10,9,11,10]
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm5, %zmm2
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,2,2,3,8,8,8,9]
-; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %xmm11
-; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %xmm7
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm10, %xmm1
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm5, %zmm13
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm19, %zmm1
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm5, %zmm19
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm25, %zmm19
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm19
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm12
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm8
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm12[0],xmm8[1],xmm12[1],xmm8[2],xmm12[2],xmm8[3],xmm12[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm2
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm9[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm9 = ymm15[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm15 = ymm9[0],ymm2[1],ymm9[2,3],ymm2[4],ymm9[5,6,7,8],ymm2[9],ymm9[10,11],ymm2[12],ymm9[13,14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm31, %zmm15
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm31 = [6,7,3,3,7,7,6,7]
-; AVX512-FCP-NEXT:    vpermd %ymm30, %ymm31, %ymm1
-; AVX512-FCP-NEXT:    vpbroadcastd 96(%rax), %ymm2
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm21
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm13, %zmm30, %zmm21
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm21
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm1
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; AVX512-FCP-NEXT:    vprold $16, %xmm26, %xmm1
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm24[1,1,2,3]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm17[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8,9,10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm5
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm4
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm28, %xmm6
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm13, %xmm13
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [2,1,3,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm13, %zmm2, %zmm1
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm17 = [0,0,2,1,8,8,9,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm17, %zmm3
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vpternlogq $184, %ymm0, %ymm18, %ymm16
+; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm0, %ymm13
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm8[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,1,1,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm3, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm24, %ymm2
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm19[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7,8,9],ymm3[10],ymm2[11,12],ymm3[13],ymm2[14,15]
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm11
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm11[0],xmm3[0],xmm11[1],xmm3[1],xmm11[2],xmm3[2],xmm11[3],xmm3[3]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm2[0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm30, %zmm8
+; AVX512-FCP-NEXT:    vpbroadcastd 100(%rax), %ymm5
+; AVX512-FCP-NEXT:    vpbroadcastd 104(%rax), %ymm19
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm5, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm6, %zmm24, %zmm5
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm4[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[3,3,3,3]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25,22,23,22,23,u,u,u,u,u,u,u,u]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm10, %xmm14
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,1,1]
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm10, %xmm10
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm12, %xmm30
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1]
+; AVX512-FCP-NEXT:    vpermd %ymm23, %ymm17, %ymm17
+; AVX512-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm19
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm17, %zmm20
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm8, %zmm18, %zmm20
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm1, %zmm27, %zmm20
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm7, %zmm1, %zmm5
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm15[4],xmm9[5],xmm15[5],xmm9[6],xmm15[6],xmm9[7],xmm15[7]
+; AVX512-FCP-NEXT:    vprold $16, %xmm9, %xmm8
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm15[1,1,2,3]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm8 = xmm9[0,1],xmm8[2],xmm9[3,4],xmm8[5],xmm9[6,7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm7, %xmm7
+; AVX512-FCP-NEXT:    vpermt2q %zmm7, %zmm21, %zmm8
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm29, %xmm0
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
-; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm24 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm13
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[1,1,2,2]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm9[0],xmm0[1],xmm9[2,3],xmm0[4],xmm9[5,6],xmm0[7]
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,2,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm27, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm25, %zmm0
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm3, %ymm30, %ymm7
-; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm3, %ymm11
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm13
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm10, %xmm3
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,1,1,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm28, %zmm3
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm6[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm26 = ymm2[3,3,3,3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25,22,23,22,23,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm29 = ymm2[2,2,2,2]
-; AVX512-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm1
-; AVX512-FCP-NEXT:    vpbroadcastd 100(%rax), %ymm2
-; AVX512-FCP-NEXT:    vpbroadcastd 104(%rax), %ymm6
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm22, %zmm2
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm18, %ymm3
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm16[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm15
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm3
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm6[0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [2,2,2,3,8,8,8,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm9, %zmm16, %zmm10
-; AVX512-FCP-NEXT:    vpermd %ymm14, %ymm31, %ymm9
-; AVX512-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm18
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm9, %zmm9
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm10, %zmm30, %zmm9
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm9
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
-; AVX512-FCP-NEXT:    vpternlogq $184, %zmm0, %zmm1, %zmm2
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
-; AVX512-FCP-NEXT:    vprold $16, %xmm4, %xmm4
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm10 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm17, %zmm4
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm0
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm12, %xmm0
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm8[4],xmm12[4],xmm8[5],xmm12[5],xmm8[6],xmm12[6],xmm8[7],xmm12[7]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[1,1,2,2]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm8[0],xmm0[1],xmm8[2,3],xmm0[4],xmm8[5,6],xmm0[7]
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,2,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm27, %zmm0
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm25, %zmm0
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm15[4],xmm3[5],xmm15[5],xmm3[6],xmm15[6],xmm3[7],xmm15[7]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm4
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm28, %zmm4
+; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm9, %xmm7
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm12 = xmm0[1,1,2,2]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm12[0],xmm7[1],xmm12[2,3],xmm7[4],xmm12[5,6],xmm7[7]
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,2,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm0 = [0,0,1,1,8,8,10,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm9, %zmm0, %zmm7
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm8, %zmm28, %zmm7
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm30, %xmm0
+; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm0 = [0,1,1,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
 ; AVX512-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm3
-; AVX512-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm5
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm3
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm4, %zmm22, %zmm3
-; AVX512-FCP-NEXT:    vpternlogq $184, %zmm0, %zmm1, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm7, %ymm0, %ymm29
+; AVX512-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm8
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm3, %zmm3
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm24, %zmm3
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm7, %zmm1, %zmm3
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd $226, 124(%r8){1to8}, %ymm0, %ymm11
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm11, %ymm0, %ymm26
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm26, %zmm0, %zmm1
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm29[0,1,2,3],zmm1[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm12
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpckhwd (%rsp), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
+; AVX512-FCP-NEXT:    vpternlogd $226, 124(%r8){1to8}, %ymm0, %ymm13
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $184, %ymm13, %ymm1, %ymm6
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
+; AVX512-FCP-NEXT:    vpternlogq $184, %ymm16, %ymm2, %ymm4
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
+; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm13
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm0 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm0 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm2 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX512-FCP-NEXT:    vpunpckhwd (%rsp), %xmm4, %xmm4 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm5 = xmm5[4],mem[4],xmm5[5],mem[5],xmm5[6],mem[6],xmm5[7],mem[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm1, %xmm1
-; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm5, %xmm5
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm6[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512-FCP-NEXT:    vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    # ymm7 = mem[0,1,1,3,4,5,5,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7,8,9],ymm6[10],ymm7[11,12],ymm6[13],ymm7[14,15]
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512-FCP-NEXT:    vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    # ymm8 = mem[1,1,1,1,5,5,5,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1],ymm8[2],ymm7[3,4],ymm8[5],ymm7[6,7,8,9],ymm8[10],ymm7[11,12],ymm8[13],ymm7[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [0,1,4,5,4,5,5,7]
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpermd %ymm10, %ymm8, %ymm8
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[16,17],zero,zero
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpandn %ymm8, %ymm11, %ymm8
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm10, %zmm8
-; AVX512-FCP-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm10 = mem[0,2,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,3]
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,2,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,1,3]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,1,3,2]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,3]
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
+; AVX512-FCP-NEXT:    # xmm6 = xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload
+; AVX512-FCP-NEXT:    # xmm7 = xmm7[4],mem[4],xmm7[5],mem[5],xmm7[6],mem[6],xmm7[7],mem[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm4, %xmm4
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm7, %xmm7
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm8[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512-FCP-NEXT:    vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    # ymm9 = mem[0,1,1,3,4,5,5,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm9[0,1],ymm8[2],ymm9[3,4],ymm8[5],ymm9[6,7,8,9],ymm8[10],ymm9[11,12],ymm8[13],ymm9[14,15]
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
+; AVX512-FCP-NEXT:    vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    # ymm10 = mem[1,1,1,1,5,5,5,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm9 = ymm9[0,1],ymm10[2],ymm9[3,4],ymm10[5],ymm9[6,7,8,9],ymm10[10],ymm9[11,12],ymm10[13],ymm9[14,15]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,1,4,5,4,5,5,7]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm13
-; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm10, %zmm25, %zmm1
+; AVX512-FCP-NEXT:    vpermd %ymm11, %ymm10, %ymm10
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[16,17],zero,zero
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpandn %ymm10, %ymm12, %ymm10
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm11, %zmm10
+; AVX512-FCP-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
+; AVX512-FCP-NEXT:    # xmm11 = mem[0,2,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,1,3]
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,2,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,1,3]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,1,3,2]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm14
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm25, %zmm5
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
-; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm1 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm6
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm8 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm8
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm11, %zmm28, %zmm4
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm6, %zmm28, %zmm7
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm6, %zmm4
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm6, %zmm7
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm2, %zmm2
+; AVX512-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm10 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm10
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, 320(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, 320(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 256(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm9, 192(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm19, 128(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm20, (%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm23, 448(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 704(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm21, 640(%rax)
-; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, 384(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 64(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 512(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, 832(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 768(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm20, 192(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm22, 128(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm26, (%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm31, 448(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 704(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm25, 640(%rax)
+; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512-FCP-NEXT:    vmovaps %zmm1, 576(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm14, 384(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, 64(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 512(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, 832(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 768(%rax)
 ; AVX512-FCP-NEXT:    addq $1496, %rsp # imm = 0x5D8
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i16_stride7_vf64:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    subq $2168, %rsp # imm = 0x878
-; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %ymm2
-; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %ymm9
+; AVX512DQ-NEXT:    subq $2808, %rsp # imm = 0xAF8
+; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %ymm6
+; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %ymm13
 ; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %ymm7
 ; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %ymm8
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm18
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm9, %ymm2
-; AVX512DQ-NEXT:    vporq %ymm1, %ymm2, %ymm16
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm8, %ymm1
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm13 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm7, %ymm2
-; AVX512DQ-NEXT:    vporq %ymm1, %ymm2, %ymm17
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm6, %ymm2
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm13, %ymm3
+; AVX512DQ-NEXT:    vporq %ymm2, %ymm3, %ymm17
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm8, %ymm2
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm7, %ymm3
+; AVX512DQ-NEXT:    vporq %ymm2, %ymm3, %ymm18
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
 ; AVX512DQ-NEXT:    vmovdqa 64(%r9), %ymm2
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqa %ymm3, %ymm10
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm21
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vmovdqa 64(%r8), %ymm3
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm15 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
-; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm20
-; AVX512DQ-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %ymm2
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm23
-; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %ymm6
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm6, %ymm2
-; AVX512DQ-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %ymm5
-; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm5, %ymm1
+; AVX512DQ-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm26
+; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %ymm4
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vmovdqa64 %ymm4, %ymm27
+; AVX512DQ-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm23
 ; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %ymm4
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm4, %ymm2
-; AVX512DQ-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqa %ymm10, %ymm3
-; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm2
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vmovdqa64 %ymm4, %ymm22
+; AVX512DQ-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm1
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm2
+; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm2
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm3
+; AVX512DQ-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm2
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm3
+; AVX512DQ-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm2
-; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm10
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm10, %ymm2
-; AVX512DQ-NEXT:    vmovdqa64 %ymm10, %ymm22
-; AVX512DQ-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm2
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm1
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm1, %ymm11
-; AVX512DQ-NEXT:    vpor %ymm0, %ymm11, %ymm0
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm3
+; AVX512DQ-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm4
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm2
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm0, %ymm12
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm11
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm11, %ymm13
-; AVX512DQ-NEXT:    vpor %ymm12, %ymm13, %ymm10
-; AVX512DQ-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm13
-; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm13, %ymm12
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm15
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm15, %ymm14
-; AVX512DQ-NEXT:    vpor %ymm14, %ymm12, %ymm10
-; AVX512DQ-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vprold $16, %ymm15, %ymm12
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm14 = ymm13[1,2,2,3,5,6,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm12 = ymm14[0,1],ymm12[2],ymm14[3,4],ymm12[5],ymm14[6,7,8,9],ymm12[10],ymm14[11,12],ymm12[13],ymm14[14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm14 = ymm15[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm14 = ymm14[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm5
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm5, %ymm0
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm3, %ymm1
+; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm1, %ymm9
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpor %ymm10, %ymm9, %ymm9
+; AVX512DQ-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
+; AVX512DQ-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm6, %ymm9
+; AVX512DQ-NEXT:    vmovdqa64 %ymm10, %ymm31
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm13[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm9 = ymm9[0,1],ymm10[2],ymm9[3,4],ymm10[5],ymm9[6,7,8,9],ymm10[10],ymm9[11,12],ymm10[13],ymm9[14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm9 = ymm6[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[2,2,2,2,6,6,6,6]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm13[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm14[0,1],ymm10[2],ymm14[3,4],ymm10[5],ymm14[6,7,8,9],ymm10[10],ymm14[11,12],ymm10[13],ymm14[14,15]
-; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} zmm14 = [2,1,3,2,10,10,10,11]
-; AVX512DQ-NEXT:    vpermi2q %zmm10, %zmm12, %zmm14
-; AVX512DQ-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
-; AVX512DQ-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm18, %ymm14
-; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm14, %ymm10
-; AVX512DQ-NEXT:    vmovdqa64 %ymm12, %ymm19
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm9[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7,8,9],ymm12[10],ymm10[11,12],ymm12[13],ymm10[14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm14[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm10[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm9[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0],ymm10[1],ymm12[2,3],ymm10[4],ymm12[5,6,7,8],ymm10[9],ymm12[10,11],ymm10[12],ymm12[13,14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm8[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm10[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm7[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0],ymm10[1],ymm12[2,3],ymm10[4],ymm12[5,6,7,8],ymm10[9],ymm12[10,11],ymm10[12],ymm12[13,14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm8[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm10[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm7[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0,1,2],ymm10[3],ymm12[4,5],ymm10[6],ymm12[7,8,9,10],ymm10[11],ymm12[12,13],ymm10[14],ymm12[15]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5,6,7,8],ymm9[9],ymm10[10,11],ymm9[12],ymm10[13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm9 = ymm8[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm7[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5,6,7,8],ymm9[9],ymm10[10,11],ymm9[12],ymm10[13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm9 = ymm8[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm7[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3],ymm10[4,5],ymm9[6],ymm10[7,8,9,10],ymm9[11],ymm10[12,13],ymm9[14],ymm10[15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,1,3,3]
+; AVX512DQ-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 96(%r9), %ymm9
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm9[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[3,3,3,3]
 ; AVX512DQ-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[0,1,1,3,4,5,5,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm14[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm10[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm9 = ymm9[0,1],ymm10[2],ymm9[3,4],ymm10[5],ymm9[6,7,8,9],ymm10[10],ymm9[11,12],ymm10[13],ymm9[14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,1,3,2]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm9[2,1,2,3,6,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm10[0,0,3,3,4,5,6,7,8,8,11,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm30 = ymm10[2,2,2,2]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm13[0,1,1,3,4,5,5,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,0,0,0,4,4,4,4]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm10[0,1],ymm6[2],ymm10[3,4],ymm6[5],ymm10[6,7,8,9],ymm6[10],ymm10[11,12],ymm6[13],ymm10[14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,1,3,2]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[1,1,1,1,5,5,5,5]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm8[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm8 = ymm8[0,0,2,1,4,4,6,5]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm7 = ymm8[0,1],ymm7[2],ymm8[3,4],ymm7[5],ymm8[6,7,8,9],ymm7[10],ymm8[11,12],ymm7[13],ymm8[14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm16, %zmm8
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm17, %zmm7
-; AVX512DQ-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm7
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm26 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vmovdqa 96(%r8), %ymm10
+; AVX512DQ-NEXT:    vprold $16, %ymm9, %ymm8
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm9, %ymm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm17, %zmm6
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm18, %zmm7
+; AVX512DQ-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm7
+; AVX512DQ-NEXT:    vmovdqa 96(%r8), %ymm6
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm6[1,2,2,3,5,6,6,7]
 ; AVX512DQ-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm10[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm10[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm10[16,17,u,u,u,u],zero,zero
-; AVX512DQ-NEXT:    vpternlogq $248, %ymm26, %ymm7, %ymm8
-; AVX512DQ-NEXT:    vmovdqa 96(%r9), %ymm12
-; AVX512DQ-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm12, %ymm9
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq $248, %ymm14, %ymm8, %ymm9
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm7, %ymm7
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm8 = ymm10[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,1,3,3]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm6[0,0,2,1,4,4,6,5]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[16,17,u,u,u,u],zero,zero
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $248, %ymm11, %ymm7, %ymm6
+; AVX512DQ-NEXT:    vmovdqa64 %zmm11, %zmm12
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $248, %ymm11, %ymm6, %ymm9
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm7, %ymm6
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm10[2,1,3,3]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm7, %ymm10, %ymm8
-; AVX512DQ-NEXT:    vprold $16, %ymm12, %ymm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,2]
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm8, %ymm14, %ymm7
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm7
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm9[0,1,2,3],zmm7[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpternlogq $184, %ymm6, %ymm10, %ymm7
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm8[2,2,2,2]
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $184, %ymm7, %ymm8, %ymm6
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm6
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm9[0,1,2,3],zmm6[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,5,0,0,0,6,0,0,6,0,0,0,7,0,0,7]
-; AVX512DQ-NEXT:    vmovdqa 96(%rax), %ymm7
-; AVX512DQ-NEXT:    vpermd %zmm7, %zmm18, %zmm9
-; AVX512DQ-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm7[0,1,1,3,4,5,5,7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm10 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm7, %ymm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,2,3]
-; AVX512DQ-NEXT:    vpandnq %ymm9, %ymm26, %ymm9
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm7, %zmm7
+; AVX512DQ-NEXT:    vmovdqa 96(%rax), %ymm6
+; AVX512DQ-NEXT:    vpermd %zmm6, %zmm18, %zmm7
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512DQ-NEXT:    vpbroadcastd 72(%rax), %ymm7
-; AVX512DQ-NEXT:    vpandnq %ymm7, %ymm16, %ymm9
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm7 = ymm6[0,1,1,3,4,5,5,7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm6, %ymm6
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,2,2,3]
+; AVX512DQ-NEXT:    vpandn %ymm7, %ymm12, %ymm7
+; AVX512DQ-NEXT:    vmovdqa64 %zmm12, %zmm19
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512DQ-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
+; AVX512DQ-NEXT:    vpbroadcastd 72(%rax), %ymm6
+; AVX512DQ-NEXT:    vpandnq %ymm6, %ymm28, %ymm6
 ; AVX512DQ-NEXT:    vmovdqa 64(%rax), %ymm7
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm7, %ymm12
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm9, %zmm9
-; AVX512DQ-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpbroadcastd 8(%rax), %ymm9
-; AVX512DQ-NEXT:    vpandnq %ymm9, %ymm16, %ymm9
-; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm8
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm8, %ymm12
-; AVX512DQ-NEXT:    vmovdqa64 %ymm8, %ymm24
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm9, %zmm9
-; AVX512DQ-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm2[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[0,1,1,3,4,5,5,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm9 = ymm12[0,1],ymm9[2],ymm12[3,4],ymm9[5],ymm12[6,7,8,9],ymm9[10],ymm12[11,12],ymm9[13],ymm12[14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm0[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm11[1,1,1,1,5,5,5,5]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm9 = ymm9[0,1],ymm12[2],ymm9[3,4],ymm12[5],ymm9[6,7,8,9],ymm12[10],ymm9[11,12],ymm12[13],ymm9[14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%rax), %ymm9
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm9, %ymm10
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm9[0,1,1,3,4,5,5,7]
+; AVX512DQ-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm7, %ymm7
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512DQ-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 64(%r9), %xmm7
+; AVX512DQ-NEXT:    vmovdqa 64(%r8), %xmm8
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm17
+; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm20
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm6, %xmm6
+; AVX512DQ-NEXT:    vmovdqa %xmm7, %xmm15
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,1,1]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %xmm9
+; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %xmm7
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm7[4],xmm9[4],xmm7[5],xmm9[5],xmm7[6],xmm9[6],xmm7[7],xmm9[7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,2,3,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,2,1]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm14
+; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %xmm10
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm10[4],xmm14[4],xmm10[5],xmm14[5],xmm10[6],xmm14[6],xmm10[7],xmm14[7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[2,1,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,5,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,1,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpbroadcastd 8(%rax), %ymm8
+; AVX512DQ-NEXT:    vpandnq %ymm8, %ymm28, %ymm8
+; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm12
+; AVX512DQ-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm12, %ymm13
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm8, %zmm8
+; AVX512DQ-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm6
+; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm12
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm12[4],xmm6[4],xmm12[5],xmm6[5],xmm12[6],xmm6[6],xmm12[7],xmm6[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm12, %xmm29
+; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm24
+; AVX512DQ-NEXT:    vpshufb %xmm15, %xmm8, %xmm8
+; AVX512DQ-NEXT:    vmovdqa64 %xmm15, %xmm25
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,1,1]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm8
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm13
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm13[4],xmm8[4],xmm13[5],xmm8[5],xmm13[6],xmm8[6],xmm13[7],xmm8[7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[0,2,3,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm15[0,0,2,1]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm6
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm12
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm12[4],xmm6[4],xmm12[5],xmm6[5],xmm12[6],xmm6[6],xmm12[7],xmm6[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm12, %xmm21
+; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm16
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[2,1,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,5,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm15[0,0,1,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm15 = ymm4[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm15 = ymm15[0,0,0,0,4,4,4,4]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm2[0,1,1,3,4,5,5,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm15[2],ymm12[3,4],ymm15[5],ymm12[6,7,8,9],ymm15[10],ymm12[11,12],ymm15[13],ymm12[14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,1,3,2]
+; AVX512DQ-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm12 = ymm5[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm12[0,0,2,1,4,4,6,5]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm15 = ymm3[1,1,1,1,5,5,5,5]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm15[2],ymm12[3,4],ymm15[5],ymm12[6,7,8,9],ymm15[10],ymm12[11,12],ymm15[13],ymm12[14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 32(%rax), %ymm15
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm15, %ymm11
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm15[0,1,1,3,4,5,5,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,2,3]
-; AVX512DQ-NEXT:    vpandnq %ymm12, %ymm26, %ymm12
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
-; AVX512DQ-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm13[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm12 = ymm15[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm12[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm8 = ymm12[0,1,2],ymm10[3],ymm12[4,5],ymm10[6],ymm12[7,8,9,10],ymm10[11],ymm12[12,13],ymm10[14],ymm12[15]
-; AVX512DQ-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm13
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm2, %ymm10
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm8 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7,8,9],ymm12[10],ymm10[11,12],ymm12[13],ymm10[14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpandnq %ymm12, %ymm19, %ymm12
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
+; AVX512DQ-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm11 = ymm0[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm11 = ymm11[0,0,0,0,4,4,4,4]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[0,0,2,1,4,4,6,5]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm11 = ymm11[0,1,2],ymm12[3],ymm11[4,5],ymm12[6],ymm11[7,8,9,10],ymm12[11],ymm11[12,13],ymm12[14],ymm11[15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,3,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa64 %ymm31, %ymm6
+; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm4, %ymm11
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm11 = ymm11[0,1],ymm12[2],ymm11[3,4],ymm12[5],ymm11[6,7,8,9],ymm12[10],ymm11[11,12],ymm12[13],ymm11[14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2,3],ymm4[4],ymm2[5,6,7,8],ymm4[9],ymm2[10,11],ymm4[12],ymm2[13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm5[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6,7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm0[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm31
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm11[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3],ymm2[4],ymm4[5,6,7,8],ymm2[9],ymm4[10,11],ymm2[12],ymm4[13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm3[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm5[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8,9,10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,3]
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vprold $16, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,2,2,3,5,6,6,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7,8,9],ymm2[10],ymm3[11,12],ymm2[13],ymm3[14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,2]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
 ; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpermd %zmm9, %zmm18, %zmm0
+; AVX512DQ-NEXT:    vpermd %zmm15, %zmm18, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm5[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm23, %ymm12
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm12[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm4[1,1,1,1,5,5,5,5]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm22[1,1,1,1,5,5,5,5]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm5[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm12[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm4[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm22[2,2,2,2,6,6,6,6]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
 ; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 %ymm23, %ymm2
+; AVX512DQ-NEXT:    vmovdqa64 %ymm26, %ymm2
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm2[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm6[0,1,1,3,4,5,5,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm27[0,1,1,3,4,5,5,7]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm30
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm2, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm23, %ymm3
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm6[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,2]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vmovdqa64 %ymm26, %ymm3
+; AVX512DQ-NEXT:    vmovdqa64 %ymm31, %ymm11
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm27[2,2,2,2,6,6,6,6]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
 ; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 %ymm21, %ymm8
-; AVX512DQ-NEXT:    vprold $16, %ymm21, %ymm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm20[1,2,2,3,5,6,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm8[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm20[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7,8,9,10],ymm2[11],ymm1[12,13],ymm2[14],ymm1[15]
-; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} zmm10 = [2,2,3,3,10,9,11,10]
-; AVX512DQ-NEXT:    vpermt2q %zmm0, %zmm10, %zmm1
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm11 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
-; AVX512DQ-NEXT:    vpermd 64(%rax), %zmm11, %zmm0
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm26, %zmm0
+; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm5[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[0,0,2,1,4,4,6,5]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7,8,9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,3,3]
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm6[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vprold $16, %ymm5, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[1,2,2,3,5,6,6,7]
+; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm6
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,2]
+; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm27[3,3,3,3,7,7,7,7]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm3[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6,7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm4[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm5[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7,8,9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
-; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm20[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm8[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm12 = [4,5,4,5,4,5,6,7,16,17,16,17,16,17,17,19]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %xmm0
+; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %xmm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm22[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm12[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8,9,10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %xmm2
+; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm3
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm12
+; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm4, %xmm4
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,1,1]
+; AVX512DQ-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm6[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
+; AVX512DQ-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vprold $16, %xmm2, %xmm4
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,2,3]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]
+; AVX512DQ-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} xmm3 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm0, %xmm2
+; AVX512DQ-NEXT:    vmovdqa %xmm3, %xmm4
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,2]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2,3],xmm2[4],xmm3[5,6],xmm2[7]
+; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 96(%r9), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa 96(%r8), %xmm1
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm2[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpermt2d %zmm4, %zmm12, %zmm3
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm7[2,3,3,3,6,7,7,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,1,3,2]
-; AVX512DQ-NEXT:    vpbroadcastd 96(%rax), %ymm5
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm3, %zmm5, %zmm4
-; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm15
-; AVX512DQ-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %xmm3
-; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm4
-; AVX512DQ-NEXT:    vprold $16, %xmm3, %xmm5
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,3]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2],xmm6[3,4],xmm5[5],xmm6[6,7]
-; AVX512DQ-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm23
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; AVX512DQ-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %xmm3
-; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %xmm4
-; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} xmm6 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm5
-; AVX512DQ-NEXT:    vmovdqa %xmm6, %xmm7
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,2]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3],xmm5[4],xmm6[5,6],xmm5[7]
-; AVX512DQ-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512DQ-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,6]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [16,18,19,19,19,19,0,0,0,1,0,1,2,3,2,3]
-; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm29, %zmm1
-; AVX512DQ-NEXT:    vpbroadcastd 100(%rax), %ymm2
-; AVX512DQ-NEXT:    vpbroadcastd 104(%rax), %ymm3
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm14, %zmm2
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %xmm1
-; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %xmm2
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm27
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vmovdqa %xmm7, %xmm8
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,2]
+; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [16,18,19,19,19,19,0,0,0,1,0,1,2,3,2,3]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5,7,6]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm18, %zmm0
+; AVX512DQ-NEXT:    vpbroadcastd 100(%rax), %ymm1
+; AVX512DQ-NEXT:    vpbroadcastd 104(%rax), %ymm2
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm31
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm31
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
+; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm9, %xmm1
+; AVX512DQ-NEXT:    vmovdqa %xmm4, %xmm6
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm7[1,1,2,2]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm3[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [16,16,17,17,17,17,0,0,0,1,0,1,2,3,2,3]
-; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm3, %zmm1
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm2
-; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,1,1]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm25
-; AVX512DQ-NEXT:    vprold $16, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2],xmm2[3,4],xmm4[5],xmm2[6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm5, %zmm2
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm1, %zmm4, %zmm2
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 64(%r9), %xmm1
-; AVX512DQ-NEXT:    vmovdqa 64(%r8), %xmm2
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm17
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,5,7,6]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
-; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm6, %zmm1
-; AVX512DQ-NEXT:    vpbroadcastd 64(%rax), %ymm2
-; AVX512DQ-NEXT:    vpbroadcastd 68(%rax), %ymm5
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm9, %zmm2
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm1
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm5
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm20
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vmovdqa %xmm8, %xmm14
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3],xmm1[4],xmm5[5,6],xmm1[7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm7[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpermt2d %zmm5, %zmm3, %zmm1
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm3
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm5
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,1,1]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm19
-; AVX512DQ-NEXT:    vprold $16, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm5[2],xmm3[3,4],xmm5[5],xmm3[6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [16,16,17,17,17,17,0,0,0,1,0,1,2,3,2,3]
+; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm2, %zmm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
+; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,1]
+; AVX512DQ-NEXT:    vprold $16, %xmm10, %xmm3
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm14[1,1,2,3]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm7, %zmm2
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm1, %zmm4, %zmm2
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm3
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm14
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm1, %zmm0, %zmm14
+; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm3
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,5,7,6]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm6, %zmm1
-; AVX512DQ-NEXT:    vpbroadcastd (%rax), %ymm3
-; AVX512DQ-NEXT:    vpbroadcastd 4(%rax), %ymm4
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm2
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm9, %zmm2
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 %ymm28, %ymm7
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm7[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm22, %ymm2
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm22[1,1,1,1,5,5,5,5]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm22
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm7[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm2[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm9
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm4[0],ymm1[1],ymm4[2,3],ymm1[4],ymm4[5,6,7,8],ymm1[9],ymm4[10,11],ymm1[12],ymm4[13,14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm21
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
+; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm4, %zmm1
+; AVX512DQ-NEXT:    vpbroadcastd 64(%rax), %ymm3
+; AVX512DQ-NEXT:    vpbroadcastd 68(%rax), %ymm5
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm25
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm19, %zmm25
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm8, %xmm3
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm13[1,1,2,2]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0],xmm3[1],xmm5[2,3],xmm3[4],xmm5[5,6],xmm3[7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm2, %zmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm2
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,1]
+; AVX512DQ-NEXT:    vprold $16, %xmm21, %xmm2
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm16[1,1,2,3]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2],xmm5[3,4],xmm2[5],xmm5[6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm5
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm3, %zmm0, %zmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm0
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5,7,6]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm4, %zmm0
+; AVX512DQ-NEXT:    vpbroadcastd (%rax), %ymm1
+; AVX512DQ-NEXT:    vpbroadcastd 4(%rax), %ymm2
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm20
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm0, %zmm19, %zmm20
+; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm6[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5]
+; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[1,1,1,1,5,5,5,5]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm6[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm10
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512DQ-NEXT:    vprold $16, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm2[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,2,2,3,5,6,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm4[0,1],ymm1[2],ymm4[3,4],ymm1[5],ymm4[6,7,8,9],ymm1[10],ymm4[11,12],ymm1[13],ymm4[14,15]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm2[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm3[0,0,2,1,4,4,6,5]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm6[3],ymm4[4,5],ymm6[6],ymm4[7,8,9,10],ymm6[11],ymm4[12,13],ymm6[14],ymm4[15]
-; AVX512DQ-NEXT:    vpermt2q %zmm1, %zmm10, %zmm4
-; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm8[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
-; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm10[0,1,1,3,4,5,5,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm6[0,1],ymm1[2],ymm6[3,4],ymm1[5],ymm6[6,7,8,9],ymm1[10],ymm6[11,12],ymm1[13],ymm6[14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm16
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm8, %ymm1
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm10[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm6[2],ymm1[3,4],ymm6[5],ymm1[6,7,8,9],ymm6[10],ymm1[11,12],ymm6[13],ymm1[14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm18
-; AVX512DQ-NEXT:    vpermd (%rax), %zmm11, %zmm1
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm4, %zmm26, %zmm1
-; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm10[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm8[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm4[1],ymm1[2,3],ymm4[4],ymm1[5,6,7,8],ymm4[9],ymm1[10,11],ymm4[12],ymm1[13,14,15]
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm9[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm7[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1,2],ymm4[3],ymm1[4,5],ymm4[6],ymm1[7,8,9,10],ymm4[11],ymm1[12,13],ymm4[14],ymm1[15]
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[3,3,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm4 = ymm2[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm4[0,1],ymm1[2],ymm4[3,4],ymm1[5],ymm4[6,7,8,9],ymm1[10],ymm4[11,12],ymm1[13],ymm4[14,15]
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm7
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm6[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpermt2d %zmm11, %zmm12, %zmm1
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm11 = ymm24[2,3,3,3,6,7,7,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,1,3,2]
-; AVX512DQ-NEXT:    vpbroadcastd 32(%rax), %ymm12
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm26
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm15, %zmm26
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm1
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm12
-; AVX512DQ-NEXT:    vprold $16, %xmm12, %xmm15
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm13 = xmm1[1,1,2,3]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm13[0,1],xmm15[2],xmm13[3,4],xmm15[5],xmm13[6,7]
-; AVX512DQ-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm1[4],xmm12[5],xmm1[5],xmm12[6],xmm1[6],xmm12[7],xmm1[7]
-; AVX512DQ-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm13, %xmm1
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm24 = mem[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm23 = mem[0,2,2,3]
-; AVX512DQ-NEXT:    vpermq $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm7 = mem[0,2,2,3]
-; AVX512DQ-NEXT:    vpermq $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm17 = mem[2,1,3,3]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm6[0,1,2,3,4,5,7,6]
-; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm29, %zmm4
-; AVX512DQ-NEXT:    vpbroadcastd 36(%rax), %ymm0
-; AVX512DQ-NEXT:    vpbroadcastd 40(%rax), %ymm6
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm28
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm28
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm6, %xmm4
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm0[1,1,2,2]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm12[0],xmm4[1],xmm12[2,3],xmm4[4],xmm12[5,6],xmm4[7]
-; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
-; AVX512DQ-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
-; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm1[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm29 = ymm0[3,3,3,3]
-; AVX512DQ-NEXT:    vpshufd $233, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm0 = mem[1,2,2,3,5,6,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm14 = ymm1[2,1,2,3,6,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm14 = ymm14[0,0,3,3,4,5,6,7,8,8,11,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,2,2,2]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,1,1]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm1[0,2,3,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm1[2,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,5,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,1,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,1]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm1[0,2,3,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,0,2,1]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm19, %xmm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm1[2,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,5,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,1,3]
-; AVX512DQ-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm4 = mem[2,1,3,2]
-; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm5 = mem[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq $250, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm6 = mem[2,2,3,3]
-; AVX512DQ-NEXT:    vpermpd $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm1 = mem[2,2,2,3]
-; AVX512DQ-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpermpd $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm1 = mem[0,2,2,3]
-; AVX512DQ-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm31 = ymm31[0,2,2,3]
-; AVX512DQ-NEXT:    vpermq $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm27 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm27 = mem[2,1,3,3]
-; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm20 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm20 = mem[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm11 = mem[0,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm19 = ymm30[2,1,3,2]
-; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm1 = mem[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm25 = ymm22[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm21[0,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm16 = ymm16[2,1,3,2]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm18[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm23, %zmm24, %zmm23
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm17, %zmm7, %zmm24
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq $184, %zmm23, %zmm30, %zmm24
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[0,1,1,3,4,5,5,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,2]
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm24, %ymm7, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm0, %ymm18, %ymm14
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm0[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm2 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm0[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm10, %zmm23, %zmm8
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm17, %zmm8
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm15, %zmm2 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm10 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm23, %zmm10
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm3, %zmm17, %zmm10
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm9
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm3[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vmovdqa %ymm3, %ymm11
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
+; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm7[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm4[0,0,2,1,4,4,6,5]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm0[0,1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7,8,9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
+; AVX512DQ-NEXT:    vprold $16, %ymm7, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm4[1,2,2,3,5,6,6,7]
+; AVX512DQ-NEXT:    vmovdqa %ymm4, %ymm8
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7,8,9],ymm0[10],ymm2[11,12],ymm0[13],ymm2[14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm11[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm9[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6,7,8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm10[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[2,2,2,2,6,6,6,6]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm6[3],ymm0[4,5],ymm6[6],ymm0[7,8,9,10],ymm6[11],ymm0[12,13],ymm6[14],ymm0[15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm8[3,3,3,3,7,7,7,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm7 = ymm7[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7,8,9],ymm6[10],ymm7[11,12],ymm6[13],ymm7[14,15]
+; AVX512DQ-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm22 = ymm1[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm26 = ymm3[2,2,3,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm24 = ymm4[2,1,3,2]
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm9
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm10
+; AVX512DQ-NEXT:    vprold $16, %xmm10, %xmm1
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm9[1,1,2,3]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]
+; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm21 = ymm2[0,2,2,3]
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm1
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm2
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
+; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm2, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm6[0,1,2,3,4,5,7,6]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm27
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm18, %zmm1
+; AVX512DQ-NEXT:    vpbroadcastd 36(%rax), %ymm2
+; AVX512DQ-NEXT:    vpbroadcastd 40(%rax), %ymm4
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm13
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm13
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm7
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm6
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm4[0,0,1,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm0[2,1,3,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm3[0,0,1,1]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm2
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm21
-; AVX512DQ-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm21
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm24, %ymm2
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd $226, 124(%r8){1to8}, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm2, %ymm3, %ymm29
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm22 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm3, %zmm4, %zmm1
+; AVX512DQ-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm3 = mem[2,1,3,2]
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $184, %ymm1, %ymm29, %ymm3
+; AVX512DQ-NEXT:    vpternlogq $184, %ymm3, %ymm28, %ymm30
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm24 = zmm0[0,1,2,3],zmm6[0,1,2,3]
-; AVX512DQ-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm24 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm29, %zmm0, %zmm2
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm14[0,1,2,3],zmm2[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm2, %zmm0, %zmm29
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm7 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm18 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm17 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm20, %zmm2
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm19, %zmm3
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm4, %zmm3
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm25, %zmm2
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm16, %zmm5
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm4, %zmm5
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq $184, %zmm3, %zmm2, %zmm20
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq $184, %zmm5, %zmm2, %zmm25
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm3 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm27, %zmm31, %zmm3
-; AVX512DQ-NEXT:    vpternlogq $184, %zmm2, %zmm30, %zmm3
-; AVX512DQ-NEXT:    vpermq $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm2 = mem[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw $180, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm4 = mem[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpermq $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm4 = mem[2,1,3,3]
-; AVX512DQ-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm5 = mem[0,0,1,1]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm5 = mem[0,0,2,1]
-; AVX512DQ-NEXT:    vpshuflw $230, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm6 = mem[2,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5,5,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,1,3]
-; AVX512DQ-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm19 = mem[0,0,1,1]
-; AVX512DQ-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm9 = mem[0,2,3,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]
-; AVX512DQ-NEXT:    vpermq $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm11 = mem[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw $180, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm12 = mem[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[0,0,1,1]
-; AVX512DQ-NEXT:    vpermq $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm13 = mem[2,1,3,3]
-; AVX512DQ-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm14 = mem[0,0,1,1]
-; AVX512DQ-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm15 = mem[0,0,2,1]
-; AVX512DQ-NEXT:    vpshuflw $230, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm0 = mem[2,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,5,4]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,3]
-; AVX512DQ-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm16 = mem[0,0,1,1]
-; AVX512DQ-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm1 = mem[0,2,3,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm30, %zmm4
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm12[0,1,1,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm11, %zmm2
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm11
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm30, %zmm11
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq $184, %zmm4, %zmm2, %zmm12
-; AVX512DQ-NEXT:    vpternlogq $184, %zmm11, %zmm2, %zmm26
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm2
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm19, %zmm4
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm23, %zmm4
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm15, %zmm0
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm16, %zmm1
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm0, %zmm23, %zmm1
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm0 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm23 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm18 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm23, %zmm28, %zmm18
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm3, %zmm2, %zmm18
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm23 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm3, %zmm28, %zmm23
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm0, %zmm2, %zmm23
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm17 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm17
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
+; AVX512DQ-NEXT:    vpternlogd $226, 124(%r8){1to8}, %ymm19, %ymm0
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
+; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-NEXT:    vpternlogq $184, %ymm0, %ymm1, %ymm2
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512DQ-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm19 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[0,1,2,3]
+; AVX512DQ-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm30[0,1,2,3],zmm1[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm2, %zmm30
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm2, %zmm11
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm14, %zmm1, %zmm25
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm5, %zmm1, %zmm20
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq $184, %zmm4, %zmm0, %zmm2
-; AVX512DQ-NEXT:    vpternlogq $184, %zmm1, %zmm0, %zmm28
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm21, %zmm24
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm7
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm1, %zmm5, %zmm2
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm22, %zmm3, %zmm22
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm1, %zmm5, %zmm22
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm14 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
+; AVX512DQ-NEXT:    vpermd 64(%rax), %zmm14, %zmm5
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm3, %zmm5
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm24, %zmm26, %zmm1
+; AVX512DQ-NEXT:    vpermd (%rax), %zmm14, %zmm14
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm3, %zmm14
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm2, %zmm1, %zmm5
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm22, %zmm1, %zmm14
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm1, %zmm4, %zmm2
+; AVX512DQ-NEXT:    vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm1 = mem[0,1,1,3]
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm22 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm1, %zmm4, %zmm22
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm12[0,1,1,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm21, %zmm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm15, %zmm8
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm1, %zmm4, %zmm8
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm7[u,u,4,5,u,u,u,u,6,7,u,u,u,u,8,9]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm6[1,1,2,2]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm9[0],xmm4[1],xmm9[2,3],xmm4[4],xmm9[5,6],xmm4[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]
+; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm7 = mem[2,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw $180, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm9 = mem[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[0,1,1,3]
+; AVX512DQ-NEXT:    vpshufd $254, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm10 = mem[2,3,3,3,6,7,7,7]
+; AVX512DQ-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm16 = mem[0,0,2,1]
+; AVX512DQ-NEXT:    vpshuflw $230, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm12 = mem[2,1,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,5,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,1,3]
+; AVX512DQ-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm21 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm21 = mem[0,0,1,1]
+; AVX512DQ-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm15 = mem[0,2,3,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,0,2,1]
+; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm24 = mem[2,2,2,3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm3
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,1,1,3]
+; AVX512DQ-NEXT:    vpshufd $254, {{[-0-9]+}}(%r{{[sb]}}p), %ymm26 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm26 = mem[2,3,3,3,6,7,7,7]
+; AVX512DQ-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm27 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm27 = mem[0,0,2,1]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,4]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,1,1]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,2,3,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,0,1]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm7, %zmm7
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm10[2,1,3,2]
+; AVX512DQ-NEXT:    vpbroadcastd 96(%rax), %ymm10
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm7, %zmm29, %zmm9
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,0,1]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm24, %zmm3
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm26[2,1,3,2]
+; AVX512DQ-NEXT:    vpbroadcastd 32(%rax), %ymm10
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm7, %zmm7
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm3, %zmm29, %zmm7
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm22, %zmm3, %zmm9
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm8, %zmm3, %zmm7
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm16, %zmm3
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm21, %zmm8
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm3, %zmm28, %zmm8
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm27, %zmm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm3
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm1, %zmm28, %zmm3
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm8, %zmm1, %zmm31
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm3, %zmm1, %zmm13
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm17, %zmm0
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm11
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vmovdqa64 %zmm24, 320(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm28, 256(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm26, 192(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm25, 128(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm10, 64(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm17, (%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm18, 448(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, 704(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm12, 640(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm20, 576(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm8, 512(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm7, 384(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm22, 768(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm29, 832(%rax)
-; AVX512DQ-NEXT:    addq $2168, %rsp # imm = 0x878
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 320(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm13, 256(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm7, 192(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm14, 128(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm23, 64(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm20, (%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm25, 448(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm31, 704(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm9, 640(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, 576(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm18, 512(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm11, 384(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm19, 768(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm30, 832(%rax)
+; AVX512DQ-NEXT:    addq $2808, %rsp # imm = 0xAF8
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i16_stride7_vf64:
 ; AVX512DQ-FCP:       # %bb.0:
 ; AVX512DQ-FCP-NEXT:    subq $1496, %rsp # imm = 0x5D8
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm15 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm20
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %ymm9
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm9, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm16
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %ymm6
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm6, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %ymm7
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm3, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm1, %ymm2
+; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm2, %ymm16
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm4, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %ymm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm7, %ymm3
-; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm17
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm2
+; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm2, %ymm7
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm2, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm26
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm12, %ymm28
+; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm5 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm0, %ymm4
-; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm4, %ymm18
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm0, %ymm8
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm0, %ymm14
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm31
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm12
-; AVX512DQ-FCP-NEXT:    vpor %ymm8, %ymm12, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm0, %ymm15
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm21
+; AVX512DQ-FCP-NEXT:    vpor %ymm14, %ymm15, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm1, %ymm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm28
-; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm14
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm29
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm15
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm17
+; AVX512DQ-FCP-NEXT:    vpor %ymm14, %ymm15, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm1, %ymm13
-; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm13, %ymm19
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm14
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm15
+; AVX512DQ-FCP-NEXT:    vpor %ymm14, %ymm15, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm10, %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm13
-; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm13, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm0, %ymm14
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm0, %ymm12
+; AVX512DQ-FCP-NEXT:    vpor %ymm14, %ymm12, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm1, %ymm13
-; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm13, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm10
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm10, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm14
+; AVX512DQ-FCP-NEXT:    vpor %ymm12, %ymm14, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm14
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm14, %ymm8
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm15
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm15, %ymm13
-; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm8
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm8, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm4
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm11
-; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm11, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm13
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm14
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm14, %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm3, %ymm12
-; AVX512DQ-FCP-NEXT:    vporq %ymm5, %ymm0, %ymm21
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm2
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm11 = ymm9[0,1,1,3,4,5,5,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm11[0,1],ymm0[2],ymm11[3,4],ymm0[5],ymm11[6,7,8,9],ymm0[10],ymm11[11,12],ymm0[13],ymm11[14,15]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,2]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[1,1,1,1,5,5,5,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm11[0,1],ymm3[2],ymm11[3,4],ymm3[5],ymm11[6,7,8,9],ymm3[10],ymm11[11,12],ymm3[13],ymm11[14,15]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm16, %zmm0
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm17, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %ymm11
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm11[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm11[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm11[16,17,u,u,u,u],zero,zero
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, %ymm25, %ymm3, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm5, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, %ymm12, %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm3, %ymm0
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [0,4,0,0,0,5,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm11, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm0, %ymm17, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm17, %zmm22
-; AVX512DQ-FCP-NEXT:    vprold $16, %ymm5, %ymm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm3, %ymm5, %ymm0
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm15, %ymm9
+; AVX512DQ-FCP-NEXT:    vpor %ymm8, %ymm9, %ymm8
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm8
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm2, %ymm9
+; AVX512DQ-FCP-NEXT:    vpor %ymm8, %ymm9, %ymm8
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm8
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm8, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm9
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm9, %ymm11
+; AVX512DQ-FCP-NEXT:    vporq %ymm11, %ymm10, %ymm22
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm3[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[0,1,1,3,4,5,5,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm11 = ymm12[0,1],ymm11[2],ymm12[3,4],ymm11[5],ymm12[6,7,8,9],ymm11[10],ymm12[11,12],ymm11[13],ymm12[14,15]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,1,3,2]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
+; AVX512DQ-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm4, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm13, %ymm30
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm13 = ymm5[1,1,1,1,5,5,5,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm13[2],ymm12[3,4],ymm13[5],ymm12[6,7,8,9],ymm13[10],ymm12[11,12],ymm13[13],ymm12[14,15]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm16, %zmm11
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm7, %zmm12
+; AVX512DQ-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm12
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %ymm7
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm7[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[16,17,u,u,u,u],zero,zero
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, %ymm13, %ymm12, %ymm11
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %ymm10
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm10, %ymm6
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, %ymm16, %ymm11, %ymm6
+; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm12, %ymm11
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [0,4,0,0,0,5,0,0]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm12, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm11, %ymm16, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, %zmm19
+; AVX512DQ-FCP-NEXT:    vprold $16, %ymm10, %ymm11
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm12, %ymm10, %ymm11
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm11
+; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm11[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,1,4,5,4,5,5,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rax), %ymm0
-; AVX512DQ-FCP-NEXT:    vpermd %ymm0, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vpandnq %ymm1, %ymm25, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm5 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm0, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm5, %ymm20
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [0,1,4,5,4,5,5,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rax), %ymm6
+; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm11, %ymm11
+; AVX512DQ-FCP-NEXT:    vpandn %ymm11, %ymm13, %ymm11
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, %zmm18
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm6, %ymm12
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm11
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm4, %ymm11
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm5[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3],ymm12[4,5],ymm11[6],ymm12[7,8,9,10],ymm11[11],ymm12[12,13],ymm11[14],ymm12[15]
+; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm12 = [151522058,0,421010202,421010202]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm12, %ymm23
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6,7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13,14,15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [0,2,2,3,10,9,11,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm11, %zmm5, %zmm4
+; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm12 = [218894094,0,488382238,488382238]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm11
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm12, %ymm16
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm11 = ymm12[0],ymm11[1],ymm12[2,3],ymm11[4],ymm12[5,6,7,8],ymm11[9],ymm12[10,11],ymm11[12],ymm12[13,14,15]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
+; AVX512DQ-FCP-NEXT:    # ymm12 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0,1],ymm1[2],ymm3[3,4],ymm1[5],ymm3[6,7,8,9],ymm1[10],ymm3[11,12],ymm1[13],ymm3[14,15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,2,2,3,8,10,10,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm11, %zmm20, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm27, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm6, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm26
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5],ymm1[6],ymm3[7,8,9,10],ymm1[11],ymm3[12,13],ymm1[14],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm5 = [151522058,0,421010202,421010202]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm6, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm5, %ymm29
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm7[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm6[0],ymm3[1],ymm6[2,3],ymm3[4],ymm6[5,6,7,8],ymm3[9],ymm6[10,11],ymm3[12],ymm6[13,14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [0,2,2,3,10,9,11,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm6, %zmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm7 = [218894094,0,488382238,488382238]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm7, %ymm27
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm9[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm7[0],ymm1[1],ymm7[2,3],ymm1[4],ymm7[5,6,7,8],ymm1[9],ymm7[10,11],ymm1[12],ymm7[13,14,15]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
-; AVX512DQ-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm2, %ymm7
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm5, %ymm16
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1],ymm9[2],ymm7[3,4],ymm9[5],ymm7[6,7,8,9],ymm9[10],ymm7[11,12],ymm9[13],ymm7[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm17 = [2,2,2,3,8,10,10,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm17, %zmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm1, %zmm7
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, %zmm24
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [5,0,0,0,6,0,0,6]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm11, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm1, %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa64 64(%rax), %zmm12
+; AVX512DQ-FCP-NEXT:    vmovdqa64 64(%rax), %zmm1
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,5,0,0,0,6,0,0,30,0,0,0,31,0,0,31]
-; AVX512DQ-FCP-NEXT:    vpermi2d %zmm12, %zmm0, %zmm3
+; AVX512DQ-FCP-NEXT:    vpermi2d %zmm1, %zmm6, %zmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 72(%rax), %ymm0
-; AVX512DQ-FCP-NEXT:    vpandnq %ymm0, %ymm23, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 72(%rax), %ymm3
+; AVX512DQ-FCP-NEXT:    vpandn %ymm3, %ymm6, %ymm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rax), %ymm7
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm5
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm7, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm7, %ymm30
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm11
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %xmm2
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm20
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [0,0,1,1,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, %zmm23, %zmm3, %zmm11
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 8(%rax), %ymm0
-; AVX512DQ-FCP-NEXT:    vpandnq %ymm0, %ymm23, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm1, %ymm3
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm5
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, %zmm23, %zmm3, %zmm5
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm14, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm13[0,0,2,1,4,4,6,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5],ymm3[6],ymm0[7,8,9,10],ymm3[11],ymm0[12,13],ymm3[14],ymm0[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,1,0,3,10,10,11,11]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm0, %zmm21, %zmm3
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm7, %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm7, %ymm25
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm26, %ymm9
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 8(%rax), %ymm3
+; AVX512DQ-FCP-NEXT:    vpandn %ymm3, %ymm6, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm4
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm4[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3],ymm3[4,5],ymm0[6],ymm3[7,8,9,10],ymm0[11],ymm3[12,13],ymm0[14],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm8, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm4[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3],ymm3[4],ymm5[5,6,7,8],ymm3[9],ymm5[10,11],ymm3[12],ymm5[13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm6, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm27, %ymm11
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm9, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm4, %ymm24
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[0,0,2,1,4,4,6,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8,9,10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,1,0,3,10,10,11,11]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm3, %zmm22, %zmm4
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm2[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5],ymm3[6],ymm4[7,8,9,10],ymm3[11],ymm4[12,13],ymm3[14],ymm4[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm23, %ymm7
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm0, %ymm4
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm2[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm6[0],ymm4[1],ymm6[2,3],ymm4[4],ymm6[5,6,7,8],ymm4[9],ymm6[10,11],ymm4[12],ymm6[13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm5, %zmm4
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm16, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm14, %ymm3
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm15[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2,3],ymm0[4],ymm5[5,6,7,8],ymm0[9],ymm5[10,11],ymm0[12],ymm5[13,14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm16, %ymm4
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm10, %ymm5
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3],ymm3[4],ymm5[5,6,7,8],ymm3[9],ymm5[10,11],ymm3[12],ymm5[13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm14, %ymm5
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm15[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7,8,9],ymm6[10],ymm5[11,12],ymm6[13],ymm5[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm17, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm24, %zmm16
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm24, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512DQ-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm14, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm13[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7,8,9],ymm3[10],ymm0[11,12],ymm3[13],ymm0[14,15]
-; AVX512DQ-FCP-NEXT:    vprold $16, %ymm14, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm13[1,2,2,3,5,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm3[2],ymm5[3,4],ymm3[5],ymm5[6,7,8,9],ymm3[10],ymm5[11,12],ymm3[13],ymm5[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [2,1,3,2,10,10,10,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rax), %zmm19
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,21,0,0,0,22,0,0,14,0,0,0,15,0,0,15]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7,8,9],ymm6[10],ymm5[11,12],ymm6[13],ymm5[14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm0
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm27, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7,8,9],ymm4[10],ymm3[11,12],ymm4[13],ymm3[14,15]
+; AVX512DQ-FCP-NEXT:    vprold $16, %ymm9, %ymm4
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm8[1,2,2,3,5,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm11 = [2,1,3,2,10,10,10,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm11, %zmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rax), %zmm16
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,21,0,0,0,22,0,0,14,0,0,0,15,0,0,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpermi2d %zmm0, %zmm19, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm22, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm28[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1],ymm3[2,3],ymm0[4],ymm3[5,6,7,8],ymm0[9],ymm3[10,11],ymm0[12],ymm3[13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm28[1,1,1,1,5,5,5,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm5[2],ymm3[3,4],ymm5[5],ymm3[6,7,8,9],ymm5[10],ymm3[11,12],ymm5[13],ymm3[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm17, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqu (%rsp), %ymm1 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm5[2],ymm0[3,4],ymm5[5],ymm0[6,7,8,9],ymm5[10],ymm0[11,12],ymm5[13],ymm0[14,15]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm1[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm2[0,1,1,3,4,5,5,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, %ymm4
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7,8,9],ymm5[10],ymm6[11,12],ymm5[13],ymm6[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vprold $16, %ymm8, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm2[1,2,2,3,5,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1],ymm0[2],ymm3[3,4],ymm0[5],ymm3[6,7,8,9],ymm0[10],ymm3[11,12],ymm0[13],ymm3[14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm8, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm2[0,0,2,1,4,4,6,5]
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, %ymm14
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm6[3],ymm3[4,5],ymm6[6],ymm3[7,8,9,10],ymm6[11],ymm3[12,13],ymm6[14],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [2,2,3,3,10,9,11,10]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm2, %zmm3
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm30, %zmm12, %zmm0
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm2, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm25, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm0
+; AVX512DQ-FCP-NEXT:    vpermi2d %zmm0, %zmm16, %zmm3
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm4, %zmm19, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm10
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm10, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm17, %ymm19
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm17[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6,7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13,14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm9
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm10, %ymm4
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm17[1,1,1,1,5,5,5,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm31, %ymm8
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm8, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm21[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7,8,9],ymm4[10],ymm3[11,12],ymm4[13],ymm3[14,15]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm8[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm21[0,1,1,3,4,5,5,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm4[2],ymm6[3,4],ymm4[5],ymm6[6,7,8,9],ymm4[10],ymm6[11,12],ymm4[13],ymm6[14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm11, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm5, %zmm27, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm26, %ymm12
+; AVX512DQ-FCP-NEXT:    vprold $16, %ymm26, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm22
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm28[1,2,2,3,5,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm3[2],ymm5[3,4],ymm3[5],ymm5[6,7,8,9],ymm3[10],ymm5[11,12],ymm3[13],ymm5[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm24, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm12, %ymm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm26, %ymm30
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm28[0,0,2,1,4,4,6,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1,2],ymm7[3],ymm5[4,5],ymm7[6],ymm5[7,8,9,10],ymm7[11],ymm5[12,13],ymm7[14],ymm5[15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [2,2,3,3,10,9,11,10]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm23, %zmm5
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm25, %zmm1, %zmm1
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
+; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm5, %zmm18, %zmm0
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm1, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm4[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm21[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6,7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm29
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, %xmm7
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm31 = [0,2,2,3,8,9,9,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm31, %zmm1
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm28[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3],ymm2[4,5],ymm0[6],ymm2[7,8,9,10],ymm0[11],ymm2[12,13],ymm0[14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm4, %xmm26
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm24
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [2,1,3,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm3, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm1, %zmm16, %zmm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm8, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm14[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm13 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %xmm1
-; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm4 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm2, %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, %xmm8
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm1[1,1,2,2]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3],xmm5[4],xmm6[5,6],xmm5[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, %xmm14
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm1, %xmm2
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,1,1,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm1, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, %zmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, (%rsp) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %xmm2
+; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm5 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm5, %xmm14
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[1,1,2,2]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0],xmm3[1],xmm5[2,3],xmm3[4],xmm5[5,6],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm0 = [0,1,1,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %xmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vprold $16, %xmm6, %xmm1
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm1[2],xmm5[3,4],xmm1[5],xmm5[6,7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm9, %xmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [0,0,1,1,8,8,10,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm27, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm5, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm20, %xmm2
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3]
+; AVX512DQ-FCP-NEXT:    vprold $16, %xmm6, %xmm3
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2],xmm5[3,4],xmm3[5],xmm5[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm5, %xmm5
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,0,1,1,8,8,10,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm28, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm3, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %xmm6
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm10 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm2, %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm2, %xmm6
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [0,0,0,1,8,9,9,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm16, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm9
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm6
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm9[1,1,2,2]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm8[0],xmm6[1],xmm8[2,3],xmm6[4],xmm8[5,6],xmm6[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm8, %xmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm14, %xmm21
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm11, %zmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vprold $16, %xmm11, %xmm6
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm4[1,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm9[0,1],xmm6[2],xmm9[3,4],xmm6[5],xmm9[6,7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm9, %xmm9
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm28
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm27, %zmm9
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm8, %zmm5, %zmm9
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 64(%rax), %ymm5
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 68(%rax), %ymm6
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm23
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm4, %zmm23
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm26 = [0,0,0,1,8,9,9,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm26, %zmm2
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 64(%rax), %ymm6
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 68(%rax), %ymm8
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm31
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm21, %zmm31
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm1, %zmm2, %zmm23
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm1, %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm10, %xmm22
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, %xmm3
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm16, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm5, %zmm2, %zmm31
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm8
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm0, %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm14, %xmm29
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm8[1,1,2,2]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3],xmm5[4],xmm6[5,6],xmm5[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm6, %xmm6
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm13, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm13
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vprold $16, %xmm13, %xmm5
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[1,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm8[0,1],xmm5[2],xmm8[3,4],xmm5[5],xmm8[6,7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm8, %xmm8
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm28, %zmm8
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm6, %zmm3, %zmm8
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm5
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm3, %xmm5
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm7, %xmm17
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm26, %zmm3
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd (%rax), %ymm5
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm6
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm20
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm1, %zmm4, %zmm20
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm9, %zmm2, %zmm20
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm4[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm17[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm17[1,1,1,1,5,5,5,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm5[2],ymm2[3,4],ymm5[5],ymm2[6,7,8,9],ymm5[10],ymm2[11,12],ymm5[13],ymm2[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,2,2,3,8,10,10,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm5, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm26
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm21, %zmm26
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm8, %zmm2, %zmm26
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm14[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm6[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm14, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm6[1,1,1,1,5,5,5,5]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm6, %ymm9
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm5[2],ymm3[3,4],ymm5[5],ymm3[6,7,8,9],ymm5[10],ymm3[11,12],ymm5[13],ymm3[14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm20, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm7[2,2,2,2,6,6,6,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm5[2],ymm1[3,4],ymm5[5],ymm1[6,7,8,9],ymm5[10],ymm1[11,12],ymm5[13],ymm1[14,15]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm9[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm5[2],ymm2[3,4],ymm5[5],ymm2[6,7,8,9],ymm5[10],ymm2[11,12],ymm5[13],ymm2[14,15]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm7[0,1,1,3,4,5,5,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm7, %ymm15
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7,8,9],ymm5[10],ymm6[11,12],ymm5[13],ymm6[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm18, %zmm8
-; AVX512DQ-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm8
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vprold $16, %ymm5, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm7, %ymm18
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7,8,9],ymm5[10],ymm6[11,12],ymm5[13],ymm6[14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm11, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %xmm11
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %xmm7
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [0,2,2,3,8,9,9,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm6, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm10, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm19[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm8[0,1,2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7,8,9,10],ymm2[11],ymm8[12,13],ymm2[14],ymm8[15]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm27, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vprold $16, %ymm4, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm19[1,2,2,3,5,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7,8,9],ymm2[10],ymm3[11,12],ymm2[13],ymm3[14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm4, %ymm24
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm19[0,0,2,1,4,4,6,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm3[0,1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8,9,10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm23, %zmm8
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,1,3,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm20, %zmm10
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm1, %zmm21, %zmm10
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm22[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm23, %zmm16, %zmm1
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
+; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm16, %zmm22
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm22
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm22
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm1, %xmm1
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm18[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2,3],ymm0[4],ymm5[5,6,7,8],ymm0[9],ymm5[10,11],ymm0[12],ymm5[13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm6, %zmm0
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [2,2,2,3,8,8,8,9]
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %xmm5
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm17, %xmm1
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm8, %xmm1
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm30, %zmm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm14, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm9[0,1,2],ymm1[3],ymm9[4,5],ymm1[6],ymm9[7,8,9,10],ymm1[11],ymm9[12,13],ymm1[14],ymm9[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm14, %xmm14
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm14, %zmm20, %zmm1
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [6,7,3,3,7,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm25, %ymm17, %ymm16
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 96(%rax), %ymm20
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm16, %zmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm18, %zmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm10, %zmm27, %zmm25
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; AVX512DQ-FCP-NEXT:    vprold $16, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm21, %zmm1
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [0,0,2,1,8,8,9,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm21, %zmm3
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm29, %xmm2
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,2,2]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0],xmm2[1],xmm7[2,3],xmm2[4],xmm7[5,6],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm28, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm28, %zmm7
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm16[1,2,2,3,5,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7,8,9],ymm1[10],ymm2[11,12],ymm1[13],ymm2[14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm5, %ymm18
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm16[0,0,2,1,4,4,6,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm5[3],ymm2[4,5],ymm5[6],ymm2[7,8,9,10],ymm5[11],ymm2[12,13],ymm5[14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,2,3,3,10,9,11,10]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm5, %zmm2
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,2,2,3,8,8,8,9]
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %xmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %xmm7
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm10, %xmm1
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm5, %zmm13
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm19, %zmm1
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm5, %zmm19
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm25, %zmm19
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm19
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm12
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm8
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm12[0],xmm8[1],xmm12[1],xmm8[2],xmm12[2],xmm8[3],xmm12[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm2
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm9[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm9 = ymm15[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm15 = ymm9[0],ymm2[1],ymm9[2,3],ymm2[4],ymm9[5,6,7,8],ymm2[9],ymm9[10,11],ymm2[12],ymm9[13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm31, %zmm15
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm31 = [6,7,3,3,7,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm30, %ymm31, %ymm1
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 96(%rax), %ymm2
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm21
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm13, %zmm30, %zmm21
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm21
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm1
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vprold $16, %xmm26, %xmm1
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm24[1,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm17[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8,9,10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm4
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm6
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm13, %xmm13
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [2,1,3,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm13, %zmm2, %zmm1
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm17 = [0,0,2,1,8,8,9,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm17, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm0, %ymm18, %ymm16
+; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm0, %ymm13
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm8[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,1,1,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm3, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm24, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm19[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7,8,9],ymm3[10],ymm2[11,12],ymm3[13],ymm2[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm11
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm11[0],xmm3[0],xmm11[1],xmm3[1],xmm11[2],xmm3[2],xmm11[3],xmm3[3]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm2[0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm30, %zmm8
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 100(%rax), %ymm5
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 104(%rax), %ymm19
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm5, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm6, %zmm24, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm4[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[3,3,3,3]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25,22,23,22,23,u,u,u,u,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm10, %xmm14
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,1,1]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm10, %xmm10
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm12, %xmm30
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm23, %ymm17, %ymm17
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm19
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm17, %zmm20
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm8, %zmm18, %zmm20
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm1, %zmm27, %zmm20
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm7, %zmm1, %zmm5
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm15[4],xmm9[5],xmm15[5],xmm9[6],xmm15[6],xmm9[7],xmm15[7]
+; AVX512DQ-FCP-NEXT:    vprold $16, %xmm9, %xmm8
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm15[1,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm8 = xmm9[0,1],xmm8[2],xmm9[3,4],xmm8[5],xmm9[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm7, %xmm7
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm7, %zmm21, %zmm8
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm29, %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm24 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm13
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[1,1,2,2]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm9[0],xmm0[1],xmm9[2,3],xmm0[4],xmm9[5,6],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,2,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm27, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm25, %zmm0
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm3, %ymm30, %ymm7
-; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm3, %ymm11
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm13
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm10, %xmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,1,1,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm28, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm6[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm26 = ymm2[3,3,3,3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25,22,23,22,23,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm29 = ymm2[2,2,2,2]
-; AVX512DQ-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm1
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 100(%rax), %ymm2
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 104(%rax), %ymm6
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm22, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm18, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm16[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm15
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm6[0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [2,2,2,3,8,8,8,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm9, %zmm16, %zmm10
-; AVX512DQ-FCP-NEXT:    vpermd %ymm14, %ymm31, %ymm9
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm18
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm9, %zmm9
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm10, %zmm30, %zmm9
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm9
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm0, %zmm1, %zmm2
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
-; AVX512DQ-FCP-NEXT:    vprold $16, %xmm4, %xmm4
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm10 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm17, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm12, %xmm0
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm8[4],xmm12[4],xmm8[5],xmm12[5],xmm8[6],xmm12[6],xmm8[7],xmm12[7]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[1,1,2,2]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm8[0],xmm0[1],xmm8[2,3],xmm0[4],xmm8[5,6],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,2,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm27, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm25, %zmm0
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm15[4],xmm3[5],xmm15[5],xmm3[6],xmm15[6],xmm3[7],xmm15[7]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm4
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm28, %zmm4
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm9, %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm12 = xmm0[1,1,2,2]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm12[0],xmm7[1],xmm12[2,3],xmm7[4],xmm12[5,6],xmm7[7]
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,2,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm0 = [0,0,1,1,8,8,10,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm9, %zmm0, %zmm7
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm8, %zmm28, %zmm7
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm30, %xmm0
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm0 = [0,1,1,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm3
-; AVX512DQ-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm5
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm4, %zmm22, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm0, %zmm1, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm7, %ymm0, %ymm29
+; AVX512DQ-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm8
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm3, %zmm3
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm24, %zmm3
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm7, %zmm1, %zmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd $226, 124(%r8){1to8}, %ymm0, %ymm11
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm11, %ymm0, %ymm26
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm26, %zmm0, %zmm1
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm29[0,1,2,3],zmm1[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm12
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpckhwd (%rsp), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
+; AVX512DQ-FCP-NEXT:    vpternlogd $226, 124(%r8){1to8}, %ymm0, %ymm13
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm13, %ymm1, %ymm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm16, %ymm2, %ymm4
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm13
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm0 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm0 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm2 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpunpckhwd (%rsp), %xmm4, %xmm4 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm5 = xmm5[4],mem[4],xmm5[5],mem[5],xmm5[6],mem[6],xmm5[7],mem[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm5, %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm6[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512DQ-FCP-NEXT:    vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # ymm7 = mem[0,1,1,3,4,5,5,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7,8,9],ymm6[10],ymm7[11,12],ymm6[13],ymm7[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512DQ-FCP-NEXT:    vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # ymm8 = mem[1,1,1,1,5,5,5,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1],ymm8[2],ymm7[3,4],ymm8[5],ymm7[6,7,8,9],ymm8[10],ymm7[11,12],ymm8[13],ymm7[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [0,1,4,5,4,5,5,7]
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpermd %ymm10, %ymm8, %ymm8
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[16,17],zero,zero
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpandn %ymm8, %ymm11, %ymm8
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm10, %zmm8
-; AVX512DQ-FCP-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm10 = mem[0,2,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,3]
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,2,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,1,3]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,1,3,2]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,3]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # xmm6 = xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # xmm7 = xmm7[4],mem[4],xmm7[5],mem[5],xmm7[6],mem[6],xmm7[7],mem[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm4, %xmm4
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm7, %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm8[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512DQ-FCP-NEXT:    vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # ymm9 = mem[0,1,1,3,4,5,5,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm9[0,1],ymm8[2],ymm9[3,4],ymm8[5],ymm9[6,7,8,9],ymm8[10],ymm9[11,12],ymm8[13],ymm9[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
+; AVX512DQ-FCP-NEXT:    vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # ymm10 = mem[1,1,1,1,5,5,5,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm9 = ymm9[0,1],ymm10[2],ymm9[3,4],ymm10[5],ymm9[6,7,8,9],ymm10[10],ymm9[11,12],ymm10[13],ymm9[14,15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,1,4,5,4,5,5,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm13
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm10, %zmm25, %zmm1
+; AVX512DQ-FCP-NEXT:    vpermd %ymm11, %ymm10, %ymm10
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[16,17],zero,zero
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpandn %ymm10, %ymm12, %ymm10
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm11, %zmm10
+; AVX512DQ-FCP-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # xmm11 = mem[0,2,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,1,3]
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,2,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,1,3]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,1,3,2]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm14
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm4, %zmm25, %zmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm1 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm6
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm8 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm8
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm11, %zmm28, %zmm4
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm6, %zmm28, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm6, %zmm4
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm6, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm2, %zmm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm10 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm10
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm8, 320(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, 320(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 256(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm9, 192(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm19, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm20, (%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm23, 448(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 704(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm21, 640(%rax)
-; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, 384(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 64(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 512(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, 832(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 768(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm20, 192(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm22, 128(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm26, (%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm31, 448(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 704(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm25, 640(%rax)
+; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovaps %zmm1, 576(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm14, 384(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 64(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 512(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, 832(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 768(%rax)
 ; AVX512DQ-FCP-NEXT:    addq $1496, %rsp # imm = 0x5D8
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
index f56c43eb49df2..06d390f053c7e 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
@@ -4879,12 +4879,10 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
 ; AVX512-NEXT:    vpternlogq $226, %zmm5, %zmm8, %zmm7
 ; AVX512-NEXT:    vpternlogq $248, %zmm28, %zmm7, %zmm2
-; AVX512-NEXT:    vpermq {{.*#+}} zmm5 = zmm24[2,2,3,3,6,6,7,7]
-; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm26[2,2,3,3,6,6,7,7]
-; AVX512-NEXT:    vporq %zmm5, %zmm7, %zmm5
-; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm25[2,2,3,3,6,6,7,7]
-; AVX512-NEXT:    vpermq {{.*#+}} zmm9 = zmm27[2,2,3,3,6,6,7,7]
-; AVX512-NEXT:    vporq %zmm7, %zmm9, %zmm7
+; AVX512-NEXT:    vporq %zmm24, %zmm26, %zmm5
+; AVX512-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[2,2,3,3,6,6,7,7]
+; AVX512-NEXT:    vporq %zmm25, %zmm27, %zmm7
+; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,2,3,3,6,6,7,7]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
 ; AVX512-NEXT:    vpternlogq $226, %zmm5, %zmm9, %zmm7
 ; AVX512-NEXT:    vpternlogd $184, %zmm7, %zmm29, %zmm30
@@ -4918,165 +4916,165 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP:       # %bb.0:
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm0, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm2
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19]
-; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm1, %ymm3
-; AVX512-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm17
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm3
+; AVX512-FCP-NEXT:    vporq %ymm1, %ymm3, %ymm17
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm3
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm3[8],zero,xmm3[u,7],zero,xmm3[9],zero,xmm3[u],zero,xmm3[u,10],zero,xmm3[12],zero,xmm3[u,11]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm31
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm15
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = zero,xmm15[8,u],zero,xmm15[7],zero,xmm15[9,u,11,u],zero,xmm15[10],zero,xmm15[12,u],zero
-; AVX512-FCP-NEXT:    vporq %xmm2, %xmm4, %xmm18
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm2
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm3[8],zero,xmm3[u,7],zero,xmm3[9],zero,xmm3[u],zero,xmm3[u,10],zero,xmm3[12],zero,xmm3[u,11]
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm5, %xmm4
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm6, %xmm28
+; AVX512-FCP-NEXT:    vporq %xmm1, %xmm4, %xmm18
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm1
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm2, %ymm4
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm4
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm11
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [u,u,12,13,128,u,u,u,14,128,u,u,14,15,128,u,u,u,16,128,u,u,16,17,128,u,u,u,18,128,u,u]
 ; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm11, %ymm6
 ; AVX512-FCP-NEXT:    vporq %ymm4, %ymm6, %ymm19
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm6
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12]
-; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm6, %xmm4
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm30
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = zero,xmm6[6],zero,xmm6[8,u],zero,xmm6[7],zero,xmm6[9],zero,xmm6[11,u],zero,xmm6[10],zero,xmm6[12]
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm7
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
-; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm7, %xmm8
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm29
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
+; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm7, %xmm8
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm13, %xmm29
 ; AVX512-FCP-NEXT:    vporq %xmm4, %xmm8, %xmm20
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[26],zero,ymm1[28],zero,zero,ymm1[27],zero,ymm1[29],zero,ymm1[31],zero,zero,ymm1[30],zero
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
+; AVX512-FCP-NEXT:    # ymm8 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm2, %ymm4
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm8, %ymm31
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128]
 ; AVX512-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm21
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30,27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30]
-; AVX512-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm0, %ymm4
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm28
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0,19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0]
-; AVX512-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm22
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm2, %ymm2
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm21
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30,27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30]
+; AVX512-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm0, %ymm8
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm30
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0,19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0]
+; AVX512-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm22
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
 ; AVX512-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm0
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128]
-; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm8
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm24
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm8
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128]
+; AVX512-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm1, %ymm15
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm15, %zmm24
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0,25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0]
 ; AVX512-FCP-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm2, %ymm0
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25]
-; AVX512-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm11, %ymm2
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm26
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm2
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm2, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25]
+; AVX512-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm11, %ymm11
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm11, %zmm26
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm10
 ; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm11
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm11, %ymm10
-; AVX512-FCP-NEXT:    vporq %ymm0, %ymm10, %ymm23
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm11, %ymm1
-; AVX512-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm25
+; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm11, %ymm12
+; AVX512-FCP-NEXT:    vporq %ymm10, %ymm12, %ymm23
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm11, %ymm10
+; AVX512-FCP-NEXT:    vporq %ymm0, %ymm10, %ymm25
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm10
 ; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm10, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm12
-; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm12, %ymm1
-; AVX512-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm27
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm12, %ymm9
+; AVX512-FCP-NEXT:    vporq %ymm0, %ymm9, %ymm27
 ; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm10, %ymm1
-; AVX512-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm16
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm2
+; AVX512-FCP-NEXT:    vporq %ymm0, %ymm2, %ymm16
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm9
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm9[8],zero,xmm9[u,7],zero,xmm9[9],zero,xmm9[u],zero,xmm9[u,10],zero,xmm9[12],zero,xmm9[u,11]
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm14
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,xmm14[8,u],zero,xmm14[7],zero,xmm14[9,u,11,u],zero,xmm14[10],zero,xmm14[12,u],zero
-; AVX512-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm28, %xmm2
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm14, %xmm2
+; AVX512-FCP-NEXT:    vporq %xmm0, %xmm2, %xmm28
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm13
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm30, %xmm0
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm13, %xmm1
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm29, %xmm5
-; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm5
-; AVX512-FCP-NEXT:    vpor %xmm1, %xmm5, %xmm5
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm1
-; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm11
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm12[26],zero,ymm12[28],zero,zero,ymm12[27],zero,ymm12[29],zero,ymm12[31],zero,zero,ymm12[30],zero
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm28, %ymm2
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = zero,xmm13[6],zero,xmm13[8,u],zero,xmm13[7],zero,xmm13[9],zero,xmm13[11,u],zero,xmm13[10],zero,xmm13[12]
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm15
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm29, %xmm2
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm15, %xmm2
+; AVX512-FCP-NEXT:    vporq %xmm0, %xmm2, %xmm29
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm31, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm12, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm30, %ymm2
 ; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm8
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm4
 ; AVX512-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm8, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm4, %ymm2
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [1,1,2,2,2,2,2,2]
-; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm10, %ymm10
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
-; AVX512-FCP-NEXT:    vpandn %ymm10, %ymm12, %ymm10
+; AVX512-FCP-NEXT:    vpermd %ymm4, %ymm10, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
+; AVX512-FCP-NEXT:    vpandn %ymm10, %ymm11, %ymm10
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm10, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm10
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm28 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31]
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0]
-; AVX512-FCP-NEXT:    vpermd %ymm10, %ymm28, %ymm30
-; AVX512-FCP-NEXT:    vpandnq %ymm30, %ymm29, %ymm30
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm10, %ymm4
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm30, %zmm4, %zmm4
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm12 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31]
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0]
+; AVX512-FCP-NEXT:    vpermd %ymm10, %ymm12, %ymm31
+; AVX512-FCP-NEXT:    vpandnq %ymm31, %ymm30, %ymm31
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm10, %ymm8
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm31, %zmm8, %zmm8
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm9[0],xmm14[0],xmm9[1],xmm14[1],xmm9[2],xmm14[2],xmm9[3],xmm14[3],xmm9[4],xmm14[4],xmm9[5],xmm14[5],xmm9[6],xmm14[6],xmm9[7],xmm14[7]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
 ; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm9, %xmm9
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm3, %zmm9, %zmm3
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3],xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm5, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa64 (%r8), %zmm5
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm5, %zmm10
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm28, %zmm9, %zmm9
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3],xmm13[4],xmm15[4],xmm13[5],xmm15[5],xmm13[6],xmm15[6],xmm13[7],xmm15[7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm29, %zmm13, %zmm13
+; AVX512-FCP-NEXT:    vmovdqa64 (%r8), %zmm28
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm28, %zmm10
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm6, %xmm6
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [2,2,3,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm7, %zmm11
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm31, %xmm6
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm15[0],xmm6[1],xmm15[1],xmm6[2],xmm15[2],xmm6[3],xmm15[3],xmm6[4],xmm15[4],xmm6[5],xmm15[5],xmm6[6],xmm15[6],xmm6[7],xmm15[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm6, %xmm6
-; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm7, %zmm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm6 = zmm21[2,2,3,3,6,6,7,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm22[2,2,3,3,6,6,7,7]
-; AVX512-FCP-NEXT:    vporq %zmm6, %zmm7, %zmm6
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm24[2,2,3,3,6,6,7,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm26[2,2,3,3,6,6,7,7]
-; AVX512-FCP-NEXT:    vporq %zmm7, %zmm9, %zmm7
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm6, %zmm9, %zmm7
-; AVX512-FCP-NEXT:    vpermt2d %zmm5, %zmm28, %zmm8
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm7, %zmm29, %zmm8
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, 256(%r9)
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm18[0,0,1,1]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm6, %zmm6
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm20[0,0,1,1]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm7, %zmm7
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm6, %zmm8, %zmm7
-; AVX512-FCP-NEXT:    vpternlogq $248, %zmm12, %zmm7, %zmm2
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm25[2,2,3,3]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm23, %zmm6
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm16[2,2,3,3]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm27, %zmm7
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm6, %zmm8, %zmm7
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm11, %zmm9, %zmm1
-; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm4
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[0,0,1,1,4,4,5,5]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,0,1,1,4,4,5,5]
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm0
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,0,0,0,0,0,1,1,9,9,0,10,10,10,10,0]
-; AVX512-FCP-NEXT:    vpermd %zmm10, %zmm3, %zmm3
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [6,6,6,0,7,7,7,7,0,8,8,8,8,0,9,9]
-; AVX512-FCP-NEXT:    vpermd %zmm5, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 128(%r9)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 64(%r9)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, (%r9)
+; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm7, %zmm0
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm7, %zmm1
+; AVX512-FCP-NEXT:    vporq %zmm21, %zmm22, %zmm3
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[2,2,3,3,6,6,7,7]
+; AVX512-FCP-NEXT:    vporq %zmm24, %zmm26, %zmm5
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[2,2,3,3,6,6,7,7]
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm6, %zmm5
+; AVX512-FCP-NEXT:    vpermt2d %zmm28, %zmm12, %zmm4
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm5, %zmm30, %zmm4
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 256(%r9)
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm18[0,0,1,1]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm3, %zmm3
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm20[0,0,1,1]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm4, %zmm4
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm5, %zmm4
+; AVX512-FCP-NEXT:    vpternlogq $248, %zmm11, %zmm4, %zmm2
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm25[2,2,3,3]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm23, %zmm3
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm16[2,2,3,3]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm27, %zmm4
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm5, %zmm4
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm6, %zmm1
+; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm8
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm9[0,0,1,1,4,4,5,5]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm13[0,0,1,1,4,4,5,5]
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,0,0,1,1,9,9,0,10,10,10,10,0]
+; AVX512-FCP-NEXT:    vpermd %zmm10, %zmm0, %zmm0
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm0
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [6,6,6,0,7,7,7,7,0,8,8,8,8,0,9,9]
+; AVX512-FCP-NEXT:    vpermd %zmm28, %zmm3, %zmm3
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 128(%r9)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, 64(%r9)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 192(%r9)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
@@ -5224,12 +5222,10 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
 ; AVX512DQ-NEXT:    vpternlogq $226, %zmm5, %zmm8, %zmm7
 ; AVX512DQ-NEXT:    vpternlogq $248, %zmm28, %zmm7, %zmm2
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm5 = zmm24[2,2,3,3,6,6,7,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm26[2,2,3,3,6,6,7,7]
-; AVX512DQ-NEXT:    vporq %zmm5, %zmm7, %zmm5
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm25[2,2,3,3,6,6,7,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm9 = zmm27[2,2,3,3,6,6,7,7]
-; AVX512DQ-NEXT:    vporq %zmm7, %zmm9, %zmm7
+; AVX512DQ-NEXT:    vporq %zmm24, %zmm26, %zmm5
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[2,2,3,3,6,6,7,7]
+; AVX512DQ-NEXT:    vporq %zmm25, %zmm27, %zmm7
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,2,3,3,6,6,7,7]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
 ; AVX512DQ-NEXT:    vpternlogq $226, %zmm5, %zmm9, %zmm7
 ; AVX512DQ-NEXT:    vpternlogd $184, %zmm7, %zmm29, %zmm30
@@ -5263,165 +5259,165 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP:       # %bb.0:
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm0, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm1, %ymm3
-; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm17
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm3
+; AVX512DQ-FCP-NEXT:    vporq %ymm1, %ymm3, %ymm17
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm3
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm3[8],zero,xmm3[u,7],zero,xmm3[9],zero,xmm3[u],zero,xmm3[u,10],zero,xmm3[12],zero,xmm3[u,11]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm31
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm15
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = zero,xmm15[8,u],zero,xmm15[7],zero,xmm15[9,u,11,u],zero,xmm15[10],zero,xmm15[12,u],zero
-; AVX512DQ-FCP-NEXT:    vporq %xmm2, %xmm4, %xmm18
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm3[8],zero,xmm3[u,7],zero,xmm3[9],zero,xmm3[u],zero,xmm3[u,10],zero,xmm3[12],zero,xmm3[u,11]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm5, %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm6, %xmm28
+; AVX512DQ-FCP-NEXT:    vporq %xmm1, %xmm4, %xmm18
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm2, %ymm4
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm11
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [u,u,12,13,128,u,u,u,14,128,u,u,14,15,128,u,u,u,16,128,u,u,16,17,128,u,u,u,18,128,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm11, %ymm6
 ; AVX512DQ-FCP-NEXT:    vporq %ymm4, %ymm6, %ymm19
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm6, %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm30
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = zero,xmm6[6],zero,xmm6[8,u],zero,xmm6[7],zero,xmm6[9],zero,xmm6[11,u],zero,xmm6[10],zero,xmm6[12]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm7, %xmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm29
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm7, %xmm8
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm13, %xmm29
 ; AVX512DQ-FCP-NEXT:    vporq %xmm4, %xmm8, %xmm20
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[26],zero,ymm1[28],zero,zero,ymm1[27],zero,ymm1[29],zero,ymm1[31],zero,zero,ymm1[30],zero
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
+; AVX512DQ-FCP-NEXT:    # ymm8 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm2, %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm8, %ymm31
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128]
 ; AVX512DQ-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm21
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30,27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30]
-; AVX512DQ-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm0, %ymm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm28
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0,19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0]
-; AVX512DQ-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm22
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm21
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30,27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30]
+; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm0, %ymm8
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm30
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0,19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0]
+; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm22
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
 ; AVX512DQ-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm0
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128]
-; AVX512DQ-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm8
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm24
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm8
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128]
+; AVX512DQ-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm1, %ymm15
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm15, %zmm24
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0,25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0]
 ; AVX512DQ-FCP-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm2, %ymm0
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25]
-; AVX512DQ-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm11, %ymm2
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm26
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm2, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25]
+; AVX512DQ-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm11, %ymm11
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm11, %zmm26
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm10
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm11
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm11, %ymm10
-; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm10, %ymm23
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm11, %ymm1
-; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm25
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm11, %ymm12
+; AVX512DQ-FCP-NEXT:    vporq %ymm10, %ymm12, %ymm23
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm11, %ymm10
+; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm10, %ymm25
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm10
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm10, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm12
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm12, %ymm1
-; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm27
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm12, %ymm9
+; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm9, %ymm27
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm10, %ymm1
-; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm16
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm2
+; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm2, %ymm16
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm9
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm9[8],zero,xmm9[u,7],zero,xmm9[9],zero,xmm9[u],zero,xmm9[u,10],zero,xmm9[12],zero,xmm9[u,11]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm14
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,xmm14[8,u],zero,xmm14[7],zero,xmm14[9,u,11,u],zero,xmm14[10],zero,xmm14[12,u],zero
-; AVX512DQ-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm2
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm14, %xmm2
+; AVX512DQ-FCP-NEXT:    vporq %xmm0, %xmm2, %xmm28
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm30, %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm13, %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm29, %xmm5
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm5
-; AVX512DQ-FCP-NEXT:    vpor %xmm1, %xmm5, %xmm5
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm11
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm12[26],zero,ymm12[28],zero,zero,ymm12[27],zero,ymm12[29],zero,ymm12[31],zero,zero,ymm12[30],zero
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = zero,xmm13[6],zero,xmm13[8,u],zero,xmm13[7],zero,xmm13[9],zero,xmm13[11,u],zero,xmm13[10],zero,xmm13[12]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm29, %xmm2
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm15, %xmm2
+; AVX512DQ-FCP-NEXT:    vporq %xmm0, %xmm2, %xmm29
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm31, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm12, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm8
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm4
 ; AVX512DQ-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm8, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm4, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [1,1,2,2,2,2,2,2]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm10, %ymm10
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
-; AVX512DQ-FCP-NEXT:    vpandn %ymm10, %ymm12, %ymm10
+; AVX512DQ-FCP-NEXT:    vpermd %ymm4, %ymm10, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
+; AVX512DQ-FCP-NEXT:    vpandn %ymm10, %ymm11, %ymm10
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm10, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm10
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm28 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm10, %ymm28, %ymm30
-; AVX512DQ-FCP-NEXT:    vpandnq %ymm30, %ymm29, %ymm30
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm10, %ymm4
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm30, %zmm4, %zmm4
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm12 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm10, %ymm12, %ymm31
+; AVX512DQ-FCP-NEXT:    vpandnq %ymm31, %ymm30, %ymm31
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm10, %ymm8
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm31, %zmm8, %zmm8
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm9[0],xmm14[0],xmm9[1],xmm14[1],xmm9[2],xmm14[2],xmm9[3],xmm14[3],xmm9[4],xmm14[4],xmm9[5],xmm14[5],xmm9[6],xmm14[6],xmm9[7],xmm14[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm9, %xmm9
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm3, %zmm9, %zmm3
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3],xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm5, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 (%r8), %zmm5
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm5, %zmm10
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm28, %zmm9, %zmm9
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3],xmm13[4],xmm15[4],xmm13[5],xmm15[5],xmm13[6],xmm15[6],xmm13[7],xmm15[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm29, %zmm13, %zmm13
+; AVX512DQ-FCP-NEXT:    vmovdqa64 (%r8), %zmm28
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm28, %zmm10
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm6, %xmm6
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [2,2,3,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm7, %zmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm31, %xmm6
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm15[0],xmm6[1],xmm15[1],xmm6[2],xmm15[2],xmm6[3],xmm15[3],xmm6[4],xmm15[4],xmm6[5],xmm15[5],xmm6[6],xmm15[6],xmm6[7],xmm15[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm6, %xmm6
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm7, %zmm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm6 = zmm21[2,2,3,3,6,6,7,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm22[2,2,3,3,6,6,7,7]
-; AVX512DQ-FCP-NEXT:    vporq %zmm6, %zmm7, %zmm6
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm24[2,2,3,3,6,6,7,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm26[2,2,3,3,6,6,7,7]
-; AVX512DQ-FCP-NEXT:    vporq %zmm7, %zmm9, %zmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm6, %zmm9, %zmm7
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm5, %zmm28, %zmm8
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm7, %zmm29, %zmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm8, 256(%r9)
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm18[0,0,1,1]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm20[0,0,1,1]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm7, %zmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm6, %zmm8, %zmm7
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, %zmm12, %zmm7, %zmm2
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm25[2,2,3,3]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm23, %zmm6
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm16[2,2,3,3]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm27, %zmm7
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm6, %zmm8, %zmm7
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm11, %zmm9, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm4
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[0,0,1,1,4,4,5,5]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,0,1,1,4,4,5,5]
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm0
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,0,0,0,0,0,1,1,9,9,0,10,10,10,10,0]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm10, %zmm3, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [6,6,6,0,7,7,7,7,0,8,8,8,8,0,9,9]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm5, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 128(%r9)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 64(%r9)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, (%r9)
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm7, %zmm0
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm7, %zmm1
+; AVX512DQ-FCP-NEXT:    vporq %zmm21, %zmm22, %zmm3
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[2,2,3,3,6,6,7,7]
+; AVX512DQ-FCP-NEXT:    vporq %zmm24, %zmm26, %zmm5
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[2,2,3,3,6,6,7,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm6, %zmm5
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm28, %zmm12, %zmm4
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm5, %zmm30, %zmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 256(%r9)
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm18[0,0,1,1]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm3, %zmm3
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm20[0,0,1,1]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm4, %zmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm5, %zmm4
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, %zmm11, %zmm4, %zmm2
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm25[2,2,3,3]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm23, %zmm3
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm16[2,2,3,3]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm27, %zmm4
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm5, %zmm4
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm6, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm8
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm9[0,0,1,1,4,4,5,5]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm13[0,0,1,1,4,4,5,5]
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,0,0,1,1,9,9,0,10,10,10,10,0]
+; AVX512DQ-FCP-NEXT:    vpermd %zmm10, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm0
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [6,6,6,0,7,7,7,7,0,8,8,8,8,0,9,9]
+; AVX512DQ-FCP-NEXT:    vpermd %zmm28, %zmm3, %zmm3
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 128(%r9)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm8, 64(%r9)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 192(%r9)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
index e43aa56c96c28..60af864597f4f 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
@@ -5405,304 +5405,289 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ;
 ; AVX512-LABEL: store_i8_stride6_vf64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    subq $328, %rsp # imm = 0x148
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm5
-; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm2
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm10
-; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm3
-; AVX512-NEXT:    vmovdqa (%r9), %xmm11
-; AVX512-NEXT:    vmovdqa 32(%r9), %xmm7
-; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm8
-; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm9
-; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm6
-; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm12
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm14 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} xmm4 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512-NEXT:    vpshufb %xmm4, %xmm2, %xmm0
-; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm1
+; AVX512-NEXT:    subq $456, %rsp # imm = 0x1C8
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm6
+; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm3
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm7
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX512-NEXT:    vmovdqa (%r8), %xmm10
+; AVX512-NEXT:    vmovdqa 32(%r8), %xmm4
+; AVX512-NEXT:    vmovdqa (%r9), %xmm12
+; AVX512-NEXT:    vmovdqa 32(%r9), %xmm8
+; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm13
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} xmm9 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512-NEXT:    vpshufb %xmm9, %xmm3, %xmm0
+; AVX512-NEXT:    vpshufb %xmm9, %xmm2, %xmm1
 ; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm16
-; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm27
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm30
+; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm31
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,5]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [u,6,u,5,u,8,u,7,u,9,u,9,u,9,u,9]
-; AVX512-NEXT:    vmovdqa %xmm7, %xmm3
-; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,2,u,1,u,0,u,3,u,4,u,4,u,4,u,4]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm7
-; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm26
-; AVX512-NEXT:    vinserti32x4 $2, %xmm2, %zmm7, %zmm2
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufb %xmm4, %xmm5, %xmm2
-; AVX512-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3],xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7]
-; AVX512-NEXT:    vmovdqa64 %xmm10, %xmm29
-; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm30
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,3,2,1,4,5,6,7]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm4, %zmm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [6,u,5,u,8,u,7,u,9,u,9,u,9,u,9,u]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,u,1,u,0,u,3,u,4,u,4,u,4,u,4,u]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm29
+; AVX512-NEXT:    vinserti32x4 $2, %xmm2, %zmm3, %zmm2
 ; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufb %xmm0, %xmm11, %xmm0
-; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm11, %xmm31
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [u,6,u,5,u,8,u,7,u,9,u,9,u,9,u,9]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm4
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,2,u,1,u,0,u,3,u,4,u,4,u,4,u,4]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm8, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm27
+; AVX512-NEXT:    vinserti32x4 $2, %xmm4, %zmm5, %zmm4
+; AVX512-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm8
+; AVX512-NEXT:    vpshufb %xmm9, %xmm6, %xmm4
+; AVX512-NEXT:    vpshufb %xmm9, %xmm7, %xmm5
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15]
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX512-NEXT:    vmovdqa64 %xmm7, %xmm18
+; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm20
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,3,2,1,4,5,6,7]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,6,5]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm4
+; AVX512-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm4
+; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm11
+; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm0
+; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm1
+; AVX512-NEXT:    vmovdqa64 %xmm10, %xmm22
+; AVX512-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm0
+; AVX512-NEXT:    vpshufb %xmm3, %xmm12, %xmm1
+; AVX512-NEXT:    vmovdqa64 %xmm12, %xmm21
 ; AVX512-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512-NEXT:    vpshufb %ymm7, %ymm6, %ymm0
-; AVX512-NEXT:    vpshufb %ymm7, %ymm12, %ymm1
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm15 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512-NEXT:    vpshufb %ymm15, %ymm4, %ymm0
+; AVX512-NEXT:    vpshufb %ymm15, %ymm11, %ymm1
 ; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm12[8],ymm6[8],ymm12[9],ymm6[9],ymm12[10],ymm6[10],ymm12[11],ymm6[11],ymm12[12],ymm6[12],ymm12[13],ymm6[13],ymm12[14],ymm6[14],ymm12[15],ymm6[15],ymm12[24],ymm6[24],ymm12[25],ymm6[25],ymm12[26],ymm6[26],ymm12[27],ymm6[27],ymm12[28],ymm6[28],ymm12[29],ymm6[29],ymm12[30],ymm6[30],ymm12[31],ymm6[31]
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31]
-; AVX512-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm13, %ymm1, %ymm1
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm11[8],ymm4[8],ymm11[9],ymm4[9],ymm11[10],ymm4[10],ymm11[11],ymm4[11],ymm11[12],ymm4[12],ymm11[13],ymm4[13],ymm11[14],ymm4[14],ymm11[15],ymm4[15],ymm11[24],ymm4[24],ymm11[25],ymm4[25],ymm11[26],ymm4[26],ymm11[27],ymm4[27],ymm11[28],ymm4[28],ymm11[29],ymm4[29],ymm11[30],ymm4[30],ymm11[31],ymm4[31]
+; AVX512-NEXT:    vmovdqa64 %ymm4, %ymm19
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31]
+; AVX512-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512-NEXT:    vpshufb %ymm14, %ymm8, %ymm0
-; AVX512-NEXT:    vpshufb %ymm14, %ymm9, %ymm1
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15],ymm9[24],ymm8[24],ymm9[25],ymm8[25],ymm9[26],ymm8[26],ymm9[27],ymm8[27],ymm9[28],ymm8[28],ymm9[29],ymm8[29],ymm9[30],ymm8[30],ymm9[31],ymm8[31]
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31]
-; AVX512-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm0, %ymm2, %ymm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%r9), %ymm4
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15]
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm12 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512-NEXT:    vpshufb %ymm12, %ymm13, %ymm0
+; AVX512-NEXT:    vpshufb %ymm12, %ymm8, %ymm1
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm8[8],ymm13[8],ymm8[9],ymm13[9],ymm8[10],ymm13[10],ymm8[11],ymm13[11],ymm8[12],ymm13[12],ymm8[13],ymm13[13],ymm8[14],ymm13[14],ymm8[15],ymm13[15],ymm8[24],ymm13[24],ymm8[25],ymm13[25],ymm8[26],ymm13[26],ymm8[27],ymm13[27],ymm8[28],ymm13[28],ymm8[29],ymm13[29],ymm8[30],ymm13[30],ymm8[31],ymm13[31]
+; AVX512-NEXT:    vmovdqa64 %ymm13, %ymm17
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31]
+; AVX512-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm7, %ymm1, %ymm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 32(%r8), %ymm13
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0]
+; AVX512-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm5, %ymm13, %ymm0
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0]
+; AVX512-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm4, %ymm13, %ymm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 32(%r9), %ymm14
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15]
+; AVX512-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm3, %ymm14, %ymm0
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0]
 ; AVX512-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm2, %ymm4, %ymm10
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0]
-; AVX512-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm1, %ymm4, %ymm11
-; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm11, %zmm3
-; AVX512-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa (%rsi), %ymm15
-; AVX512-NEXT:    vmovdqa (%rdi), %ymm10
-; AVX512-NEXT:    vpshufb %ymm7, %ymm15, %ymm11
-; AVX512-NEXT:    vpshufb %ymm7, %ymm10, %ymm7
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm7[0],ymm11[0],ymm7[1],ymm11[1],ymm7[2],ymm11[2],ymm7[3],ymm11[3],ymm7[4],ymm11[4],ymm7[5],ymm11[5],ymm7[6],ymm11[6],ymm7[7],ymm11[7],ymm7[16],ymm11[16],ymm7[17],ymm11[17],ymm7[18],ymm11[18],ymm7[19],ymm11[19],ymm7[20],ymm11[20],ymm7[21],ymm11[21],ymm7[22],ymm11[22],ymm7[23],ymm11[23]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm10[8],ymm15[8],ymm10[9],ymm15[9],ymm10[10],ymm15[10],ymm10[11],ymm15[11],ymm10[12],ymm15[12],ymm10[13],ymm15[13],ymm10[14],ymm15[14],ymm10[15],ymm15[15],ymm10[24],ymm15[24],ymm10[25],ymm15[25],ymm10[26],ymm15[26],ymm10[27],ymm15[27],ymm10[28],ymm15[28],ymm10[29],ymm15[29],ymm10[30],ymm15[30],ymm10[31],ymm15[31]
-; AVX512-NEXT:    vpshufb %ymm13, %ymm11, %ymm11
-; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm7, %zmm25
-; AVX512-NEXT:    vmovdqa (%rcx), %ymm7
-; AVX512-NEXT:    vmovdqa (%rdx), %ymm11
-; AVX512-NEXT:    vpshufb %ymm14, %ymm7, %ymm13
-; AVX512-NEXT:    vpshufb %ymm14, %ymm11, %ymm5
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm13[0],ymm5[1],ymm13[1],ymm5[2],ymm13[2],ymm5[3],ymm13[3],ymm5[4],ymm13[4],ymm5[5],ymm13[5],ymm5[6],ymm13[6],ymm5[7],ymm13[7],ymm5[16],ymm13[16],ymm5[17],ymm13[17],ymm5[18],ymm13[18],ymm5[19],ymm13[19],ymm5[20],ymm13[20],ymm5[21],ymm13[21],ymm5[22],ymm13[22],ymm5[23],ymm13[23]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm13 = ymm11[8],ymm7[8],ymm11[9],ymm7[9],ymm11[10],ymm7[10],ymm11[11],ymm7[11],ymm11[12],ymm7[12],ymm11[13],ymm7[13],ymm11[14],ymm7[14],ymm11[15],ymm7[15],ymm11[24],ymm7[24],ymm11[25],ymm7[25],ymm11[26],ymm7[26],ymm11[27],ymm7[27],ymm11[28],ymm7[28],ymm11[29],ymm7[29],ymm11[30],ymm7[30],ymm11[31],ymm7[31]
-; AVX512-NEXT:    vpshufb %ymm0, %ymm13, %ymm0
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm5, %zmm24
-; AVX512-NEXT:    vmovdqa (%r9), %ymm3
-; AVX512-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
-; AVX512-NEXT:    vpshufb %ymm1, %ymm3, %ymm1
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm23
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm11[0],ymm7[0],ymm11[1],ymm7[1],ymm11[2],ymm7[2],ymm11[3],ymm7[3],ymm11[4],ymm7[4],ymm11[5],ymm7[5],ymm11[6],ymm7[6],ymm11[7],ymm7[7],ymm11[16],ymm7[16],ymm11[17],ymm7[17],ymm11[18],ymm7[18],ymm11[19],ymm7[19],ymm11[20],ymm7[20],ymm11[21],ymm7[21],ymm11[22],ymm7[22],ymm11[23],ymm7[23]
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm28
-; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm1
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm2
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm10[0],ymm15[0],ymm10[1],ymm15[1],ymm10[2],ymm15[2],ymm10[3],ymm15[3],ymm10[4],ymm15[4],ymm10[5],ymm15[5],ymm10[6],ymm15[6],ymm10[7],ymm15[7],ymm10[16],ymm15[16],ymm10[17],ymm15[17],ymm10[18],ymm15[18],ymm10[19],ymm15[19],ymm10[20],ymm15[20],ymm10[21],ymm15[21],ymm10[22],ymm15[22],ymm10[23],ymm15[23]
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm19
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm10[8],xmm2[8],xmm10[9],xmm2[9],xmm10[10],xmm2[10],xmm10[11],xmm2[11],xmm10[12],xmm2[12],xmm10[13],xmm2[13],xmm10[14],xmm2[14],xmm10[15],xmm2[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm15 = [10,11,8,9,6,7,12,13,14,15,14,15,14,15,14,15]
-; AVX512-NEXT:    vpshufb %xmm15, %xmm5, %xmm0
+; AVX512-NEXT:    vpshufb %ymm2, %ymm14, %ymm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm28
+; AVX512-NEXT:    vmovdqa (%rsi), %ymm1
+; AVX512-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512-NEXT:    vpshufb %ymm15, %ymm1, %ymm10
+; AVX512-NEXT:    vpshufb %ymm15, %ymm0, %ymm15
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm15[0],ymm10[0],ymm15[1],ymm10[1],ymm15[2],ymm10[2],ymm15[3],ymm10[3],ymm15[4],ymm10[4],ymm15[5],ymm10[5],ymm15[6],ymm10[6],ymm15[7],ymm10[7],ymm15[16],ymm10[16],ymm15[17],ymm10[17],ymm15[18],ymm10[18],ymm15[19],ymm10[19],ymm15[20],ymm10[20],ymm15[21],ymm10[21],ymm15[22],ymm10[22],ymm15[23],ymm10[23]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm15 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512-NEXT:    vpshufb %ymm6, %ymm15, %ymm6
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm10, %zmm26
+; AVX512-NEXT:    vmovdqa (%rcx), %ymm10
+; AVX512-NEXT:    vmovdqa (%rdx), %ymm15
+; AVX512-NEXT:    vpshufb %ymm12, %ymm10, %ymm6
+; AVX512-NEXT:    vpshufb %ymm12, %ymm15, %ymm9
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm9[0],ymm6[0],ymm9[1],ymm6[1],ymm9[2],ymm6[2],ymm9[3],ymm6[3],ymm9[4],ymm6[4],ymm9[5],ymm6[5],ymm9[6],ymm6[6],ymm9[7],ymm6[7],ymm9[16],ymm6[16],ymm9[17],ymm6[17],ymm9[18],ymm6[18],ymm9[19],ymm6[19],ymm9[20],ymm6[20],ymm9[21],ymm6[21],ymm9[22],ymm6[22],ymm9[23],ymm6[23]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm15[8],ymm10[8],ymm15[9],ymm10[9],ymm15[10],ymm10[10],ymm15[11],ymm10[11],ymm15[12],ymm10[12],ymm15[13],ymm10[13],ymm15[14],ymm10[14],ymm15[15],ymm10[15],ymm15[24],ymm10[24],ymm15[25],ymm10[25],ymm15[26],ymm10[26],ymm15[27],ymm10[27],ymm15[28],ymm10[28],ymm15[29],ymm10[29],ymm15[30],ymm10[30],ymm15[31],ymm10[31]
+; AVX512-NEXT:    vpshufb %ymm7, %ymm9, %ymm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm6, %zmm24
+; AVX512-NEXT:    vmovdqa (%r8), %ymm6
+; AVX512-NEXT:    vpshufb %ymm5, %ymm6, %ymm5
+; AVX512-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm23
+; AVX512-NEXT:    vmovdqa (%r9), %ymm4
+; AVX512-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512-NEXT:    vpshufb %ymm2, %ymm4, %ymm2
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm25
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm15[0],ymm10[0],ymm15[1],ymm10[1],ymm15[2],ymm10[2],ymm15[3],ymm10[3],ymm15[4],ymm10[4],ymm15[5],ymm10[5],ymm15[6],ymm10[6],ymm15[7],ymm10[7],ymm15[16],ymm10[16],ymm15[17],ymm10[17],ymm15[18],ymm10[18],ymm15[19],ymm10[19],ymm15[20],ymm10[20],ymm15[21],ymm10[21],ymm15[22],ymm10[22],ymm15[23],ymm10[23]
+; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm16
+; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm2
+; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm3
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm18
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm15
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm2
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm15[8],xmm2[9],xmm15[9],xmm2[10],xmm15[10],xmm2[11],xmm15[11],xmm2[12],xmm15[12],xmm2[13],xmm15[13],xmm2[14],xmm15[14],xmm2[15],xmm15[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [10,11,8,9,6,7,12,13,14,15,14,15,14,15,14,15]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm10
 ; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm1
 ; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512-NEXT:    vpshufb %xmm15, %xmm5, %xmm5
-; AVX512-NEXT:    vmovdqa64 %ymm5, %ymm29
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[16],ymm8[16],ymm9[17],ymm8[17],ymm9[18],ymm8[18],ymm9[19],ymm8[19],ymm9[20],ymm8[20],ymm9[21],ymm8[21],ymm9[22],ymm8[22],ymm9[23],ymm8[23]
-; AVX512-NEXT:    vmovdqa64 %ymm5, %ymm30
-; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm5
-; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm8
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm5[8],xmm8[8],xmm5[9],xmm8[9],xmm5[10],xmm8[10],xmm5[11],xmm8[11],xmm5[12],xmm8[12],xmm5[13],xmm8[13],xmm5[14],xmm8[14],xmm5[15],xmm8[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm9 = [8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15]
-; AVX512-NEXT:    vpshufb %xmm9, %xmm7, %xmm11
-; AVX512-NEXT:    vpshufb %xmm9, %xmm8, %xmm5
-; AVX512-NEXT:    vmovdqa64 %ymm5, %ymm21
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm12[0],ymm6[0],ymm12[1],ymm6[1],ymm12[2],ymm6[2],ymm12[3],ymm6[3],ymm12[4],ymm6[4],ymm12[5],ymm6[5],ymm12[6],ymm6[6],ymm12[7],ymm6[7],ymm12[16],ymm6[16],ymm12[17],ymm6[17],ymm12[18],ymm6[18],ymm12[19],ymm6[19],ymm12[20],ymm6[20],ymm12[21],ymm6[21],ymm12[22],ymm6[22],ymm12[23],ymm6[23]
-; AVX512-NEXT:    vmovdqa64 %ymm5, %ymm20
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [u,10,u,13,u,12,u,11,u,14,u,13,u,14,u,15]
-; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm5
-; AVX512-NEXT:    vpshufb %xmm6, %xmm5, %xmm13
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX512-NEXT:    vmovdqa64 %ymm5, %ymm31
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
-; AVX512-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm8, %ymm3, %ymm9
-; AVX512-NEXT:    vpshufb %ymm8, %ymm4, %ymm3
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm9, %xmm3
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm20
+; AVX512-NEXT:    vmovdqa64 %ymm17, %ymm3
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm8[0],ymm3[0],ymm8[1],ymm3[1],ymm8[2],ymm3[2],ymm8[3],ymm3[3],ymm8[4],ymm3[4],ymm8[5],ymm3[5],ymm8[6],ymm3[6],ymm8[7],ymm3[7],ymm8[16],ymm3[16],ymm8[17],ymm3[17],ymm8[18],ymm3[18],ymm8[19],ymm3[19],ymm8[20],ymm3[20],ymm8[21],ymm3[21],ymm8[22],ymm3[22],ymm8[23],ymm3[23]
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm17
+; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm7
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm8 = [8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15]
+; AVX512-NEXT:    vpshufb %xmm8, %xmm5, %xmm9
+; AVX512-NEXT:    vpshufb %xmm8, %xmm3, %xmm3
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm31
+; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm3
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm11[0],ymm3[0],ymm11[1],ymm3[1],ymm11[2],ymm3[2],ymm11[3],ymm3[3],ymm11[4],ymm3[4],ymm11[5],ymm3[5],ymm11[6],ymm3[6],ymm11[7],ymm3[7],ymm11[16],ymm3[16],ymm11[17],ymm3[17],ymm11[18],ymm3[18],ymm11[19],ymm3[19],ymm11[20],ymm3[20],ymm11[21],ymm3[21],ymm11[22],ymm3[22],ymm11[23],ymm3[23]
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm19
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [10,u,13,u,12,u,11,u,14,u,13,u,14,u,15,u]
+; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm5
+; AVX512-NEXT:    vpshufb %xmm3, %xmm5, %xmm7
+; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm5
+; AVX512-NEXT:    vpshufb %xmm3, %xmm5, %xmm3
 ; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm22
-; AVX512-NEXT:    vpshufb %xmm14, %xmm1, %xmm3
-; AVX512-NEXT:    vpshufb %xmm14, %xmm0, %xmm4
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
+; AVX512-NEXT:    # ymm11 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm11, %ymm6, %ymm5
+; AVX512-NEXT:    vpshufb %ymm11, %ymm13, %ymm6
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm11 = [u,10,u,13,u,12,u,11,u,14,u,13,u,14,u,15]
+; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm3
+; AVX512-NEXT:    vpshufb %xmm11, %xmm3, %xmm13
+; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm3
+; AVX512-NEXT:    vpshufb %xmm11, %xmm3, %xmm11
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
+; AVX512-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm3, %ymm4, %ymm4
+; AVX512-NEXT:    vpshufb %ymm3, %ymm14, %ymm3
+; AVX512-NEXT:    vpshufb %xmm12, %xmm1, %xmm14
+; AVX512-NEXT:    vpshufb %xmm12, %xmm0, %xmm8
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm14[0],xmm8[1],xmm14[1],xmm8[2],xmm14[2],xmm8[3],xmm14[3],xmm8[4],xmm14[4],xmm8[5],xmm14[5],xmm8[6],xmm14[6],xmm8[7],xmm14[7]
 ; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX512-NEXT:    vprold $16, %xmm0, %xmm0
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm27
-; AVX512-NEXT:    vpshufb %xmm14, %xmm2, %xmm0
-; AVX512-NEXT:    vpshufb %xmm14, %xmm10, %xmm1
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512-NEXT:    vmovdqa (%r8), %xmm1
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3],xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7]
-; AVX512-NEXT:    vmovdqa 32(%r8), %xmm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
+; AVX512-NEXT:    vpshufb %xmm12, %xmm15, %xmm1
+; AVX512-NEXT:    vpshufb %xmm12, %xmm2, %xmm8
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3],xmm8[4],xmm1[4],xmm8[5],xmm1[5],xmm8[6],xmm1[6],xmm8[7],xmm1[7]
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3],xmm2[4],xmm15[4],xmm2[5],xmm15[5],xmm2[6],xmm15[6],xmm2[7],xmm15[7]
 ; AVX512-NEXT:    vprold $16, %xmm2, %xmm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [10,u,13,u,12,u,11,u,14,u,13,u,14,u,15,u]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm26
-; AVX512-NEXT:    vpshufb %xmm4, %xmm1, %xmm6
-; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm15
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [6,u,5,u,8,u,7,u,9,u,9,u,9,u,9,u]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm3, %xmm10
-; AVX512-NEXT:    vmovdqa64 %ymm10, %ymm17
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [2,u,1,u,0,u,3,u,4,u,4,u,4,u,4,u]
-; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm10
-; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm14
-; AVX512-NEXT:    vpshufb %xmm4, %xmm1, %xmm0
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm16
-; AVX512-NEXT:    vmovdqa (%r8), %ymm1
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
-; AVX512-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm2, %ymm1, %ymm5
-; AVX512-NEXT:    vmovdqa 32(%r8), %ymm3
-; AVX512-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0]
-; AVX512-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm7
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0]
-; AVX512-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm0, %ymm3, %ymm12
-; AVX512-NEXT:    vpshufb %ymm4, %ymm1, %ymm3
-; AVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm4
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm18[0,0,0,1]
-; AVX512-NEXT:    vprold $16, %ymm28, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,0,1]
-; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm8
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm8[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm30
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm10[0,0,0,1]
+; AVX512-NEXT:    vprold $16, %ymm16, %ymm8
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,0,1]
+; AVX512-NEXT:    vmovdqa64 %ymm18, %ymm1
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,0,1]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm28 = ymm6[0,0,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm13[0,0,0,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm20[0,0,0,1]
+; AVX512-NEXT:    vprold $16, %ymm17, %ymm14
+; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm31[0,0,0,1]
+; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm1
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm11, %zmm1
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogq $226, %zmm0, %zmm6, %zmm1
-; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
-; AVX512-NEXT:    vpternlogq $184, %ymm0, %ymm6, %ymm5
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512-NEXT:    vpternlogq $184, %ymm1, %ymm0, %ymm28
-; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm1
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm28[0,1,2,3],zmm1[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm13, %zmm1
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
-; AVX512-NEXT:    vpternlogd $184, %zmm5, %zmm8, %zmm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm29[0,0,0,1]
-; AVX512-NEXT:    vprold $16, %ymm30, %ymm9
-; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm21[0,0,0,1]
-; AVX512-NEXT:    vmovdqa64 %ymm20, %ymm13
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm13 = ymm13[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,0,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm28 = ymm31[0,0,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm29 = ymm22[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm30 = ymm17[0,0,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm31 = ymm16[0,0,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm31 = ymm22[0,0,0,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,0,1]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm5, %zmm5
-; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm9
-; AVX512-NEXT:    vpternlogq $226, %zmm5, %zmm6, %zmm9
-; AVX512-NEXT:    vextracti64x4 $1, %zmm9, %ymm5
-; AVX512-NEXT:    vpternlogq $184, %ymm5, %ymm6, %ymm2
-; AVX512-NEXT:    vpternlogq $184, %ymm9, %ymm0, %ymm15
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm15[0,1,2,3],zmm2[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm29, %zmm28, %zmm5
-; AVX512-NEXT:    vpternlogd $184, %zmm2, %zmm8, %zmm5
-; AVX512-NEXT:    vpermq {{.*#+}} zmm2 = zmm27[0,0,0,1,4,4,4,5]
-; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm8 = mem[0,0,0,1,4,4,4,5]
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
+; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm8
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm9, %zmm8
-; AVX512-NEXT:    vextracti64x4 $1, %zmm8, %ymm2
-; AVX512-NEXT:    vpternlogq $184, %ymm2, %ymm9, %ymm30
-; AVX512-NEXT:    vpternlogq $184, %ymm8, %ymm6, %ymm10
-; AVX512-NEXT:    vinserti64x4 $1, %ymm30, %zmm0, %zmm2
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm10[0,1,2,3],zmm2[4,5,6,7]
-; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm8 = mem[0,0,0,1,4,4,4,5]
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
-; AVX512-NEXT:    vpternlogd $184, %zmm2, %zmm10, %zmm8
-; AVX512-NEXT:    vpermq {{.*#+}} zmm2 = zmm26[0,0,0,1,4,4,4,5]
-; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm11 = mem[0,0,0,1,4,4,4,5]
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm9, %zmm11
-; AVX512-NEXT:    vextracti64x4 $1, %zmm11, %ymm2
-; AVX512-NEXT:    vpternlogq $184, %ymm2, %ymm9, %ymm14
-; AVX512-NEXT:    vpternlogq $184, %ymm11, %ymm6, %ymm31
-; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm2
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm31[0,1,2,3],zmm2[4,5,6,7]
-; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm11 = mem[0,0,0,1,4,4,4,5]
-; AVX512-NEXT:    vpternlogd $184, %zmm2, %zmm10, %zmm11
-; AVX512-NEXT:    vpermq $234, (%rsp), %zmm2 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm2 = mem[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm10 = mem[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm6, %zmm10
-; AVX512-NEXT:    vpermq {{.*#+}} zmm2 = zmm25[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} zmm13 = zmm24[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm6, %zmm13
-; AVX512-NEXT:    vextracti64x4 $1, %zmm10, %ymm2
-; AVX512-NEXT:    vpternlogq $184, %ymm2, %ymm0, %ymm7
-; AVX512-NEXT:    vextracti64x4 $1, %zmm13, %ymm2
-; AVX512-NEXT:    vpternlogq $184, %ymm2, %ymm0, %ymm3
-; AVX512-NEXT:    vpternlogq $184, %ymm10, %ymm9, %ymm12
-; AVX512-NEXT:    vpternlogq $184, %ymm13, %ymm9, %ymm4
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm0
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm12[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm2 = mem[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
-; AVX512-NEXT:    vpternlogd $184, %zmm0, %zmm6, %zmm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} zmm3 = zmm23[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpternlogd $184, %zmm0, %zmm6, %zmm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm7, %zmm2
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512-NEXT:    vpternlogd $184, %zmm8, %zmm5, %zmm2
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm14[2,2,2,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm15, %zmm1
+; AVX512-NEXT:    vpternlogq $226, %zmm7, %zmm9, %zmm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm31, %zmm6
+; AVX512-NEXT:    vpternlogd $184, %zmm1, %zmm5, %zmm6
+; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm12, %zmm1
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
+; AVX512-NEXT:    vpternlogd $184, %zmm2, %zmm4, %zmm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm11, %zmm2
+; AVX512-NEXT:    vpternlogd $184, %zmm6, %zmm4, %zmm2
+; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,0,0,1,4,4,4,5]
+; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm3 = mem[0,0,0,1,4,4,4,5]
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
+; AVX512-NEXT:    vpternlogq $226, %zmm0, %zmm4, %zmm3
+; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm0 = mem[0,0,0,1,4,4,4,5]
+; AVX512-NEXT:    vpternlogq $184, %zmm3, %zmm9, %zmm0
+; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm3 = mem[0,0,0,1,4,4,4,5]
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
+; AVX512-NEXT:    vpternlogd $184, %zmm0, %zmm5, %zmm3
+; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm30[0,0,0,1,4,4,4,5]
+; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm6 = mem[0,0,0,1,4,4,4,5]
+; AVX512-NEXT:    vpternlogq $226, %zmm0, %zmm4, %zmm6
+; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm0 = mem[0,0,0,1,4,4,4,5]
+; AVX512-NEXT:    vpternlogq $184, %zmm6, %zmm9, %zmm0
+; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm6 = mem[0,0,0,1,4,4,4,5]
+; AVX512-NEXT:    vpternlogd $184, %zmm0, %zmm5, %zmm6
+; AVX512-NEXT:    vpermq $234, (%rsp), %zmm0 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm5 = mem[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpternlogq $226, %zmm0, %zmm9, %zmm5
+; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm26[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm24[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpternlogq $226, %zmm0, %zmm9, %zmm7
+; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpternlogq $184, %zmm5, %zmm4, %zmm0
+; AVX512-NEXT:    vpermq {{.*#+}} zmm5 = zmm23[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpternlogq $184, %zmm7, %zmm4, %zmm5
+; AVX512-NEXT:    vpermq {{.*#+}} zmm4 = zmm28[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
+; AVX512-NEXT:    vpternlogd $184, %zmm0, %zmm7, %zmm4
+; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm25[2,2,2,3,6,6,6,7]
+; AVX512-NEXT:    vpternlogd $184, %zmm5, %zmm7, %zmm0
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vmovdqa64 %zmm3, 128(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm2, 320(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm8, 192(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm5, 256(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm4, 320(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm6, (%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm3, 192(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm2, 256(%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm1, 64(%rax)
-; AVX512-NEXT:    addq $328, %rsp # imm = 0x148
+; AVX512-NEXT:    addq $456, %rsp # imm = 0x1C8
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i8_stride6_vf64:
 ; AVX512-FCP:       # %bb.0:
-; AVX512-FCP-NEXT:    subq $392, %rsp # imm = 0x188
+; AVX512-FCP-NEXT:    subq $424, %rsp # imm = 0x1A8
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm4
 ; AVX512-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm4, %ymm1
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vmovdqu %ymm3, (%rsp) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm2
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
 ; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15],ymm3[24],ymm4[24],ymm3[25],ymm4[25],ymm3[26],ymm4[26],ymm3[27],ymm4[27],ymm3[28],ymm4[28],ymm3[29],ymm4[29],ymm3[30],ymm4[30],ymm3[31],ymm4[31]
@@ -5712,561 +5697,526 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm5
-; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm14 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm5, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm6
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm6, %ymm3
+; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm5, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm4
+; AVX512-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm4, %ymm3
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[16],ymm2[16],ymm3[17],ymm2[17],ymm3[18],ymm2[18],ymm3[19],ymm2[19],ymm3[20],ymm2[20],ymm3[21],ymm2[21],ymm3[22],ymm2[22],ymm3[23],ymm2[23]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15],ymm6[24],ymm5[24],ymm6[25],ymm5[25],ymm6[26],ymm5[26],ymm6[27],ymm5[27],ymm6[28],ymm5[28],ymm6[29],ymm5[29],ymm6[30],ymm5[30],ymm6[31],ymm5[31]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm6, %ymm25
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm5, %ymm30
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm5, %ymm22
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31]
 ; AVX512-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm4, %ymm4
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm7
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15]
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm6
+; AVX512-FCP-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0]
 ; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm7, %ymm5
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0]
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm6, %ymm5
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0]
 ; AVX512-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm7, %ymm6
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm7, %ymm31
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm6, %ymm6
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm8
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm8, %ymm5
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm7
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm7, %ymm0
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[4],ymm5[4],ymm0[5],ymm5[5],ymm0[6],ymm5[6],ymm0[7],ymm5[7],ymm0[16],ymm5[16],ymm0[17],ymm5[17],ymm0[18],ymm5[18],ymm0[19],ymm5[19],ymm0[20],ymm5[20],ymm0[21],ymm5[21],ymm0[22],ymm5[22],ymm0[23],ymm5[23]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15],ymm7[24],ymm8[24],ymm7[25],ymm8[25],ymm7[26],ymm8[26],ymm7[27],ymm8[27],ymm7[28],ymm8[28],ymm7[29],ymm8[29],ymm7[30],ymm8[30],ymm7[31],ymm8[31]
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm7
+; AVX512-FCP-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15]
+; AVX512-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm7, %ymm6
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0]
+; AVX512-FCP-NEXT:    # ymm9 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm7, %ymm8
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm8, %zmm6
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm7
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm7, %ymm6
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm8
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm8, %ymm0
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm6[0],ymm0[1],ymm6[1],ymm0[2],ymm6[2],ymm0[3],ymm6[3],ymm0[4],ymm6[4],ymm0[5],ymm6[5],ymm0[6],ymm6[6],ymm0[7],ymm6[7],ymm0[16],ymm6[16],ymm0[17],ymm6[17],ymm0[18],ymm6[18],ymm0[19],ymm6[19],ymm0[20],ymm6[20],ymm0[21],ymm6[21],ymm0[22],ymm6[22],ymm0[23],ymm6[23]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11],ymm8[12],ymm7[12],ymm8[13],ymm7[13],ymm8[14],ymm7[14],ymm8[15],ymm7[15],ymm8[24],ymm7[24],ymm8[25],ymm7[25],ymm8[26],ymm7[26],ymm8[27],ymm7[27],ymm8[28],ymm7[28],ymm8[29],ymm7[29],ymm8[30],ymm7[30],ymm8[31],ymm7[31]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm8, %ymm18
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm7, %ymm17
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm11
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm11, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm12
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm12, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm15
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm15, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm11
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm11, %ymm1
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm12[8],ymm11[8],ymm12[9],ymm11[9],ymm12[10],ymm11[10],ymm12[11],ymm11[11],ymm12[12],ymm11[12],ymm12[13],ymm11[13],ymm12[14],ymm11[14],ymm12[15],ymm11[15],ymm12[24],ymm11[24],ymm12[25],ymm11[25],ymm12[26],ymm11[26],ymm12[27],ymm11[27],ymm12[28],ymm11[28],ymm12[29],ymm11[29],ymm12[30],ymm11[30],ymm12[31],ymm11[31]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm11[8],ymm15[8],ymm11[9],ymm15[9],ymm11[10],ymm15[10],ymm11[11],ymm15[11],ymm11[12],ymm15[12],ymm11[13],ymm15[13],ymm11[14],ymm15[14],ymm11[15],ymm15[15],ymm11[24],ymm15[24],ymm11[25],ymm15[25],ymm11[26],ymm15[26],ymm11[27],ymm15[27],ymm11[28],ymm15[28],ymm11[29],ymm15[29],ymm11[30],ymm15[30],ymm11[31],ymm15[31]
 ; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm9
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm9, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm9, %ymm1
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm3
-; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm1
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm6
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm6, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm2
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm19
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm23
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm2
+; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm2, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm20
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm24
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm0
+; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm5 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm1
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm12
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm12, %xmm2
 ; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm18
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,4,5,2,3,8,9,10,11,12,13,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm2, %zmm1
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [u,6,u,5,u,8,u,7,u,9,u,9,u,9,u,9]
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm4, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,2,u,1,u,0,u,3,u,4,u,4,u,4,u,4]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm4, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm4, %xmm19
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm2, %zmm3, %zmm26
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm5
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm3
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm5, %xmm4
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm4, %xmm4
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm4, %zmm27
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm15
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm15, %xmm0
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm15, %xmm1
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm28
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm13
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0]
-; AVX512-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm13, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm21
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm22 = ymm0[2,2,2,3]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0]
-; AVX512-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm13, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm20
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm24 = ymm0[2,2,2,3]
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm2
-; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm2, %xmm4
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm1
-; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm1, %xmm10
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm12[0],xmm0[0],xmm12[1],xmm0[1],xmm12[2],xmm0[2],xmm12[3],xmm0[3],xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm16
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [0,1,6,7,4,5,2,3,8,9,10,11,12,13,10,11]
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm2, %zmm26
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm13
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [6,u,5,u,8,u,7,u,9,u,9,u,9,u,9,u]
+; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm13, %xmm1
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,u,1,u,0,u,3,u,4,u,4,u,4,u,4,u]
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm13, %xmm3
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm3, %zmm29
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm14
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [u,6,u,5,u,8,u,7,u,9,u,9,u,9,u,9]
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm14, %xmm1
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,2,u,1,u,0,u,3,u,4,u,4,u,4,u,4]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm14, %xmm4
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm4, %zmm21
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm6, %xmm0
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm5
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm5, %zmm25
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm1
+; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm5
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm2
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm5, %zmm2, %zmm27
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm8
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm8, %xmm5
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm8, %xmm3
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm5, %zmm3, %zmm28
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm5
+; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm5, %xmm7
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm3
+; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm3, %xmm9
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3],xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; AVX512-FCP-NEXT:    vprold $16, %xmm9, %xmm9
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm9, %zmm30
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm7
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm9
+; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm7, %xmm0
+; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm9, %xmm10
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1],xmm10[2],xmm0[2],xmm10[3],xmm0[3],xmm10[4],xmm0[4],xmm10[5],xmm0[5],xmm10[6],xmm0[6],xmm10[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3],xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]
 ; AVX512-FCP-NEXT:    vprold $16, %xmm10, %xmm10
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm10, %zmm23
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm4, %xmm0
-; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm10, %xmm14
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3],xmm14[4],xmm0[4],xmm14[5],xmm0[5],xmm14[6],xmm0[6],xmm14[7],xmm0[7]
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; AVX512-FCP-NEXT:    vprold $16, %xmm14, %xmm14
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm14, %zmm29
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm12[0],ymm11[0],ymm12[1],ymm11[1],ymm12[2],ymm11[2],ymm12[3],ymm11[3],ymm12[4],ymm11[4],ymm12[5],ymm11[5],ymm12[6],ymm11[6],ymm12[7],ymm11[7],ymm12[16],ymm11[16],ymm12[17],ymm11[17],ymm12[18],ymm11[18],ymm12[19],ymm11[19],ymm12[20],ymm11[20],ymm12[21],ymm11[21],ymm12[22],ymm11[22],ymm12[23],ymm11[23]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm10, %zmm31
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm11[0],ymm15[0],ymm11[1],ymm15[1],ymm11[2],ymm15[2],ymm11[3],ymm15[3],ymm11[4],ymm15[4],ymm11[5],ymm15[5],ymm11[6],ymm15[6],ymm11[7],ymm15[7],ymm11[16],ymm15[16],ymm11[17],ymm15[17],ymm11[18],ymm15[18],ymm11[19],ymm15[19],ymm11[20],ymm15[20],ymm11[21],ymm15[21],ymm11[22],ymm15[22],ymm11[23],ymm15[23]
 ; AVX512-FCP-NEXT:    vprold $16, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15]
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[16],ymm8[16],ymm7[17],ymm8[17],ymm7[18],ymm8[18],ymm7[19],ymm8[19],ymm7[20],ymm8[20],ymm7[21],ymm8[21],ymm7[22],ymm8[22],ymm7[23],ymm8[23]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [16,17,22,23,20,21,18,19,24,25,26,27,28,29,26,27,16,17,22,23,20,21,18,19,24,25,26,27,28,29,26,27]
-; AVX512-FCP-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm4, %ymm4
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm3, %xmm11
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [0,0,0,1,10,10,10,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm7, %zmm11
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [10,11,8,9,6,7,12,13,14,15,14,15,14,15,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm10, %xmm3
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm7, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm16, %zmm11
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
-; AVX512-FCP-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm9, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [u,10,u,13,u,12,u,11,u,14,u,13,u,14,u,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm15, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm5, %xmm17
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm7, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm25, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm30, %ymm5
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[4],ymm5[4],ymm0[5],ymm5[5],ymm0[6],ymm5[6],ymm0[7],ymm5[7],ymm0[16],ymm5[16],ymm0[17],ymm5[17],ymm0[18],ymm5[18],ymm0[19],ymm5[19],ymm0[20],ymm5[20],ymm0[21],ymm5[21],ymm0[22],ymm5[22],ymm0[23],ymm5[23]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; AVX512-FCP-NEXT:    vprold $16, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm1, %xmm2
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    # ymm4 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[16],mem[16],ymm0[17],mem[17],ymm0[18],mem[18],ymm0[19],mem[19],ymm0[20],mem[20],ymm0[21],mem[21],ymm0[22],mem[22],ymm0[23],mem[23]
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm5
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm21, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm1
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm4, %ymm4
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm14
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm18, %xmm8
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm8[8],xmm6[9],xmm8[9],xmm6[10],xmm8[10],xmm6[11],xmm8[11],xmm6[12],xmm8[12],xmm6[13],xmm8[13],xmm6[14],xmm8[14],xmm6[15],xmm8[15]
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm15
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm6, %xmm8
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [6,u,5,u,8,u,7,u,9,u,9,u,9,u,9,u]
-; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm7, %zmm8
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm15, %xmm4
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm16, %zmm8
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm9[8],xmm7[8],xmm9[9],xmm7[9],xmm9[10],xmm7[10],xmm9[11],xmm7[11],xmm9[12],xmm7[12],xmm9[13],xmm7[13],xmm9[14],xmm7[14],xmm9[15],xmm7[15]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm18, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm17, %ymm9
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm2[0],ymm9[0],ymm2[1],ymm9[1],ymm2[2],ymm9[2],ymm2[3],ymm9[3],ymm2[4],ymm9[4],ymm2[5],ymm9[5],ymm2[6],ymm9[6],ymm2[7],ymm9[7],ymm2[16],ymm9[16],ymm2[17],ymm9[17],ymm2[18],ymm9[18],ymm2[19],ymm9[19],ymm2[20],ymm9[20],ymm2[21],ymm9[21],ymm2[22],ymm9[22],ymm2[23],ymm9[23]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [16,17,22,23,20,21,18,19,24,25,26,27,28,29,26,27,16,17,22,23,20,21,18,19,24,25,26,27,28,29,26,27]
+; AVX512-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm9, %ymm9
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm18
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm4, %xmm15
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm17
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,0,0,1,10,10,10,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm9, %zmm4, %zmm15
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [10,11,8,9,6,7,12,13,14,15,14,15,14,15,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm4, %zmm7
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm7, %zmm9, %zmm15
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
 ; AVX512-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm5, %ymm12
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm13, %ymm9
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,u,1,u,0,u,3,u,4,u,4,u,4,u,4,u]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm14, %xmm6
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm15, %xmm5
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm14, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [10,u,13,u,12,u,11,u,14,u,13,u,14,u,15,u]
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm14, %xmm14
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm15, %xmm13
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm31, %ymm15
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm15, %ymm15
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm30 = ymm0[2,2,2,3]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm31 = ymm1[2,2,2,3]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,0,1]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,0,1]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm6[0,0,0,1]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm2[0,0,0,1]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm14[0,0,0,1]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm6
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm17, %xmm14
-; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm6, %xmm6
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm11, %ymm14, %ymm2
-; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm11, %ymm11
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,0,1]
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm8, %ymm14, %ymm13
-; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm8, %ymm8
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
-; AVX512-FCP-NEXT:    vpermt2q %zmm15, %zmm7, %zmm6
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm8, %ymm16, %ymm9
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm7
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm13[0,1,2,3],zmm7[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm7, %zmm8, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm19
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [10,u,13,u,12,u,11,u,14,u,13,u,14,u,15,u]
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm1, %xmm1
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm4, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm15, %zmm10, %zmm1
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
+; AVX512-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [u,10,u,13,u,12,u,11,u,14,u,13,u,14,u,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm8, %xmm0
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm1, %zmm2, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm22, %ymm8
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm8[0],ymm1[1],ymm8[1],ymm1[2],ymm8[2],ymm1[3],ymm8[3],ymm1[4],ymm8[4],ymm1[5],ymm8[5],ymm1[6],ymm8[6],ymm1[7],ymm8[7],ymm1[16],ymm8[16],ymm1[17],ymm8[17],ymm1[18],ymm8[18],ymm1[19],ymm8[19],ymm1[20],ymm8[20],ymm1[21],ymm8[21],ymm1[22],ymm8[22],ymm1[23],ymm8[23]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vmovdqu (%rsp), %ymm5 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    # ymm5 = ymm5[0],mem[0],ymm5[1],mem[1],ymm5[2],mem[2],ymm5[3],mem[3],ymm5[4],mem[4],ymm5[5],mem[5],ymm5[6],mem[6],ymm5[7],mem[7],ymm5[16],mem[16],ymm5[17],mem[17],ymm5[18],mem[18],ymm5[19],mem[19],ymm5[20],mem[20],ymm5[21],mem[21],ymm5[22],mem[22],ymm5[23],mem[23]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm18, %ymm6
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm16, %xmm6
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm12[8],xmm6[8],xmm12[9],xmm6[9],xmm12[10],xmm6[10],xmm12[11],xmm6[11],xmm12[12],xmm6[12],xmm12[13],xmm6[13],xmm12[14],xmm6[14],xmm12[15],xmm6[15]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm17, %xmm8
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm6, %xmm6
+; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm4, %zmm6
+; AVX512-FCP-NEXT:    vprold $16, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm3
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm9, %zmm6
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm3
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm13, %xmm3
+; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm3
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm6, %zmm10, %zmm3
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm14, %xmm5
+; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm5
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm2, %zmm5
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 256(%rax)
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm11, %ymm16, %ymm12
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm0, %zmm6
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm6[4,5,6,7]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm8, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 64(%rax)
-; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm2 = mem[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm3 = mem[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm16, %zmm3
-; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm3, %ymm2
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm2, %ymm14, %ymm22
-; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm2 = mem[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm6 = mem[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm16, %zmm6
-; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm6, %ymm2
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm2, %ymm14, %ymm30
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 256(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
+; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    # zmm1 = mem[2,2,2,3,6,6,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm9, %zmm1
+; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm3, %ymm2, %ymm24
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm22, %zmm0, %zmm3
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm24[0,1,2,3],zmm3[4,5,6,7]
-; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm7 = mem[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm8, %zmm7
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm6, %ymm2, %ymm31
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm30, %zmm0, %zmm3
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm31[0,1,2,3],zmm3[4,5,6,7]
-; AVX512-FCP-NEXT:    vpermq $234, (%rsp), %zmm6 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm6 = mem[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm8, %zmm6
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm23[0,0,0,1,4,4,4,5]
-; AVX512-FCP-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm8 = mem[0,0,0,1,4,4,4,5]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm2, %zmm8
-; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm8, %ymm3
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm3, %ymm2, %ymm4
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm29[0,0,0,1,4,4,4,5]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm27[0,0,0,1,4,4,4,5]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm2, %zmm9
-; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm9, %ymm3
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm3, %ymm2, %ymm0
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm8, %ymm16, %ymm5
-; AVX512-FCP-NEXT:    vpternlogq $184, %ymm9, %ymm16, %ymm1
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm2
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm5[0,1,2,3],zmm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm1, %zmm2, %zmm0
+; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    # zmm1 = mem[2,2,2,3,6,6,6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm1
+; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
+; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    # zmm4 = mem[2,2,2,3,6,6,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm9, %zmm4
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm23[2,2,2,3,6,6,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm4, %zmm2, %zmm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm4 = zmm24[2,2,2,3,6,6,6,7]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm4
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm30[0,0,0,1,4,4,4,5]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm26[0,0,0,1,4,4,4,5]
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm4, %zmm3
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm28[0,0,0,1,4,4,4,5]
-; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm4, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm2, %zmm3
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm31[0,0,0,1,4,4,4,5]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm25[0,0,0,1,4,4,4,5]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm2, %zmm5
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm29[0,0,0,1,4,4,4,5]
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm3, %zmm9, %zmm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm2 = zmm27[0,0,0,1,4,4,4,5]
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm5, %zmm9, %zmm2
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm21[0,0,0,1,4,4,4,5]
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm5, %zmm3
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm28[0,0,0,1,4,4,4,5]
+; AVX512-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm5, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 192(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 128(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, 320(%rax)
-; AVX512-FCP-NEXT:    addq $392, %rsp # imm = 0x188
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 128(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 320(%rax)
+; AVX512-FCP-NEXT:    addq $424, %rsp # imm = 0x1A8
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i8_stride6_vf64:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    subq $328, %rsp # imm = 0x148
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm5
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm2
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm10
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm3
-; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm11
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm7
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm8
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm9
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm6
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm12
-; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} ymm14 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} xmm4 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm2, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm3, %xmm1
+; AVX512DQ-NEXT:    subq $456, %rsp # imm = 0x1C8
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm6
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm3
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm7
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm10
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm4
+; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm12
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm8
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm13
+; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} xmm9 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm3, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm2, %xmm1
 ; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm16
-; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm27
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm30
+; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm31
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,5]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [u,6,u,5,u,8,u,7,u,9,u,9,u,9,u,9]
-; AVX512DQ-NEXT:    vmovdqa %xmm7, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm2
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,2,u,1,u,0,u,3,u,4,u,4,u,4,u,4]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm7
-; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm26
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm2, %zmm7, %zmm2
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm5, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3],xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm10, %xmm29
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm30
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,3,2,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm4, %zmm2
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [6,u,5,u,8,u,7,u,9,u,9,u,9,u,9,u]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm2
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,u,1,u,0,u,3,u,4,u,4,u,4,u,4,u]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm29
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm2, %zmm3, %zmm2
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm11, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm11, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm11, %xmm31
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [u,6,u,5,u,8,u,7,u,9,u,9,u,9,u,9]
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm4
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,2,u,1,u,0,u,3,u,4,u,4,u,4,u,4]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm8, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm27
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm4, %zmm5, %zmm4
+; AVX512DQ-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm8
+; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm6, %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm7, %xmm5
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm18
+; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm20
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,3,2,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,6,5]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm4
+; AVX512DQ-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm4
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm11
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm10, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 %xmm10, %xmm22
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm12, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm12, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 %xmm12, %xmm21
 ; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm6, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm12, %ymm1
+; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} ymm15 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm4, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm11, %ymm1
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm12[8],ymm6[8],ymm12[9],ymm6[9],ymm12[10],ymm6[10],ymm12[11],ymm6[11],ymm12[12],ymm6[12],ymm12[13],ymm6[13],ymm12[14],ymm6[14],ymm12[15],ymm6[15],ymm12[24],ymm6[24],ymm12[25],ymm6[25],ymm12[26],ymm6[26],ymm12[27],ymm6[27],ymm12[28],ymm6[28],ymm12[29],ymm6[29],ymm12[30],ymm6[30],ymm12[31],ymm6[31]
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31]
-; AVX512DQ-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm11[8],ymm4[8],ymm11[9],ymm4[9],ymm11[10],ymm4[10],ymm11[11],ymm4[11],ymm11[12],ymm4[12],ymm11[13],ymm4[13],ymm11[14],ymm4[14],ymm11[15],ymm4[15],ymm11[24],ymm4[24],ymm11[25],ymm4[25],ymm11[26],ymm4[26],ymm11[27],ymm4[27],ymm11[28],ymm4[28],ymm11[29],ymm4[29],ymm11[30],ymm4[30],ymm11[31],ymm4[31]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm4, %ymm19
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31]
+; AVX512DQ-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm8, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm9, %ymm1
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15],ymm9[24],ymm8[24],ymm9[25],ymm8[25],ymm9[26],ymm8[26],ymm9[27],ymm8[27],ymm9[28],ymm8[28],ymm9[29],ymm8[29],ymm9[30],ymm8[30],ymm9[31],ymm8[31]
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31]
-; AVX512DQ-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm4
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15]
+; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} ymm12 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm13, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm8, %ymm1
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm8[8],ymm13[8],ymm8[9],ymm13[9],ymm8[10],ymm13[10],ymm8[11],ymm13[11],ymm8[12],ymm13[12],ymm8[13],ymm13[13],ymm8[14],ymm13[14],ymm8[15],ymm13[15],ymm8[24],ymm13[24],ymm8[25],ymm13[25],ymm8[26],ymm13[26],ymm8[27],ymm13[27],ymm8[28],ymm13[28],ymm8[29],ymm13[29],ymm8[30],ymm13[30],ymm8[31],ymm13[31]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm13, %ymm17
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31]
+; AVX512DQ-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm13
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0]
+; AVX512DQ-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm13, %ymm0
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0]
+; AVX512DQ-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm13, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm14
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15]
+; AVX512DQ-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm14, %ymm0
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0]
 ; AVX512DQ-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm4, %ymm10
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0]
-; AVX512DQ-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm11
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm11, %zmm3
-; AVX512DQ-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm15
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm10
-; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm15, %ymm11
-; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm10, %ymm7
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm7[0],ymm11[0],ymm7[1],ymm11[1],ymm7[2],ymm11[2],ymm7[3],ymm11[3],ymm7[4],ymm11[4],ymm7[5],ymm11[5],ymm7[6],ymm11[6],ymm7[7],ymm11[7],ymm7[16],ymm11[16],ymm7[17],ymm11[17],ymm7[18],ymm11[18],ymm7[19],ymm11[19],ymm7[20],ymm11[20],ymm7[21],ymm11[21],ymm7[22],ymm11[22],ymm7[23],ymm11[23]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm10[8],ymm15[8],ymm10[9],ymm15[9],ymm10[10],ymm15[10],ymm10[11],ymm15[11],ymm10[12],ymm15[12],ymm10[13],ymm15[13],ymm10[14],ymm15[14],ymm10[15],ymm15[15],ymm10[24],ymm15[24],ymm10[25],ymm15[25],ymm10[26],ymm15[26],ymm10[27],ymm15[27],ymm10[28],ymm15[28],ymm10[29],ymm15[29],ymm10[30],ymm15[30],ymm10[31],ymm15[31]
-; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm11, %ymm11
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm7, %zmm25
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm7
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm11
-; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm7, %ymm13
-; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm11, %ymm5
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm13[0],ymm5[1],ymm13[1],ymm5[2],ymm13[2],ymm5[3],ymm13[3],ymm5[4],ymm13[4],ymm5[5],ymm13[5],ymm5[6],ymm13[6],ymm5[7],ymm13[7],ymm5[16],ymm13[16],ymm5[17],ymm13[17],ymm5[18],ymm13[18],ymm5[19],ymm13[19],ymm5[20],ymm13[20],ymm5[21],ymm13[21],ymm5[22],ymm13[22],ymm5[23],ymm13[23]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm13 = ymm11[8],ymm7[8],ymm11[9],ymm7[9],ymm11[10],ymm7[10],ymm11[11],ymm7[11],ymm11[12],ymm7[12],ymm11[13],ymm7[13],ymm11[14],ymm7[14],ymm11[15],ymm7[15],ymm11[24],ymm7[24],ymm11[25],ymm7[25],ymm11[26],ymm7[26],ymm11[27],ymm7[27],ymm11[28],ymm7[28],ymm11[29],ymm7[29],ymm11[30],ymm7[30],ymm11[31],ymm7[31]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm13, %ymm0
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm5, %zmm24
-; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm3, %ymm1
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm23
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm11[0],ymm7[0],ymm11[1],ymm7[1],ymm11[2],ymm7[2],ymm11[3],ymm7[3],ymm11[4],ymm7[4],ymm11[5],ymm7[5],ymm11[6],ymm7[6],ymm11[7],ymm7[7],ymm11[16],ymm7[16],ymm11[17],ymm7[17],ymm11[18],ymm7[18],ymm11[19],ymm7[19],ymm11[20],ymm7[20],ymm11[21],ymm7[21],ymm11[22],ymm7[22],ymm11[23],ymm7[23]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm28
-; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm0
-; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm1
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm2
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm10[0],ymm15[0],ymm10[1],ymm15[1],ymm10[2],ymm15[2],ymm10[3],ymm15[3],ymm10[4],ymm15[4],ymm10[5],ymm15[5],ymm10[6],ymm15[6],ymm10[7],ymm15[7],ymm10[16],ymm15[16],ymm10[17],ymm15[17],ymm10[18],ymm15[18],ymm10[19],ymm15[19],ymm10[20],ymm15[20],ymm10[21],ymm15[21],ymm10[22],ymm15[22],ymm10[23],ymm15[23]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm19
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm10[8],xmm2[8],xmm10[9],xmm2[9],xmm10[10],xmm2[10],xmm10[11],xmm2[11],xmm10[12],xmm2[12],xmm10[13],xmm2[13],xmm10[14],xmm2[14],xmm10[15],xmm2[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm15 = [10,11,8,9,6,7,12,13,14,15,14,15,14,15,14,15]
-; AVX512DQ-NEXT:    vpshufb %xmm15, %xmm5, %xmm0
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm14, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm28
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm1
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm1, %ymm10
+; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm0, %ymm15
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm15[0],ymm10[0],ymm15[1],ymm10[1],ymm15[2],ymm10[2],ymm15[3],ymm10[3],ymm15[4],ymm10[4],ymm15[5],ymm10[5],ymm15[6],ymm10[6],ymm15[7],ymm10[7],ymm15[16],ymm10[16],ymm15[17],ymm10[17],ymm15[18],ymm10[18],ymm15[19],ymm10[19],ymm15[20],ymm10[20],ymm15[21],ymm10[21],ymm15[22],ymm10[22],ymm15[23],ymm10[23]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm15 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm15, %ymm6
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm10, %zmm26
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm10
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm15
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm10, %ymm6
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm15, %ymm9
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm9[0],ymm6[0],ymm9[1],ymm6[1],ymm9[2],ymm6[2],ymm9[3],ymm6[3],ymm9[4],ymm6[4],ymm9[5],ymm6[5],ymm9[6],ymm6[6],ymm9[7],ymm6[7],ymm9[16],ymm6[16],ymm9[17],ymm6[17],ymm9[18],ymm6[18],ymm9[19],ymm6[19],ymm9[20],ymm6[20],ymm9[21],ymm6[21],ymm9[22],ymm6[22],ymm9[23],ymm6[23]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm15[8],ymm10[8],ymm15[9],ymm10[9],ymm15[10],ymm10[10],ymm15[11],ymm10[11],ymm15[12],ymm10[12],ymm15[13],ymm10[13],ymm15[14],ymm10[14],ymm15[15],ymm10[15],ymm15[24],ymm10[24],ymm15[25],ymm10[25],ymm15[26],ymm10[26],ymm15[27],ymm10[27],ymm15[28],ymm10[28],ymm15[29],ymm10[29],ymm15[30],ymm10[30],ymm15[31],ymm10[31]
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm9, %ymm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm6, %zmm24
+; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm6
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm6, %ymm5
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm23
+; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm4
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm4, %ymm2
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm25
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm15[0],ymm10[0],ymm15[1],ymm10[1],ymm15[2],ymm10[2],ymm15[3],ymm10[3],ymm15[4],ymm10[4],ymm15[5],ymm10[5],ymm15[6],ymm10[6],ymm15[7],ymm10[7],ymm15[16],ymm10[16],ymm15[17],ymm10[17],ymm15[18],ymm10[18],ymm15[19],ymm10[19],ymm15[20],ymm10[20],ymm15[21],ymm10[21],ymm15[22],ymm10[22],ymm15[23],ymm10[23]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm16
+; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm2
+; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm3
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm18
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm15
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm2
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm15[8],xmm2[9],xmm15[9],xmm2[10],xmm15[10],xmm2[11],xmm15[11],xmm2[12],xmm15[12],xmm2[13],xmm15[13],xmm2[14],xmm15[14],xmm2[15],xmm15[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [10,11,8,9,6,7,12,13,14,15,14,15,14,15,14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm1, %xmm10
 ; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512DQ-NEXT:    vpshufb %xmm15, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vmovdqa64 %ymm5, %ymm29
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[16],ymm8[16],ymm9[17],ymm8[17],ymm9[18],ymm8[18],ymm9[19],ymm8[19],ymm9[20],ymm8[20],ymm9[21],ymm8[21],ymm9[22],ymm8[22],ymm9[23],ymm8[23]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm5, %ymm30
-; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm5
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm8
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm5[8],xmm8[8],xmm5[9],xmm8[9],xmm5[10],xmm8[10],xmm5[11],xmm8[11],xmm5[12],xmm8[12],xmm5[13],xmm8[13],xmm5[14],xmm8[14],xmm5[15],xmm8[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm9 = [8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15]
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm7, %xmm11
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm8, %xmm5
-; AVX512DQ-NEXT:    vmovdqa64 %ymm5, %ymm21
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm12[0],ymm6[0],ymm12[1],ymm6[1],ymm12[2],ymm6[2],ymm12[3],ymm6[3],ymm12[4],ymm6[4],ymm12[5],ymm6[5],ymm12[6],ymm6[6],ymm12[7],ymm6[7],ymm12[16],ymm6[16],ymm12[17],ymm6[17],ymm12[18],ymm6[18],ymm12[19],ymm6[19],ymm12[20],ymm6[20],ymm12[21],ymm6[21],ymm12[22],ymm6[22],ymm12[23],ymm6[23]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm5, %ymm20
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [u,10,u,13,u,12,u,11,u,14,u,13,u,14,u,15]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm5, %xmm13
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vmovdqa64 %ymm5, %ymm31
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
-; AVX512DQ-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm8, %ymm3, %ymm9
-; AVX512DQ-NEXT:    vpshufb %ymm8, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm9, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm20
+; AVX512DQ-NEXT:    vmovdqa64 %ymm17, %ymm3
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm8[0],ymm3[0],ymm8[1],ymm3[1],ymm8[2],ymm3[2],ymm8[3],ymm3[3],ymm8[4],ymm3[4],ymm8[5],ymm3[5],ymm8[6],ymm3[6],ymm8[7],ymm3[7],ymm8[16],ymm3[16],ymm8[17],ymm3[17],ymm8[18],ymm3[18],ymm8[19],ymm3[19],ymm8[20],ymm3[20],ymm8[21],ymm3[21],ymm8[22],ymm3[22],ymm8[23],ymm3[23]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm17
+; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm7
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm8 = [8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm5, %xmm9
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm31
+; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm3
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm11[0],ymm3[0],ymm11[1],ymm3[1],ymm11[2],ymm3[2],ymm11[3],ymm3[3],ymm11[4],ymm3[4],ymm11[5],ymm3[5],ymm11[6],ymm3[6],ymm11[7],ymm3[7],ymm11[16],ymm3[16],ymm11[17],ymm3[17],ymm11[18],ymm3[18],ymm11[19],ymm3[19],ymm11[20],ymm3[20],ymm11[21],ymm3[21],ymm11[22],ymm3[22],ymm11[23],ymm3[23]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm19
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [10,u,13,u,12,u,11,u,14,u,13,u,14,u,15,u]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm5, %xmm7
+; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm5, %xmm3
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm22
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm1, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm0, %xmm4
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
+; AVX512DQ-NEXT:    # ymm11 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm6, %ymm5
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm13, %ymm6
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm11 = [u,10,u,13,u,12,u,11,u,14,u,13,u,14,u,15]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm11, %xmm3, %xmm13
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm11, %xmm3, %xmm11
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
+; AVX512DQ-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm4
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm14, %ymm3
+; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm1, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm0, %xmm8
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm14[0],xmm8[1],xmm14[1],xmm8[2],xmm14[2],xmm8[3],xmm14[3],xmm8[4],xmm14[4],xmm8[5],xmm14[5],xmm8[6],xmm14[6],xmm8[7],xmm14[7]
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX512DQ-NEXT:    vprold $16, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm27
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm2, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm10, %xmm1
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm1
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3],xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7]
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm3
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm15, %xmm1
+; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm2, %xmm8
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3],xmm8[4],xmm1[4],xmm8[5],xmm1[5],xmm8[6],xmm1[6],xmm8[7],xmm1[7]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3],xmm2[4],xmm15[4],xmm2[5],xmm15[5],xmm2[6],xmm15[6],xmm2[7],xmm15[7]
 ; AVX512DQ-NEXT:    vprold $16, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm4 = [10,u,13,u,12,u,11,u,14,u,13,u,14,u,15,u]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm26
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm1, %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm3, %xmm15
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [6,u,5,u,8,u,7,u,9,u,9,u,9,u,9,u]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm3, %xmm10
-; AVX512DQ-NEXT:    vmovdqa64 %ymm10, %ymm17
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm4 = [2,u,1,u,0,u,3,u,4,u,4,u,4,u,4,u]
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm3, %xmm10
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm1, %xmm14
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm1, %xmm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm16
-; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm1
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
-; AVX512DQ-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm1, %ymm5
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0]
-; AVX512DQ-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm7
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0]
-; AVX512DQ-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm3, %ymm12
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm1, %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm1, %ymm4
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm18[0,0,0,1]
-; AVX512DQ-NEXT:    vprold $16, %ymm28, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,0,1]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm8
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm8 = ymm8[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm30
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm10[0,0,0,1]
+; AVX512DQ-NEXT:    vprold $16, %ymm16, %ymm8
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,0,1]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm18, %ymm1
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,0,1]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm28 = ymm6[0,0,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm13[0,0,0,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm20[0,0,0,1]
+; AVX512DQ-NEXT:    vprold $16, %ymm17, %ymm14
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm31[0,0,0,1]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm1
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm11, %zmm1
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm0, %zmm6, %zmm1
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm0, %ymm6, %ymm5
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm1, %ymm0, %ymm28
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm1
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm28[0,1,2,3],zmm1[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm13, %zmm1
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm5, %zmm8, %zmm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm29[0,0,0,1]
-; AVX512DQ-NEXT:    vprold $16, %ymm30, %ymm9
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm21[0,0,0,1]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm20, %ymm13
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm13 = ymm13[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,0,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm28 = ymm31[0,0,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm29 = ymm22[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm30 = ymm17[0,0,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm31 = ymm16[0,0,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm31 = ymm22[0,0,0,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,0,1]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm5, %zmm5
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm9
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm5, %zmm6, %zmm9
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm9, %ymm5
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm5, %ymm6, %ymm2
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm9, %ymm0, %ymm15
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm15[0,1,2,3],zmm2[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm29, %zmm28, %zmm5
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm2, %zmm8, %zmm5
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm2 = zmm27[0,0,0,1,4,4,4,5]
-; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm8 = mem[0,0,0,1,4,4,4,5]
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm8
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm9, %zmm8
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm8, %ymm2
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm2, %ymm9, %ymm30
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm8, %ymm6, %ymm10
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm30, %zmm0, %zmm2
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm10[0,1,2,3],zmm2[4,5,6,7]
-; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm8 = mem[0,0,0,1,4,4,4,5]
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm2, %zmm10, %zmm8
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm2 = zmm26[0,0,0,1,4,4,4,5]
-; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm11 = mem[0,0,0,1,4,4,4,5]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm9, %zmm11
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm11, %ymm2
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm2, %ymm9, %ymm14
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm11, %ymm6, %ymm31
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm2
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm31[0,1,2,3],zmm2[4,5,6,7]
-; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm11 = mem[0,0,0,1,4,4,4,5]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm2, %zmm10, %zmm11
-; AVX512DQ-NEXT:    vpermq $234, (%rsp), %zmm2 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm2 = mem[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm10 = mem[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm6, %zmm10
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm2 = zmm25[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm13 = zmm24[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm6, %zmm13
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm10, %ymm2
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm2, %ymm0, %ymm7
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm13, %ymm2
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm2, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm10, %ymm9, %ymm12
-; AVX512DQ-NEXT:    vpternlogq $184, %ymm13, %ymm9, %ymm4
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm12[0,1,2,3],zmm0[4,5,6,7]
-; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm2 = mem[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm0, %zmm6, %zmm2
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[4,5,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm3 = zmm23[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpternlogd $184, %zmm0, %zmm6, %zmm3
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm7, %zmm2
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm8, %zmm5, %zmm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm14[2,2,2,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm15, %zmm1
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm7, %zmm9, %zmm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm31, %zmm6
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm1, %zmm5, %zmm6
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm12, %zmm1
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm2, %zmm4, %zmm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm11, %zmm2
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm6, %zmm4, %zmm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,0,0,1,4,4,4,5]
+; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm3 = mem[0,0,0,1,4,4,4,5]
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm0, %zmm4, %zmm3
+; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm0 = mem[0,0,0,1,4,4,4,5]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm3, %zmm9, %zmm0
+; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm3 = mem[0,0,0,1,4,4,4,5]
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm0, %zmm5, %zmm3
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm30[0,0,0,1,4,4,4,5]
+; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm6 = mem[0,0,0,1,4,4,4,5]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm0, %zmm4, %zmm6
+; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm0 = mem[0,0,0,1,4,4,4,5]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm6, %zmm9, %zmm0
+; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm6 = mem[0,0,0,1,4,4,4,5]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm0, %zmm5, %zmm6
+; AVX512DQ-NEXT:    vpermq $234, (%rsp), %zmm0 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm5 = mem[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm0, %zmm9, %zmm5
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm26[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm24[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm0, %zmm9, %zmm7
+; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm5, %zmm4, %zmm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm5 = zmm23[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm7, %zmm4, %zmm5
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm4 = zmm28[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm0, %zmm7, %zmm4
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm25[2,2,2,3,6,6,6,7]
+; AVX512DQ-NEXT:    vpternlogd $184, %zmm5, %zmm7, %zmm0
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vmovdqa64 %zmm3, 128(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, 320(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm8, 192(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm5, 256(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 320(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm6, (%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm3, 192(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, 256(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 64(%rax)
-; AVX512DQ-NEXT:    addq $328, %rsp # imm = 0x148
+; AVX512DQ-NEXT:    addq $456, %rsp # imm = 0x1C8
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i8_stride6_vf64:
 ; AVX512DQ-FCP:       # %bb.0:
-; AVX512DQ-FCP-NEXT:    subq $392, %rsp # imm = 0x188
+; AVX512DQ-FCP-NEXT:    subq $424, %rsp # imm = 0x1A8
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm4
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm4, %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm3, (%rsp) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
 ; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15],ymm3[24],ymm4[24],ymm3[25],ymm4[25],ymm3[26],ymm4[26],ymm3[27],ymm4[27],ymm3[28],ymm4[28],ymm3[29],ymm4[29],ymm3[30],ymm4[30],ymm3[31],ymm4[31]
@@ -6276,258 +6226,238 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm5
-; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm14 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm5, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm6, %ymm3
+; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm5, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm4, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[16],ymm2[16],ymm3[17],ymm2[17],ymm3[18],ymm2[18],ymm3[19],ymm2[19],ymm3[20],ymm2[20],ymm3[21],ymm2[21],ymm3[22],ymm2[22],ymm3[23],ymm2[23]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15],ymm6[24],ymm5[24],ymm6[25],ymm5[25],ymm6[26],ymm5[26],ymm6[27],ymm5[27],ymm6[28],ymm5[28],ymm6[29],ymm5[29],ymm6[30],ymm5[30],ymm6[31],ymm5[31]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm6, %ymm25
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm5, %ymm30
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm5, %ymm22
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31]
 ; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm4, %ymm4
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm7
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm6
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0]
 ; AVX512DQ-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm7, %ymm5
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm6, %ymm5
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0]
 ; AVX512DQ-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm7, %ymm6
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm7, %ymm31
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm6, %ymm6
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm8
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm8, %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm7
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm7, %ymm0
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[4],ymm5[4],ymm0[5],ymm5[5],ymm0[6],ymm5[6],ymm0[7],ymm5[7],ymm0[16],ymm5[16],ymm0[17],ymm5[17],ymm0[18],ymm5[18],ymm0[19],ymm5[19],ymm0[20],ymm5[20],ymm0[21],ymm5[21],ymm0[22],ymm5[22],ymm0[23],ymm5[23]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15],ymm7[24],ymm8[24],ymm7[25],ymm8[25],ymm7[26],ymm8[26],ymm7[27],ymm8[27],ymm7[28],ymm8[28],ymm7[29],ymm8[29],ymm7[30],ymm8[30],ymm7[31],ymm8[31]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm7
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15]
+; AVX512DQ-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm7, %ymm6
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0]
+; AVX512DQ-FCP-NEXT:    # ymm9 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm7, %ymm8
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm8, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm7
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm7, %ymm6
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm8
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm6[0],ymm0[1],ymm6[1],ymm0[2],ymm6[2],ymm0[3],ymm6[3],ymm0[4],ymm6[4],ymm0[5],ymm6[5],ymm0[6],ymm6[6],ymm0[7],ymm6[7],ymm0[16],ymm6[16],ymm0[17],ymm6[17],ymm0[18],ymm6[18],ymm0[19],ymm6[19],ymm0[20],ymm6[20],ymm0[21],ymm6[21],ymm0[22],ymm6[22],ymm0[23],ymm6[23]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11],ymm8[12],ymm7[12],ymm8[13],ymm7[13],ymm8[14],ymm7[14],ymm8[15],ymm7[15],ymm8[24],ymm7[24],ymm8[25],ymm7[25],ymm8[26],ymm7[26],ymm8[27],ymm7[27],ymm8[28],ymm7[28],ymm8[29],ymm7[29],ymm8[30],ymm7[30],ymm8[31],ymm7[31]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm8, %ymm18
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm7, %ymm17
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm11
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm11, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm12
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm12, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm15
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm15, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm11
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm11, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm12[8],ymm11[8],ymm12[9],ymm11[9],ymm12[10],ymm11[10],ymm12[11],ymm11[11],ymm12[12],ymm11[12],ymm12[13],ymm11[13],ymm12[14],ymm11[14],ymm12[15],ymm11[15],ymm12[24],ymm11[24],ymm12[25],ymm11[25],ymm12[26],ymm11[26],ymm12[27],ymm11[27],ymm12[28],ymm11[28],ymm12[29],ymm11[29],ymm12[30],ymm11[30],ymm12[31],ymm11[31]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm11[8],ymm15[8],ymm11[9],ymm15[9],ymm11[10],ymm15[10],ymm11[11],ymm15[11],ymm11[12],ymm15[12],ymm11[13],ymm15[13],ymm11[14],ymm15[14],ymm11[15],ymm15[15],ymm11[24],ymm15[24],ymm11[25],ymm15[25],ymm11[26],ymm15[26],ymm11[27],ymm15[27],ymm11[28],ymm15[28],ymm11[29],ymm15[29],ymm11[30],ymm15[30],ymm11[31],ymm15[31]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm9
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm9, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm9, %ymm1
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm3
-; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm6
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm6, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm19
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm23
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm2, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm20
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm24
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm0
+; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm5 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm12
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm12, %xmm2
 ; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm18
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,4,5,2,3,8,9,10,11,12,13,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm2, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [u,6,u,5,u,8,u,7,u,9,u,9,u,9,u,9]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm4, %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,2,u,1,u,0,u,3,u,4,u,4,u,4,u,4]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm4, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm4, %xmm19
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm2, %zmm3, %zmm26
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm3
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm5, %xmm4
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm4, %xmm4
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm4, %zmm27
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm15
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm15, %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm15, %xmm1
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm28
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm13
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0,10,0,13,0,12,0,11,0,14,0,0,0,0,0,15,0]
-; AVX512DQ-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm21
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm22 = ymm0[2,2,2,3]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0,6,0,5,0,8,0,7,0,0,0,9,0,0,0,0,0]
-; AVX512DQ-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm20
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm24 = ymm0[2,2,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm2
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm2, %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm1
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm1, %xmm10
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm12[0],xmm0[0],xmm12[1],xmm0[1],xmm12[2],xmm0[2],xmm12[3],xmm0[3],xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm16
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [0,1,6,7,4,5,2,3,8,9,10,11,12,13,10,11]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm2, %zmm26
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm13
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [6,u,5,u,8,u,7,u,9,u,9,u,9,u,9,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm13, %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,u,1,u,0,u,3,u,4,u,4,u,4,u,4,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm13, %xmm3
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm3, %zmm29
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm14
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [u,6,u,5,u,8,u,7,u,9,u,9,u,9,u,9]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm14, %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,2,u,1,u,0,u,3,u,4,u,4,u,4,u,4]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm14, %xmm4
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm4, %zmm21
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm6, %xmm0
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm5
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm5, %zmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm1
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm5
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm2
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm5, %zmm2, %zmm27
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm8
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm8, %xmm5
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm8, %xmm3
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm5, %zmm3, %zmm28
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm5
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm5, %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm3
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm3, %xmm9
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3],xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; AVX512DQ-FCP-NEXT:    vprold $16, %xmm9, %xmm9
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm9, %zmm30
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm9
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm7, %xmm0
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm9, %xmm10
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1],xmm10[2],xmm0[2],xmm10[3],xmm0[3],xmm10[4],xmm0[4],xmm10[5],xmm0[5],xmm10[6],xmm0[6],xmm10[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3],xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]
 ; AVX512DQ-FCP-NEXT:    vprold $16, %xmm10, %xmm10
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm10, %zmm23
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm4, %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm10, %xmm14
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3],xmm14[4],xmm0[4],xmm14[5],xmm0[5],xmm14[6],xmm0[6],xmm14[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; AVX512DQ-FCP-NEXT:    vprold $16, %xmm14, %xmm14
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm14, %zmm29
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm12[0],ymm11[0],ymm12[1],ymm11[1],ymm12[2],ymm11[2],ymm12[3],ymm11[3],ymm12[4],ymm11[4],ymm12[5],ymm11[5],ymm12[6],ymm11[6],ymm12[7],ymm11[7],ymm12[16],ymm11[16],ymm12[17],ymm11[17],ymm12[18],ymm11[18],ymm12[19],ymm11[19],ymm12[20],ymm11[20],ymm12[21],ymm11[21],ymm12[22],ymm11[22],ymm12[23],ymm11[23]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm10, %zmm31
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm11[0],ymm15[0],ymm11[1],ymm15[1],ymm11[2],ymm15[2],ymm11[3],ymm15[3],ymm11[4],ymm15[4],ymm11[5],ymm15[5],ymm11[6],ymm15[6],ymm11[7],ymm15[7],ymm11[16],ymm15[16],ymm11[17],ymm15[17],ymm11[18],ymm15[18],ymm11[19],ymm15[19],ymm11[20],ymm15[20],ymm11[21],ymm15[21],ymm11[22],ymm15[22],ymm11[23],ymm15[23]
 ; AVX512DQ-FCP-NEXT:    vprold $16, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15]
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[16],ymm8[16],ymm7[17],ymm8[17],ymm7[18],ymm8[18],ymm7[19],ymm8[19],ymm7[20],ymm8[20],ymm7[21],ymm8[21],ymm7[22],ymm8[22],ymm7[23],ymm8[23]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [16,17,22,23,20,21,18,19,24,25,26,27,28,29,26,27,16,17,22,23,20,21,18,19,24,25,26,27,28,29,26,27]
-; AVX512DQ-FCP-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm4, %ymm4
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm3, %xmm11
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [0,0,0,1,10,10,10,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm7, %zmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [10,11,8,9,6,7,12,13,14,15,14,15,14,15,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm10, %xmm3
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm7, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm16, %zmm11
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
-; AVX512DQ-FCP-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm9, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [u,10,u,13,u,12,u,11,u,14,u,13,u,14,u,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm15, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm5, %xmm17
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm7, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm5
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[4],ymm5[4],ymm0[5],ymm5[5],ymm0[6],ymm5[6],ymm0[7],ymm5[7],ymm0[16],ymm5[16],ymm0[17],ymm5[17],ymm0[18],ymm5[18],ymm0[19],ymm5[19],ymm0[20],ymm5[20],ymm0[21],ymm5[21],ymm0[22],ymm5[22],ymm0[23],ymm5[23]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; AVX512DQ-FCP-NEXT:    vprold $16, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm1, %xmm2
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # ymm4 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[16],mem[16],ymm0[17],mem[17],ymm0[18],mem[18],ymm0[19],mem[19],ymm0[20],mem[20],ymm0[21],mem[21],ymm0[22],mem[22],ymm0[23],mem[23]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm21, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm4, %ymm4
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm14
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm18, %xmm8
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm8[8],xmm6[9],xmm8[9],xmm6[10],xmm8[10],xmm6[11],xmm8[11],xmm6[12],xmm8[12],xmm6[13],xmm8[13],xmm6[14],xmm8[14],xmm6[15],xmm8[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm15
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm6, %xmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [6,u,5,u,8,u,7,u,9,u,9,u,9,u,9,u]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm7, %zmm8
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm15, %xmm4
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm16, %zmm8
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm9[8],xmm7[8],xmm9[9],xmm7[9],xmm9[10],xmm7[10],xmm9[11],xmm7[11],xmm9[12],xmm7[12],xmm9[13],xmm7[13],xmm9[14],xmm7[14],xmm9[15],xmm7[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm18, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm17, %ymm9
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm2[0],ymm9[0],ymm2[1],ymm9[1],ymm2[2],ymm9[2],ymm2[3],ymm9[3],ymm2[4],ymm9[4],ymm2[5],ymm9[5],ymm2[6],ymm9[6],ymm2[7],ymm9[7],ymm2[16],ymm9[16],ymm2[17],ymm9[17],ymm2[18],ymm9[18],ymm2[19],ymm9[19],ymm2[20],ymm9[20],ymm2[21],ymm9[21],ymm2[22],ymm9[22],ymm2[23],ymm9[23]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [16,17,22,23,20,21,18,19,24,25,26,27,28,29,26,27,16,17,22,23,20,21,18,19,24,25,26,27,28,29,26,27]
+; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm9, %ymm9
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm18
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm4, %xmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm17
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,0,0,1,10,10,10,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm9, %zmm4, %zmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [10,11,8,9,6,7,12,13,14,15,14,15,14,15,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm4, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm7, %zmm9, %zmm15
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
 ; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm5, %ymm12
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm13, %ymm9
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,u,1,u,0,u,3,u,4,u,4,u,4,u,4,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm14, %xmm6
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm15, %xmm5
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm14, %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [10,u,13,u,12,u,11,u,14,u,13,u,14,u,15,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm14, %xmm14
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm15, %xmm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm31, %ymm15
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm15, %ymm15
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm30 = ymm0[2,2,2,3]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm31 = ymm1[2,2,2,3]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,0,1]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,0,1]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm6[0,0,0,1]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm2[0,0,0,1]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm14[0,0,0,1]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm19, %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm17, %xmm14
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm6, %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm11, %ymm14, %ymm2
-; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm11, %ymm11
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,0,1]
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm8, %ymm14, %ymm13
-; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm8, %ymm8
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm15, %zmm7, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm8, %ymm16, %ymm9
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm7
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm13[0,1,2,3],zmm7[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm7, %zmm8, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm19, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm19
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [10,u,13,u,12,u,11,u,14,u,13,u,14,u,15,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm1, %xmm1
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm4, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm15, %zmm10, %zmm1
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
+; AVX512DQ-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [u,10,u,13,u,12,u,11,u,14,u,13,u,14,u,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm8, %xmm0
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm1, %zmm2, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm22, %ymm8
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm8[0],ymm1[1],ymm8[1],ymm1[2],ymm8[2],ymm1[3],ymm8[3],ymm1[4],ymm8[4],ymm1[5],ymm8[5],ymm1[6],ymm8[6],ymm1[7],ymm8[7],ymm1[16],ymm8[16],ymm1[17],ymm8[17],ymm1[18],ymm8[18],ymm1[19],ymm8[19],ymm1[20],ymm8[20],ymm1[21],ymm8[21],ymm1[22],ymm8[22],ymm1[23],ymm8[23]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqu (%rsp), %ymm5 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # ymm5 = ymm5[0],mem[0],ymm5[1],mem[1],ymm5[2],mem[2],ymm5[3],mem[3],ymm5[4],mem[4],ymm5[5],mem[5],ymm5[6],mem[6],ymm5[7],mem[7],ymm5[16],mem[16],ymm5[17],mem[17],ymm5[18],mem[18],ymm5[19],mem[19],ymm5[20],mem[20],ymm5[21],mem[21],ymm5[22],mem[22],ymm5[23],mem[23]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm18, %ymm6
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm16, %xmm6
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm12[8],xmm6[8],xmm12[9],xmm6[9],xmm12[10],xmm6[10],xmm12[11],xmm6[11],xmm12[12],xmm6[12],xmm12[13],xmm6[13],xmm12[14],xmm6[14],xmm12[15],xmm6[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm17, %xmm8
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm6, %xmm6
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm4, %zmm6
+; AVX512DQ-FCP-NEXT:    vprold $16, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm3
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm9, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm19, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm13, %xmm3
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm3
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm6, %zmm10, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm14, %xmm5
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm5
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm2, %zmm5
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 256(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm11, %ymm16, %ymm12
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm0, %zmm6
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm6[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm8, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 64(%rax)
-; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm2 = mem[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm3 = mem[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm16, %zmm3
-; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm3, %ymm2
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm2, %ymm14, %ymm22
-; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm2 = mem[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm6 = mem[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm2, %zmm16, %zmm6
-; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm6, %ymm2
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm2, %ymm14, %ymm30
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 256(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # zmm1 = mem[2,2,2,3,6,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm9, %zmm1
+; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm3, %ymm2, %ymm24
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm22, %zmm0, %zmm3
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm24[0,1,2,3],zmm3[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm7 = mem[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm8, %zmm7
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm6, %ymm2, %ymm31
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm30, %zmm0, %zmm3
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm31[0,1,2,3],zmm3[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq $234, (%rsp), %zmm6 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm6 = mem[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm3, %zmm8, %zmm6
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm23[0,0,0,1,4,4,4,5]
-; AVX512DQ-FCP-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm8 = mem[0,0,0,1,4,4,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm2, %zmm8
-; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm8, %ymm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm3, %ymm2, %ymm4
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm29[0,0,0,1,4,4,4,5]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm27[0,0,0,1,4,4,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm3, %zmm2, %zmm9
-; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm9, %ymm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm3, %ymm2, %ymm0
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm8, %ymm16, %ymm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %ymm9, %ymm16, %ymm1
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm2
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm5[0,1,2,3],zmm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm1, %zmm2, %zmm0
+; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # zmm1 = mem[2,2,2,3,6,6,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm1
+; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # zmm4 = mem[2,2,2,3,6,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm9, %zmm4
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm23[2,2,2,3,6,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm4, %zmm2, %zmm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm4 = zmm24[2,2,2,3,6,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm3, %zmm4
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm30[0,0,0,1,4,4,4,5]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm26[0,0,0,1,4,4,4,5]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm4, %zmm3
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm28[0,0,0,1,4,4,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm4, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm2, %zmm3
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm31[0,0,0,1,4,4,4,5]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm25[0,0,0,1,4,4,4,5]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm0, %zmm2, %zmm5
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm29[0,0,0,1,4,4,4,5]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm3, %zmm9, %zmm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm2 = zmm27[0,0,0,1,4,4,4,5]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm5, %zmm9, %zmm2
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm21[0,0,0,1,4,4,4,5]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm0, %zmm5, %zmm3
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm28[0,0,0,1,4,4,4,5]
+; AVX512DQ-FCP-NEXT:    vpternlogd $184, %zmm2, %zmm5, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 192(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 320(%rax)
-; AVX512DQ-FCP-NEXT:    addq $392, %rsp # imm = 0x188
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 128(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 320(%rax)
+; AVX512DQ-FCP-NEXT:    addq $424, %rsp # imm = 0x1A8
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index 8091afbbfd70c..416fbe9aa340c 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -4254,7 +4254,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm9, %zmm1
 ; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
 ; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm14
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm14[u,u,u,u,u,14],zero,ymm14[u,u,u,u,u,15],zero,ymm14[u,u,u,u,u,16],zero,ymm14[u,u,u,u,u,17],zero,ymm14[u,u,u]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm14[u],zero,zero,zero,zero,ymm14[14],zero,ymm14[u],zero,zero,zero,zero,ymm14[15],zero,ymm14[u],zero,zero,zero,zero,ymm14[16],zero,ymm14[u],zero,zero,zero,zero,ymm14[17],zero,ymm14[u],zero,zero
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
 ; AVX512-NEXT:    vpor %ymm0, %ymm9, %ymm0
 ; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
@@ -4357,7 +4357,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm8, %zmm9
 ; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm9
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm1[u,u,u,u,u,14],zero,ymm1[u,u,u,u,u,15],zero,ymm1[u,u,u,u,u,16],zero,ymm1[u,u,u,u,u,17],zero,ymm1[u,u,u]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm1[u],zero,zero,zero,zero,ymm1[14],zero,ymm1[u],zero,zero,zero,zero,ymm1[15],zero,ymm1[u],zero,zero,zero,zero,ymm1[16],zero,ymm1[u],zero,zero,zero,zero,ymm1[17],zero,ymm1[u],zero,zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm8, %ymm0
 ; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm14[8],xmm13[8],xmm14[9],xmm13[9],xmm14[10],xmm13[10],xmm14[11],xmm13[11],xmm14[12],xmm13[12],xmm14[13],xmm13[13],xmm14[14],xmm13[14],xmm14[15],xmm13[15]
@@ -4533,7 +4533,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm9, %zmm1
 ; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm14
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm14[u,u,u,u,u,14],zero,ymm14[u,u,u,u,u,15],zero,ymm14[u,u,u,u,u,16],zero,ymm14[u,u,u,u,u,17],zero,ymm14[u,u,u]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm14[u],zero,zero,zero,zero,ymm14[14],zero,ymm14[u],zero,zero,zero,zero,ymm14[15],zero,ymm14[u],zero,zero,zero,zero,ymm14[16],zero,ymm14[u],zero,zero,zero,zero,ymm14[17],zero,ymm14[u],zero,zero
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
 ; AVX512DQ-NEXT:    vpor %ymm0, %ymm9, %ymm0
 ; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
@@ -4636,7 +4636,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm8, %zmm9
 ; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm9
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm1[u,u,u,u,u,14],zero,ymm1[u,u,u,u,u,15],zero,ymm1[u,u,u,u,u,16],zero,ymm1[u,u,u,u,u,17],zero,ymm1[u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm1[u],zero,zero,zero,zero,ymm1[14],zero,ymm1[u],zero,zero,zero,zero,ymm1[15],zero,ymm1[u],zero,zero,zero,zero,ymm1[16],zero,ymm1[u],zero,zero,zero,zero,ymm1[17],zero,ymm1[u],zero,zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm8, %ymm0
 ; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm14[8],xmm13[8],xmm14[9],xmm13[9],xmm14[10],xmm13[10],xmm14[11],xmm13[11],xmm14[12],xmm13[12],xmm14[13],xmm13[13],xmm14[14],xmm13[14],xmm14[15],xmm13[15]
@@ -8544,39 +8544,35 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ;
 ; AVX512-LABEL: store_i8_stride7_vf64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    subq $1720, %rsp # imm = 0x6B8
+; AVX512-NEXT:    subq $1384, %rsp # imm = 0x568
 ; AVX512-NEXT:    vmovdqa (%rsi), %ymm7
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
-; AVX512-NEXT:    vpshufb %ymm2, %ymm7, %ymm0
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm26
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm7[14],zero,zero,zero,zero,zero,zero,ymm7[15],zero,zero,zero,zero,zero,zero,ymm7[16],zero,zero,zero,zero,zero,zero,ymm7[17],zero,zero,zero,zero,zero,zero,ymm7[18]
 ; AVX512-NEXT:    vmovdqa (%rdi), %ymm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
-; AVX512-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm25
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm18
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[0,1,14],zero,ymm2[12,13,0,1,14,15],zero,ymm2[3,12,13,2,3,16],zero,ymm2[30,31,28,29,16,17],zero,ymm2[31,18,19,28,29,18],zero
+; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm20
 ; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa (%rcx), %ymm14
+; AVX512-NEXT:    vmovdqa (%rcx), %ymm15
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512-NEXT:    vpshufb %ymm2, %ymm14, %ymm0
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512-NEXT:    vmovdqa (%rdx), %ymm8
+; AVX512-NEXT:    vpshufb %ymm2, %ymm15, %ymm0
+; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm23
+; AVX512-NEXT:    vmovdqa (%rdx), %ymm2
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512-NEXT:    vpshufb %ymm3, %ymm8, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm23
-; AVX512-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm26
+; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm18
 ; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa (%r8), %ymm15
+; AVX512-NEXT:    vmovdqa (%r8), %ymm14
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
-; AVX512-NEXT:    vpshufb %ymm2, %ymm15, %ymm0
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm24
-; AVX512-NEXT:    vmovdqa (%r9), %ymm2
+; AVX512-NEXT:    vpshufb %ymm2, %ymm14, %ymm0
+; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm27
+; AVX512-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovdqa (%r9), %ymm8
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0,13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0]
 ; AVX512-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm27
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm19
+; AVX512-NEXT:    vpshufb %ymm3, %ymm8, %ymm1
+; AVX512-NEXT:    vmovdqa64 %ymm3, %ymm17
 ; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm10
@@ -8589,7 +8585,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [12,13,14,128,12,128,14,15,14,15,128,13,128,15,12,13,28,29,30,128,28,128,30,31,30,31,128,29,128,31,28,29]
 ; AVX512-NEXT:    vpshufb %ymm1, %ymm9, %ymm0
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm16
+; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm19
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm10[23,u,u,u],zero,ymm10[26],zero,ymm10[24,u,u,u],zero,ymm10[27],zero
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -8604,7 +8600,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0]
 ; AVX512-NEXT:    # ymm0 = mem[0,1,0,1]
 ; AVX512-NEXT:    vpshufb %ymm0, %ymm4, %ymm2
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm20
+; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm21
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm5[23],zero,ymm5[21,22,23,26],zero,ymm5[24],zero,ymm5[28,29,26,27]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
 ; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -8612,7 +8608,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29]
 ; AVX512-NEXT:    # ymm0 = mem[0,1,0,1]
 ; AVX512-NEXT:    vpshufb %ymm0, %ymm3, %ymm11
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm21
+; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm25
 ; AVX512-NEXT:    vmovdqa 32(%r9), %ymm2
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm12 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm11
@@ -8631,23 +8627,23 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
 ; AVX512-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vpshufb %ymm6, %ymm7, %ymm6
-; AVX512-NEXT:    vmovdqa64 %ymm7, %ymm17
-; AVX512-NEXT:    vmovdqa64 %ymm18, %ymm13
-; AVX512-NEXT:    vmovdqa64 %ymm16, %ymm0
-; AVX512-NEXT:    vpshufb %ymm0, %ymm13, %ymm7
+; AVX512-NEXT:    vmovdqa64 %ymm7, %ymm22
+; AVX512-NEXT:    vmovdqa64 %ymm20, %ymm7
+; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm0
+; AVX512-NEXT:    vpshufb %ymm0, %ymm7, %ymm7
 ; AVX512-NEXT:    vpor %ymm6, %ymm7, %ymm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufb %ymm1, %ymm8, %ymm6
-; AVX512-NEXT:    vmovdqa64 %ymm20, %ymm0
-; AVX512-NEXT:    vpshufb %ymm0, %ymm14, %ymm7
-; AVX512-NEXT:    vmovdqa64 %ymm14, %ymm22
+; AVX512-NEXT:    vmovdqa64 %ymm18, %ymm12
+; AVX512-NEXT:    vpshufb %ymm1, %ymm12, %ymm6
+; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm0
+; AVX512-NEXT:    vpshufb %ymm0, %ymm15, %ymm7
+; AVX512-NEXT:    vmovdqa64 %ymm15, %ymm19
 ; AVX512-NEXT:    vpor %ymm6, %ymm7, %ymm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm0
-; AVX512-NEXT:    vpshufb %ymm0, %ymm15, %ymm6
-; AVX512-NEXT:    vmovdqa64 %ymm15, %ymm16
-; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm15
-; AVX512-NEXT:    vpshufb %ymm11, %ymm15, %ymm7
+; AVX512-NEXT:    vmovdqa64 %ymm25, %ymm0
+; AVX512-NEXT:    vpshufb %ymm0, %ymm14, %ymm6
+; AVX512-NEXT:    vpshufb %ymm11, %ymm8, %ymm7
+; AVX512-NEXT:    vmovdqa64 %ymm8, %ymm16
 ; AVX512-NEXT:    vpor %ymm6, %ymm7, %ymm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm0
@@ -8656,379 +8652,348 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm11 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
 ; AVX512-NEXT:    vpshufb %xmm11, %xmm1, %xmm6
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm12 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512-NEXT:    vpshufb %xmm12, %xmm0, %xmm7
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm7
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm21
 ; AVX512-NEXT:    vpor %xmm6, %xmm7, %xmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm0
-; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm14
+; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm15
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm14, %xmm7
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm8
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm18
+; AVX512-NEXT:    vpshufb %xmm6, %xmm15, %xmm7
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm13 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm0, %xmm8
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm30
 ; AVX512-NEXT:    vpor %xmm7, %xmm8, %xmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,2,3,0,1,0,18,0,19,18,0,19,0]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm28 = [0,0,0,0,2,3,0,1,0,18,0,19,18,0,19,0]
 ; AVX512-NEXT:    vmovdqa 32(%rax), %xmm0
 ; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm0[0,1,2,3,4,5,5,6]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm8 = xmm0[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-NEXT:    vpermi2d %zmm7, %zmm8, %zmm1
-; AVX512-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpermi2d %zmm7, %zmm8, %zmm28
 ; AVX512-NEXT:    vmovdqa 32(%r9), %xmm0
-; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa 32(%r8), %xmm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm8 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512-NEXT:    vpshufb %xmm8, %xmm0, %xmm7
-; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm20
+; AVX512-NEXT:    vmovdqa 32(%r8), %xmm14
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm7
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm24
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm18
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm30
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm29
-; AVX512-NEXT:    vpor %xmm7, %xmm8, %xmm0
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa64 %ymm28, %ymm0
-; AVX512-NEXT:    vpshufb %ymm0, %ymm4, %ymm7
+; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm8
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm29
+; AVX512-NEXT:    vporq %xmm7, %xmm8, %xmm31
 ; AVX512-NEXT:    vmovdqa64 %ymm23, %ymm0
+; AVX512-NEXT:    vpshufb %ymm0, %ymm4, %ymm7
+; AVX512-NEXT:    vmovdqa64 %ymm26, %ymm0
 ; AVX512-NEXT:    vpshufb %ymm0, %ymm5, %ymm8
 ; AVX512-NEXT:    vpor %ymm7, %ymm8, %ymm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20]
 ; AVX512-NEXT:    # ymm1 = mem[0,1,0,1]
 ; AVX512-NEXT:    vpshufb %ymm1, %ymm4, %ymm0
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm21
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128]
-; AVX512-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm5, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm4, %ymm19
+; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm23
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128]
+; AVX512-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm7, %ymm5, %ymm1
 ; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vmovdqa64 %ymm26, %ymm0
-; AVX512-NEXT:    vpshufb %ymm0, %ymm10, %ymm0
-; AVX512-NEXT:    vmovdqa64 %ymm25, %ymm1
-; AVX512-NEXT:    vpshufb %ymm1, %ymm9, %ymm1
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm10[14],zero,zero,zero,zero,zero,zero,ymm10[15],zero,zero,zero,zero,zero,zero,ymm10[16],zero,zero,zero,zero,zero,zero,ymm10[17],zero,zero,zero,zero,zero,zero,ymm10[18]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm9[0,1,14],zero,ymm9[12,13,0,1,14,15],zero,ymm9[3,12,13,2,3,16],zero,ymm9[30,31,28,29,16,17],zero,ymm9[31,18,19,28,29,18],zero
 ; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
-; AVX512-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm7, %ymm10, %ymm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [2,3,4,5,128,3,128,5,4,5,6,128,4,128,6,7,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
-; AVX512-NEXT:    vpshufb %ymm1, %ymm9, %ymm4
-; AVX512-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22]
-; AVX512-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm10, %ymm2, %ymm4
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128]
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
 ; AVX512-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm0, %ymm3, %ymm5
+; AVX512-NEXT:    vpshufb %ymm0, %ymm10, %ymm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm8 = [2,3,4,5,128,3,128,5,4,5,6,128,4,128,6,7,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
+; AVX512-NEXT:    vpshufb %ymm8, %ymm9, %ymm4
+; AVX512-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22]
+; AVX512-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm1, %ymm2, %ymm4
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128]
+; AVX512-NEXT:    # ymm9 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm9, %ymm3, %ymm5
 ; AVX512-NEXT:    vpor %ymm4, %ymm5, %ymm4
 ; AVX512-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa64 %ymm24, %ymm4
-; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
 ; AVX512-NEXT:    vmovdqa64 %ymm27, %ymm4
+; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512-NEXT:    vmovdqa64 %ymm17, %ymm4
 ; AVX512-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
 ; AVX512-NEXT:    vpor %ymm3, %ymm2, %ymm2
 ; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqa (%rsi), %xmm3
 ; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm2
-; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm31
+; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm25
 ; AVX512-NEXT:    vmovdqa (%rdi), %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm3
-; AVX512-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm28
+; AVX512-NEXT:    vpshufb %xmm13, %xmm4, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm27
 ; AVX512-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vmovdqa (%rcx), %xmm3
 ; AVX512-NEXT:    vpshufb %xmm11, %xmm3, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm3, %xmm6
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm4
-; AVX512-NEXT:    vpshufb %xmm12, %xmm4, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm18
+; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm17
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm13
+; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm3
+; AVX512-NEXT:    vpshufb %xmm3, %xmm13, %xmm3
 ; AVX512-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa (%r9), %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm2
-; AVX512-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm3, %xmm8
-; AVX512-NEXT:    vmovdqa (%r8), %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm4
-; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
-; AVX512-NEXT:    vmovdqa %xmm3, %xmm9
-; AVX512-NEXT:    vpor %xmm2, %xmm4, %xmm2
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm13[23],zero,zero,zero,zero,ymm13[26],zero,ymm13[24],zero,zero,zero,zero,ymm13[27],zero,ymm13[25]
-; AVX512-NEXT:    vmovdqa64 %ymm17, %ymm3
-; AVX512-NEXT:    vpshufb %ymm7, %ymm3, %ymm4
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm4, %zmm2
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm3[23,u,u,u],zero,ymm3[26],zero,ymm3[24,u,u,u],zero,ymm3[27],zero
-; AVX512-NEXT:    vpshufb %ymm1, %ymm13, %ymm1
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa64 %ymm22, %ymm2
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero,zero,zero
-; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm3
-; AVX512-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm2[23],zero,ymm2[21,22,23,26],zero,ymm2[24],zero,ymm2[28,29,26,27]
-; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm3
-; AVX512-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm30
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm15[25],zero,ymm15[23],zero,zero,zero,zero,ymm15[26],zero,ymm15[24],zero,zero
-; AVX512-NEXT:    vmovdqa64 %ymm16, %ymm3
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm3[23],zero,ymm3[23,24,25,26],zero,ymm3[24],zero,ymm3[30,31]
-; AVX512-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufb %ymm10, %ymm15, %ymm1
+; AVX512-NEXT:    vmovdqa (%r9), %xmm11
+; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm2
+; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
+; AVX512-NEXT:    vmovdqa (%r8), %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm3
+; AVX512-NEXT:    vpshufb %xmm3, %xmm10, %xmm3
+; AVX512-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovdqa64 %ymm20, %ymm4
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[23],zero,zero,zero,zero,ymm4[26],zero,ymm4[24],zero,zero,zero,zero,ymm4[27],zero,ymm4[25]
+; AVX512-NEXT:    vmovdqa64 %ymm22, %ymm3
 ; AVX512-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa (%rax), %ymm4
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm4[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512-NEXT:    vpermi2d %zmm0, %zmm1, %zmm15
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm9[8],xmm8[8],xmm9[9],xmm8[9],xmm9[10],xmm8[10],xmm9[11],xmm8[11],xmm9[12],xmm8[12],xmm9[13],xmm8[13],xmm9[14],xmm8[14],xmm9[15],xmm8[15]
-; AVX512-NEXT:    vmovdqa64 %xmm9, %xmm21
-; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm22
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm24
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm1[0,1,0,1],zmm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqa (%rax), %xmm11
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm11[0,1,2,3,4,5,5,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
-; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm3[23,u,u,u],zero,ymm3[26],zero,ymm3[24,u,u,u],zero,ymm3[27],zero
+; AVX512-NEXT:    vpshufb %ymm8, %ymm4, %ymm2
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm29
+; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm2
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero,zero,zero
+; AVX512-NEXT:    vmovdqa64 %ymm23, %ymm3
+; AVX512-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm26
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm12[23],zero,ymm12[21,22,23,26],zero,ymm12[24],zero,ymm12[28,29,26,27]
+; AVX512-NEXT:    vpshufb %ymm7, %ymm12, %ymm2
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm23
+; AVX512-NEXT:    vmovdqa64 %ymm16, %ymm2
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
+; AVX512-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm1[23],zero,ymm1[23,24,25,26],zero,ymm1[24],zero,ymm1[30,31]
+; AVX512-NEXT:    vpshufb %ymm9, %ymm1, %ymm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa (%rax), %ymm8
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm8[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512-NEXT:    vpermi2d %zmm1, %zmm2, %zmm16
+; AVX512-NEXT:    vmovdqa (%rax), %xmm5
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,4,5,5,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [128,13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128]
-; AVX512-NEXT:    vpshufb %ymm9, %ymm4, %ymm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm16
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3],xmm10[4],xmm14[4],xmm10[5],xmm14[5],xmm10[6],xmm14[6],xmm10[7],xmm14[7]
+; AVX512-NEXT:    vpshufb %ymm9, %ymm8, %ymm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm19
+; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm4
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm4[0],xmm15[0],xmm4[1],xmm15[1],xmm4[2],xmm15[2],xmm4[3],xmm15[3],xmm4[4],xmm15[4],xmm4[5],xmm15[5],xmm4[6],xmm15[6],xmm4[7],xmm15[7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm27
+; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm24
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm12 = zmm0[2,3,2,3],zmm2[0,1,0,1]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm0[2,3,2,3],zmm3[0,1,0,1]
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm7[0],xmm1[1],xmm7[1],xmm1[2],xmm7[2],xmm1[3],xmm7[3],xmm1[4],xmm7[4],xmm1[5],xmm7[5],xmm1[6],xmm7[6],xmm1[7],xmm7[7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm26
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm21
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm17 = zmm0[2,3,2,3],zmm3[0,1,0,1]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm3
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm0[2,3,2,3],zmm3[0,1,0,1]
+; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm2
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm12 = xmm14[0],xmm2[0],xmm14[1],xmm2[1],xmm14[2],xmm2[2],xmm14[3],xmm2[3],xmm14[4],xmm2[4],xmm14[5],xmm2[5],xmm14[6],xmm2[6],xmm14[7],xmm2[7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm13
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm25
+; AVX512-NEXT:    vpshufb %xmm0, %xmm12, %xmm12
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm20
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm29 = zmm0[2,3,2,3],zmm13[0,1,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
-; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm20
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; AVX512-NEXT:    vmovdqa %xmm6, %xmm8
-; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm7
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm30 = zmm0[2,3,2,3],zmm12[0,1,0,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
+; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm18
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15]
+; AVX512-NEXT:    vmovdqa64 %xmm17, %xmm3
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm3[8],xmm13[8],xmm3[9],xmm13[9],xmm3[10],xmm13[10],xmm3[11],xmm13[11],xmm3[12],xmm13[12],xmm3[13],xmm13[13],xmm3[14],xmm13[14],xmm3[15],xmm13[15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm2
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm18
+; AVX512-NEXT:    vpshufb %xmm1, %xmm8, %xmm12
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm19
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm14[8],xmm10[8],xmm14[9],xmm10[9],xmm14[10],xmm10[10],xmm14[11],xmm10[11],xmm14[12],xmm10[12],xmm14[13],xmm10[13],xmm14[14],xmm10[14],xmm14[15],xmm10[15]
-; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm6
-; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm4
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; AVX512-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm8
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm4[8],xmm15[9],xmm4[9],xmm15[10],xmm4[10],xmm15[11],xmm4[11],xmm15[12],xmm4[12],xmm15[13],xmm4[13],xmm15[14],xmm4[14],xmm15[15],xmm4[15]
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm7
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm4[8],xmm7[8],xmm4[9],xmm7[9],xmm4[10],xmm7[10],xmm4[11],xmm7[11],xmm4[12],xmm7[12],xmm4[13],xmm7[13],xmm4[14],xmm7[14],xmm4[15],xmm7[15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm13
+; AVX512-NEXT:    vpshufb %xmm0, %xmm15, %xmm15
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
-; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm2
-; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm14 = zmm2[0,1,0,1],zmm1[0,1,0,1]
-; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX512-NEXT:    vpshufb %ymm9, %ymm10, %ymm1
-; AVX512-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm10[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm17
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm14[8],xmm2[8],xmm14[9],xmm2[9],xmm14[10],xmm2[10],xmm14[11],xmm2[11],xmm14[12],xmm2[12],xmm14[13],xmm2[13],xmm14[14],xmm2[14],xmm14[15],xmm2[15]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm10[8],xmm11[8],xmm10[9],xmm11[9],xmm10[10],xmm11[10],xmm10[11],xmm11[11],xmm10[12],xmm11[12],xmm10[13],xmm11[13],xmm10[14],xmm11[14],xmm10[15],xmm11[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm14
+; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vinserti32x4 $2, %xmm0, %zmm31, %zmm31
+; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-NEXT:    vpshufb %ymm9, %ymm0, %ymm1
+; AVX512-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm0[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[0,1,1,3,4,5,5,7]
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
-; AVX512-NEXT:    vpternlogq $226, %zmm12, %zmm10, %zmm17
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm2 = mem[2,3,2,3]
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm28 # 32-byte Folded Reload
-; AVX512-NEXT:    # ymm28 = mem[2,3,2,3]
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-NEXT:    vinserti64x4 $1, %ymm28, %zmm3, %zmm28
-; AVX512-NEXT:    vpternlogq $226, %zmm2, %zmm10, %zmm28
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm9[2,2,3,2]
-; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm10
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
-; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm2
-; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm27 # 16-byte Folded Reload
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3],xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm1
-; AVX512-NEXT:    vpshufb %xmm1, %xmm2, %xmm2
-; AVX512-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
-; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm3
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm1
-; AVX512-NEXT:    vpshufb %xmm1, %xmm8, %xmm8
-; AVX512-NEXT:    vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm8 = zmm8[0,1,0,1],mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm9 = xmm11[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm11[1,1,0,0,4,5,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,3,2]
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm1, %zmm9
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
+; AVX512-NEXT:    vpternlogq $226, %zmm6, %zmm1, %zmm22
+; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm6 = mem[2,3,2,3]
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm6
+; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm27 # 32-byte Folded Reload
+; AVX512-NEXT:    # ymm27 = mem[2,3,2,3]
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vinserti64x4 $1, %ymm27, %zmm0, %zmm27
+; AVX512-NEXT:    vpternlogq $226, %zmm6, %zmm1, %zmm27
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
+; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm0
+; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
+; AVX512-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 16-byte Folded Reload
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm13[0],xmm3[0],xmm13[1],xmm3[1],xmm13[2],xmm3[2],xmm13[3],xmm3[3],xmm13[4],xmm3[4],xmm13[5],xmm3[5],xmm13[6],xmm3[6],xmm13[7],xmm3[7]
+; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm0
+; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm6
+; AVX512-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 16-byte Folded Reload
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3],xmm10[4],xmm11[4],xmm10[5],xmm11[5],xmm10[6],xmm11[6],xmm10[7],xmm11[7]
+; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm0
+; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm10
+; AVX512-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 16-byte Folded Reload
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm11 = xmm5[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[1,1,0,0,4,5,6,7]
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[0,1,2,0]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm5, %zmm5
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm9 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm11 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vporq %zmm9, %zmm11, %zmm9
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm11 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm12 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vporq %zmm11, %zmm12, %zmm11
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm11
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm9 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm12 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vporq %zmm9, %zmm12, %zmm9
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
-; AVX512-NEXT:    vpternlogq $184, %zmm11, %zmm12, %zmm9
-; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm13[0,1,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm18[0,1,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm20[2,3,2,3]
-; AVX512-NEXT:    vpshuflw $5, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm1 = mem[1,1,0,0,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX512-NEXT:    vpermq {{.*#+}} zmm13 = zmm19[0,1,0,1,4,5,4,5]
-; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,1,0,1,4,5,4,5]
-; AVX512-NEXT:    vpternlogq $226, %zmm13, %zmm12, %zmm0
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm13 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm21 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vporq %zmm13, %zmm21, %zmm13
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm20 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} zmm18 = zmm30[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vporq %zmm20, %zmm18, %zmm18
-; AVX512-NEXT:    vpternlogq $226, %zmm13, %zmm12, %zmm18
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 32-byte Folded Reload
-; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 32-byte Folded Reload
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm4
-; AVX512-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm23, %zmm16
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm16
-; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm4 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm4
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,0]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm29, %zmm1
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm17, %zmm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm5
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 64-byte Folded Reload
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm13 # 64-byte Folded Reload
+; AVX512-NEXT:    vpermq {{.*#+}} zmm11 = zmm11[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} zmm13 = zmm13[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm13
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 64-byte Folded Reload
+; AVX512-NEXT:    vpermq {{.*#+}} zmm11 = zmm11[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
+; AVX512-NEXT:    vpternlogq $184, %zmm13, %zmm21, %zmm11
+; AVX512-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[0,1,0,1,4,5,4,5]
+; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm17[0,1,0,1,4,5,4,5]
+; AVX512-NEXT:    vpternlogq $226, %zmm8, %zmm21, %zmm7
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm15[0,1,0,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,0,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm14[0,1,0,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm14 = ymm18[2,3,2,3]
+; AVX512-NEXT:    vpshuflw $5, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm15 = mem[1,1,0,0,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[0,1,2,0]
+; AVX512-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm20 # 64-byte Folded Reload
+; AVX512-NEXT:    vporq %zmm26, %zmm23, %zmm17
+; AVX512-NEXT:    vpermq {{.*#+}} zmm18 = zmm20[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} zmm17 = zmm17[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpternlogq $226, %zmm18, %zmm21, %zmm17
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 32-byte Folded Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm12 # 32-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm12
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm8 # 32-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm19
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm19
+; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm8 = mem[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm8
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm15[0,0,1,0]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm14, %zmm11
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm30, %zmm11
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm11
+; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm31[0,1,0,1,4,5,4,5]
+; AVX512-NEXT:    vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm0
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm0
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-NEXT:    vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm14
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm14
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[2,3,2,3]
-; AVX512-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm10
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm10
-; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm27[0,1,0,1,4,5,4,5]
-; AVX512-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[0,1,0,1,4,5,4,5]
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2
-; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm5[0,0,1,0,4,4,5,4]
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm0
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm2 = zmm2[2,3,2,3],mem[2,3,2,3]
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm15
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm15
+; AVX512-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
+; AVX512-NEXT:    # zmm3 = zmm3[0,1,2,3],mem[2,3,2,3]
+; AVX512-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm9
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm9
+; AVX512-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5]
+; AVX512-NEXT:    vpermq {{.*#+}} zmm3 = zmm6[0,1,0,1,4,5,4,5]
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm3
+; AVX512-NEXT:    vpermq {{.*#+}} zmm1 = zmm10[0,1,0,1,4,5,4,5]
+; AVX512-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[0,0,1,0,4,4,5,4]
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm5
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5
+; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
+; AVX512-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm16
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm17, %zmm16
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vmovdqa64 %zmm15, 128(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm10, 320(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm14, 256(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm1, 192(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm4, 384(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm16, 64(%rax)
-; AVX512-NEXT:    addq $1720, %rsp # imm = 0x6B8
+; AVX512-NEXT:    vmovdqa64 %zmm16, 128(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm9, 320(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm0, 256(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm11, 192(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm8, 384(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm19, 64(%rax)
+; AVX512-NEXT:    addq $1384, %rsp # imm = 0x568
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i8_stride7_vf64:
 ; AVX512-FCP:       # %bb.0:
 ; AVX512-FCP-NEXT:    subq $1432, %rsp # imm = 0x598
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm2
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm2[30],zero,ymm2[28],zero,zero,zero,zero,ymm2[31],zero,ymm2[29],zero,zero,zero
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
-; AVX512-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm3, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm3
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm3[30],zero,ymm3[28],zero,zero,zero,zero,ymm3[31],zero,ymm3[29],zero,zero,zero
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm2
+; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
+; AVX512-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm25
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm3[28],zero,ymm3[30,31,30,31],zero,ymm3[29],zero,ymm3[31,28,29]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm25
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [25,128,23,0,0,0,128,26,128,24,0,0,0,128,27,128,25,128,23,0,0,0,128,26,128,24,0,0,0,128,27,128]
-; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm2[28],zero,ymm2[30,31,30,31],zero,ymm2[29],zero,ymm2[31,28,29]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [25,128,23,0,0,0,128,26,128,24,0,0,0,128,27,128,25,128,23,0,0,0,128,26,128,24,0,0,0,128,27,128]
+; AVX512-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm3, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm16
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm27
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm18
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm2
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[30],zero,ymm2[28],zero,zero,zero,zero,ymm2[31],zero,ymm2[29],zero,zero
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm3
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
-; AVX512-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm28
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm1
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm1[30],zero,ymm1[28],zero,zero,zero,zero,ymm1[31],zero,ymm1[29],zero,zero
+; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm15
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
+; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm15, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm29
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u],zero,ymm3[30],zero,ymm3[28,u,u,u],zero,ymm3[31],zero,ymm3[29,u]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm19
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u],zero,ymm15[30],zero,ymm15[28,u,u,u],zero,ymm15[31],zero,ymm15[29,u]
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
 ; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm24
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm17
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm31
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm26
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm3
-; AVX512-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm3[27],zero,zero,zero,zero,ymm3[30],zero,ymm3[28],zero,zero,zero,zero,ymm3[31],zero,ymm3[29]
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm2
-; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm2
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[27],zero,zero,zero,zero,ymm2[30],zero,ymm2[28],zero,zero,zero,zero,ymm2[31],zero,ymm2[29]
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
 ; AVX512-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm23
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm21
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm2[27,u,u,u],zero,ymm2[30],zero,ymm2[28,u,u,u],zero,ymm2[31],zero
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
-; AVX512-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm3, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm22
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm3[27,u,u,u],zero,ymm3[30],zero,ymm3[28,u,u,u],zero,ymm3[31],zero
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm23
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
+; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm20
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm19
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
@@ -9038,10 +9003,10 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm5
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm5[14],zero,zero,zero,zero,zero,zero,ymm5[15],zero,zero,zero,zero,zero,zero,ymm5[16],zero,zero,zero,zero,zero,zero,ymm5[17],zero,zero,zero,zero,zero,zero,ymm5[18]
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm4
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm4[0,1,14],zero,ymm4[12,13,0,1,14,15],zero,ymm4[3,12,13,2,3,16],zero,ymm4[30,31,28,29,16,17],zero,ymm4[31,18,19,28,29,18],zero
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm6
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm6[14],zero,zero,zero,zero,zero,zero,ymm6[15],zero,zero,zero,zero,zero,zero,ymm6[16],zero,zero,zero,zero,zero,zero,ymm6[17],zero,zero,zero,zero,zero,zero,ymm6[18]
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm5
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm5[0,1,14],zero,ymm5[12,13,0,1,14,15],zero,ymm5[3,12,13,2,3,16],zero,ymm5[30,31,28,29,16,17],zero,ymm5[31,18,19,28,29,18],zero
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm3
@@ -9051,366 +9016,351 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm1
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,ymm1[14],zero,zero,zero,zero,zero,zero,ymm1[15],zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,ymm1[17],zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm0
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm0[13,u,u,u,u,u],zero,ymm0[14,u,u,u,u,u],zero,ymm0[15,u,u,u,u,u],zero,ymm0[16,u,u,u,u,u],zero,ymm0[17,u,u,u]
-; AVX512-FCP-NEXT:    vpor %ymm6, %ymm7, %ymm6
-; AVX512-FCP-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,ymm1[14],zero,zero,zero,zero,zero,zero,ymm1[15],zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,ymm1[17],zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm14
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm14[13,u,u,u,u,u],zero,ymm14[14,u,u,u,u,u],zero,ymm14[15,u,u,u,u,u],zero,ymm14[16,u,u,u,u,u],zero,ymm14[17,u,u,u]
+; AVX512-FCP-NEXT:    vpor %ymm4, %ymm7, %ymm4
+; AVX512-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm9
 ; AVX512-FCP-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm7
-; AVX512-FCP-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
+; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm0, %xmm7
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
 ; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm9, %xmm9
-; AVX512-FCP-NEXT:    vpor %xmm7, %xmm9, %xmm7
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm11
-; AVX512-FCP-NEXT:    vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpor %xmm7, %xmm9, %xmm0
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm0
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
 ; AVX512-FCP-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
 ; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm9, %xmm9
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm10 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm11, %xmm11
-; AVX512-FCP-NEXT:    vpor %xmm9, %xmm11, %xmm9
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm11
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm28
+; AVX512-FCP-NEXT:    vpor %xmm9, %xmm11, %xmm0
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm9
 ; AVX512-FCP-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm14
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm9, %xmm9
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm9, %xmm9
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm14, %xmm11
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm14, %xmm30
-; AVX512-FCP-NEXT:    vpor %xmm9, %xmm11, %xmm9
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm9
-; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm9, %xmm7
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm9, %xmm26
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm9
-; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm9, %xmm10
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm9, %xmm21
-; AVX512-FCP-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX512-FCP-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm7
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm10, %xmm8
-; AVX512-FCP-NEXT:    vpor %xmm6, %xmm8, %xmm6
-; AVX512-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm11
-; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm11, %xmm12
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm6
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm13
-; AVX512-FCP-NEXT:    vmovdqa %xmm6, %xmm9
-; AVX512-FCP-NEXT:    vporq %xmm12, %xmm13, %xmm31
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm4, %ymm12
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
-; AVX512-FCP-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm5, %ymm13
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm6
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm27, %ymm6
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm12
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
-; AVX512-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm4, %ymm13
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm6, %ymm20
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm29
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm28, %ymm6
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm12
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20]
-; AVX512-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm13
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm6, %ymm16
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm28
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm24, %ymm6
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm2, %ymm13
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128]
+; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm0, %xmm12
+; AVX512-FCP-NEXT:    vpor %xmm9, %xmm12, %xmm0
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm0
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm0, %xmm7
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm24
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm10
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm17
+; AVX512-FCP-NEXT:    vpor %xmm7, %xmm10, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm10
+; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm9
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm9, %xmm8
+; AVX512-FCP-NEXT:    vpor %xmm4, %xmm8, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm8
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm8, %xmm11
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm7
+; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm7, %xmm13
+; AVX512-FCP-NEXT:    vporq %xmm11, %xmm13, %xmm30
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm25, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm5, %ymm11
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
 ; AVX512-FCP-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm2, %ymm15
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm15, %zmm27
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm23, %ymm6
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm13
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm22, %ymm6
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm15
-; AVX512-FCP-NEXT:    vpor %ymm13, %ymm15, %ymm6
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22]
-; AVX512-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm15
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm6, %ymm24
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128]
-; AVX512-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm1, %ymm14
-; AVX512-FCP-NEXT:    vpor %ymm15, %ymm14, %ymm6
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm5[30],zero,ymm5[28],zero,zero,zero,zero,ymm5[31],zero,ymm5[29],zero,zero,zero
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm4[28],zero,ymm4[30,31,30,31],zero,ymm4[29],zero,ymm4[31,28,29]
-; AVX512-FCP-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm6, %ymm13
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm0
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm16, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm6, %ymm11
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
+; AVX512-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm5, %ymm13
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm22
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm0
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm29, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm11
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20]
+; AVX512-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm13
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm29
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm0
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm31, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm11
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128]
+; AVX512-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm13
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm25
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm21, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm11
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22]
+; AVX512-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm13
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm18
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm0
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm1, %ymm11
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128]
+; AVX512-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm1, %ymm13
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm16
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm0
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm6[30],zero,ymm6[28],zero,zero,zero,zero,ymm6[31],zero,ymm6[29],zero,zero,zero
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm5[28],zero,ymm5[30,31,30,31],zero,ymm5[29],zero,ymm5[31,28,29]
+; AVX512-FCP-NEXT:    vporq %ymm6, %ymm5, %ymm20
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[30],zero,ymm2[28],zero,zero,zero,zero,ymm2[31],zero,ymm2[29],zero,zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u],zero,ymm3[30],zero,ymm3[28,u,u,u],zero,ymm3[31],zero,ymm3[29,u]
-; AVX512-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm23
+; AVX512-FCP-NEXT:    vpor %ymm2, %ymm3, %ymm0
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[27],zero,zero,zero,zero,ymm1[30],zero,ymm1[28],zero,zero,zero,zero,ymm1[31],zero,ymm1[29]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm0[27,u,u,u],zero,ymm0[30],zero,ymm0[28,u,u,u],zero,ymm0[31],zero
-; AVX512-FCP-NEXT:    vporq %ymm1, %ymm0, %ymm22
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm2
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,ymm2[14],zero,zero,zero,zero,zero,zero,ymm2[15],zero,zero,zero,zero,zero,zero,ymm2[16],zero,zero,zero,zero,zero,zero,ymm2[17],zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm17, %ymm3
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm3[0,1,0,1,14],zero,ymm3[14,15,0,1,14,15],zero,ymm3[13,14,15,16,17,16],zero,ymm3[30,31,30,31,16,17],zero,ymm3[31,28,29,30,31]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm14[27,u,u,u],zero,ymm14[30],zero,ymm14[28,u,u,u],zero,ymm14[31],zero
+; AVX512-FCP-NEXT:    vporq %ymm1, %ymm0, %ymm21
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,ymm15[14],zero,zero,zero,zero,zero,zero,ymm15[15],zero,zero,zero,zero,zero,zero,ymm15[16],zero,zero,zero,zero,zero,zero,ymm15[17],zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm26, %ymm2
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[0,1,0,1,14],zero,ymm2[14,15,0,1,14,15],zero,ymm2[13,14,15,16,17,16],zero,ymm2[30,31,30,31,16,17],zero,ymm2[31,28,29,30,31]
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm16, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm29, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm15, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm1
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm18, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm27, %ymm3
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero,zero,zero,ymm3[18]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm25, %ymm2
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[0,1,14],zero,ymm2[12,13,0,1,14,15],zero,ymm2[3,12,13,2,3,16],zero,ymm2[30,31,28,29,16,17],zero,ymm2[31,18,19,28,29,18],zero
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm3, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm22, %ymm1
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
-; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm24, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm2, %ymm1
+; AVX512-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm29
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm23, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm18, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm16, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm3, %ymm1
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm2[14],zero,zero,zero,zero,zero,zero,ymm2[15],zero,zero,zero,zero,zero,zero,ymm2[16],zero,zero,zero,zero,zero,zero,ymm2[17],zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm3[13,u,u,u,u,u],zero,ymm3[14,u,u,u,u,u],zero,ymm3[15,u,u,u,u,u],zero,ymm3[16,u,u,u,u,u],zero,ymm3[17,u,u,u]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm9[8],xmm11[8],xmm9[9],xmm11[9],xmm9[10],xmm11[10],xmm9[11],xmm11[11],xmm9[12],xmm11[12],xmm9[13],xmm11[13],xmm9[14],xmm11[14],xmm9[15],xmm11[15]
-; AVX512-FCP-NEXT:    vmovdqa %xmm9, %xmm8
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm11, %xmm17
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [2,2,3,3,2,2,3,3]
-; AVX512-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vmovdqa (%rax), %xmm0
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5,5,6]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm19
-; AVX512-FCP-NEXT:    vpermd %ymm1, %ymm6, %ymm1
-; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm12
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = zero,ymm12[13],zero,zero,zero,zero,zero,zero,ymm12[14],zero,zero,zero,zero,zero,zero,ymm12[15],zero,zero,zero,zero,zero,zero,ymm12[16],zero,zero,zero,zero,zero,zero,ymm12[17],zero,zero
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm20
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm7[8],xmm10[8],xmm7[9],xmm10[9],xmm7[10],xmm10[10],xmm7[11],xmm10[11],xmm7[12],xmm10[12],xmm7[13],xmm10[13],xmm7[14],xmm10[14],xmm7[15],xmm10[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm25
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX512-FCP-NEXT:    vmovdqa (%rax), %xmm1
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [2,2,3,3,2,2,3,3]
+; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5,5,6]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm23
+; AVX512-FCP-NEXT:    vpermd %ymm0, %ymm3, %ymm0
+; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm15
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,ymm15[13],zero,zero,zero,zero,zero,zero,ymm15[14],zero,zero,zero,zero,zero,zero,ymm15[15],zero,zero,zero,zero,zero,zero,ymm15[16],zero,zero,zero,zero,zero,zero,ymm15[17],zero,zero
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm18
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm27
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm16 # 16-byte Reload
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm16, %xmm2
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm11[8],xmm2[8],xmm11[9],xmm2[9],xmm11[10],xmm2[10],xmm11[11],xmm2[11],xmm11[12],xmm2[12],xmm11[13],xmm2[13],xmm11[14],xmm2[14],xmm11[15],xmm2[15]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm16, %xmm1
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm12[8],xmm1[8],xmm12[9],xmm1[9],xmm12[10],xmm1[10],xmm12[11],xmm1[11],xmm12[12],xmm1[12],xmm12[13],xmm1[13],xmm12[14],xmm1[14],xmm12[15],xmm1[15]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm19
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm4
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm17, %xmm1
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm24
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm28, %xmm5
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm13[8],xmm5[8],xmm13[9],xmm5[9],xmm13[10],xmm5[10],xmm13[11],xmm5[11],xmm13[12],xmm5[12],xmm13[13],xmm5[13],xmm13[14],xmm5[14],xmm13[15],xmm5[15]
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm2, %zmm18
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm0
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm21
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm13 = xmm14[8],xmm4[8],xmm14[9],xmm4[9],xmm14[10],xmm4[10],xmm14[11],xmm4[11],xmm14[12],xmm4[12],xmm14[13],xmm4[13],xmm14[14],xmm4[14],xmm14[15],xmm4[15]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm13, %xmm1
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm3, %zmm13
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm30, %xmm5
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm5[8],xmm15[8],xmm5[9],xmm15[9],xmm5[10],xmm15[10],xmm5[11],xmm15[11],xmm5[12],xmm15[12],xmm5[13],xmm15[13],xmm5[14],xmm15[14],xmm5[15],xmm15[15]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm2, %zmm17
+; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %xmm6
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm14 = xmm6[0,1,2,3,4,5,5,6]
+; AVX512-FCP-NEXT:    vpermd %ymm14, %ymm3, %ymm31
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm7[8],xmm8[8],xmm7[9],xmm8[9],xmm7[10],xmm8[10],xmm7[11],xmm8[11],xmm7[12],xmm8[12],xmm7[13],xmm8[13],xmm7[14],xmm8[14],xmm7[15],xmm8[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm3, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm22
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm11, %xmm11
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,0,1],zmm1[0,1,0,1]
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %xmm3
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5,5,6]
-; AVX512-FCP-NEXT:    vpermd %ymm1, %ymm6, %ymm26
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm24 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3],xmm10[4],xmm7[4],xmm10[5],xmm7[5],xmm10[6],xmm7[6],xmm10[7],xmm7[7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm9, %xmm9
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm11, %zmm3, %zmm28
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm11, %xmm11
+; AVX512-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm26 # 16-byte Folded Reload
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3],xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm9, %xmm9
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm9 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm17, %xmm2
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm2[0],xmm8[1],xmm2[1],xmm8[2],xmm2[2],xmm8[3],xmm2[3],xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm8, %xmm8
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,0,1],zmm31[0,1,0,1]
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3],xmm4[4],xmm14[4],xmm4[5],xmm14[5],xmm4[6],xmm14[6],xmm4[7],xmm14[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm14, %xmm1
-; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm14 = mem[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm31 = mem[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vporq %zmm14, %zmm31, %zmm14
-; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm31 = mem[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm30 = mem[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vporq %zmm31, %zmm30, %zmm30
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm30
-; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm14 = mem[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm31 = mem[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vporq %zmm14, %zmm31, %zmm14
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
-; AVX512-FCP-NEXT:    vpternlogq $184, %zmm30, %zmm31, %zmm14
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm18[0,1,0,1,4,5,4,5]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm13 = zmm13[0,1,0,1,4,5,4,5]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm7, %zmm31, %zmm13
-; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm7 = mem[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm29 = zmm29[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vporq %zmm7, %zmm29, %zmm7
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm28 = zmm28[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm27 = zmm27[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vporq %zmm28, %zmm27, %zmm27
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm7, %zmm31, %zmm27
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[2,3,2,3],zmm1[0,1,0,1]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm16, %xmm2
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm2[0],xmm11[0],xmm2[1],xmm11[1],xmm2[2],xmm11[2],xmm2[3],xmm11[3],xmm2[4],xmm11[4],xmm2[5],xmm11[5],xmm2[6],xmm11[6],xmm2[7],xmm11[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm7, %xmm7
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm23[2,3,2,3],zmm7[0,1,0,1]
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm5[0],xmm15[0],xmm5[1],xmm15[1],xmm5[2],xmm15[2],xmm5[3],xmm15[3],xmm5[4],xmm15[4],xmm5[5],xmm15[5],xmm5[6],xmm15[6],xmm5[7],xmm15[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm10, %xmm10
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm22[2,3,2,3],zmm10[0,1,0,1]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm2
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm2[1,1,0,0,4,5,6,7]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [0,1,0,1,2,0,0,1]
-; AVX512-FCP-NEXT:    vpermd %ymm11, %ymm17, %ymm28
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm2, %xmm6
-; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm3[1,1,0,0,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermd %ymm4, %ymm17, %ymm17
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm12[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm23 = [4,5,4,5,5,7,4,5]
-; AVX512-FCP-NEXT:    vpermd %ymm2, %ymm23, %ymm2
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = zero,ymm15[13],zero,zero,zero,zero,zero,zero,ymm15[14],zero,zero,zero,zero,zero,zero,ymm15[15],zero,zero,zero,zero,zero,zero,ymm15[16],zero,zero,zero,zero,zero,zero,ymm15[17],zero,zero
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm15 = ymm15[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512-FCP-NEXT:    vpermd %ymm15, %ymm23, %ymm15
-; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm15, %ymm15
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm11, %zmm11
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm1, %zmm15, %zmm7
-; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    # ymm1 = mem[2,3,2,3]
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3],xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm7, %xmm7
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm30, %zmm7, %zmm7
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm5[0],xmm13[0],xmm5[1],xmm13[1],xmm5[2],xmm13[2],xmm5[3],xmm13[3],xmm5[4],xmm13[4],xmm5[5],xmm13[5],xmm5[6],xmm13[6],xmm5[7],xmm13[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm8, %xmm8
+; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm14 = zmm20[2,3,2,3],zmm8[0,1,0,1]
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm20 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,3,2,3,6,7,6,7]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm20 = zmm20[2,3,2,3,6,7,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm20
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm30 = zmm8[2,3,2,3,6,7,6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
+; AVX512-FCP-NEXT:    vpternlogq $184, %zmm20, %zmm8, %zmm30
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm19[0,1,0,1,4,5,4,5]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm20 = zmm17[0,1,0,1,4,5,4,5]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm5, %zmm8, %zmm20
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm5 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm25 = zmm1[2,3,2,3,6,7,6,7]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm5[2,3,2,3,6,7,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm25, %zmm8, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm16, %xmm4
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3],xmm4[4],xmm12[4],xmm4[5],xmm12[5],xmm4[6],xmm12[6],xmm4[7],xmm12[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm5, %xmm5
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
-; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    # ymm18 = mem[2,3,2,3]
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm3, %zmm18
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm21[0,1,0,1]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm19 = ymm25[0,1,0,1]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,1,0]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,3,2,3]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX512-FCP-NEXT:    vpternlogq $226, %zmm1, %zmm15, %zmm18
-; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm1 = mem[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm1
-; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm14 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm14
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm20
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm20
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm26, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm0
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm24[0,1,0,1,4,5,4,5]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[0,1,0,1,4,5,4,5]
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm9
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm28, %zmm5
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm5
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm5
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT:    vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm6 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm6 = zmm3[2,3,2,3],mem[2,3,2,3]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm2
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm2
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm4
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm4
+; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm10 = zmm3[2,3,2,3],zmm5[0,1,0,1]
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm5, %xmm5
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm14, %zmm8, %zmm10
+; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm21[2,3,2,3],zmm5[0,1,0,1]
+; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    # ymm11 = mem[2,3,2,3]
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm11
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm29[2,3,2,3]
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
+; AVX512-FCP-NEXT:    vpternlogq $226, %zmm11, %zmm8, %zmm14
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm23, %xmm0
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm0[1,1,0,0,4,5,6,7]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [0,1,0,1,2,0,0,1]
+; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm11, %ymm16
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm12
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm6[1,1,0,0,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermd %ymm0, %ymm11, %ymm2
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm15 = ymm15[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [4,5,4,5,5,7,4,5]
+; AVX512-FCP-NEXT:    vpermd %ymm15, %ymm17, %ymm15
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm13[13],zero,zero,zero,zero,zero,zero,ymm13[14],zero,zero,zero,zero,zero,zero,ymm13[15],zero,zero,zero,zero,zero,zero,ymm13[16],zero,zero,zero,zero,zero,zero,ymm13[17],zero,zero
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm13 = ymm13[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512-FCP-NEXT:    vpermd %ymm13, %ymm17, %ymm13
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm24[0,1,0,1]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm17 = ymm27[0,1,0,1]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm22[0,1,0,1]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,1,0]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[0,0,1,0]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,3,2,3]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm8, %zmm8
+; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    # zmm13 = mem[2,3,2,3,6,7,6,7]
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm30, %zmm13
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm17 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm17
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm18
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm17, %zmm18
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm31, %zmm5, %zmm3
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm28[0,1,0,1,4,5,4,5]
+; AVX512-FCP-NEXT:    vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm5
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm26[0,1,0,1,4,5,4,5]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm6 = zmm9[0,1,0,1,4,5,4,5]
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm6
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm16, %zmm3
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm4 = zmm7[0,1,0,1,4,5,4,5]
+; AVX512-FCP-NEXT:    vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm4
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm4
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm6 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm6 = zmm3[0,1,2,3],mem[2,3,2,3]
-; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm11
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm11
+; AVX512-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm15, %zmm6
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[2,3,2,3,6,7,6,7]
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm6
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm6
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm21, %zmm0
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm0
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512-FCP-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    # zmm1 = zmm1[0,1,2,3],mem[2,3,2,3]
+; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm8
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm8
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, 320(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 192(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 128(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 256(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm20, 64(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 384(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, 320(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 192(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 128(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 256(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm18, 64(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, 384(%rax)
 ; AVX512-FCP-NEXT:    addq $1432, %rsp # imm = 0x598
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i8_stride7_vf64:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    subq $1720, %rsp # imm = 0x6B8
+; AVX512DQ-NEXT:    subq $1384, %rsp # imm = 0x568
 ; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm7
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
-; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm7, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm26
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm7[14],zero,zero,zero,zero,zero,zero,ymm7[15],zero,zero,zero,zero,zero,zero,ymm7[16],zero,zero,zero,zero,zero,zero,ymm7[17],zero,zero,zero,zero,zero,zero,ymm7[18]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm2
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm25
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm18
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[0,1,14],zero,ymm2[12,13,0,1,14,15],zero,ymm2[3,12,13,2,3,16],zero,ymm2[30,31,28,29,16,17],zero,ymm2[31,18,19,28,29,18],zero
+; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm20
 ; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm14
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm15
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm14, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm8
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm15, %ymm0
+; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm23
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm2
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm8, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm23
-; AVX512DQ-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm26
+; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm18
 ; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm15
+; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm14
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm15, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm24
-; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm2
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm14, %ymm0
+; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm27
+; AVX512DQ-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm8
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0,13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0]
 ; AVX512DQ-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm27
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm19
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm8, %ymm1
+; AVX512DQ-NEXT:    vmovdqa64 %ymm3, %ymm17
 ; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm10
@@ -9423,7 +9373,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [12,13,14,128,12,128,14,15,14,15,128,13,128,15,12,13,28,29,30,128,28,128,30,31,30,31,128,29,128,31,28,29]
 ; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm9, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm16
+; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm19
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm10[23,u,u,u],zero,ymm10[26],zero,ymm10[24,u,u,u],zero,ymm10[27],zero
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -9438,7 +9388,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0]
 ; AVX512DQ-NEXT:    # ymm0 = mem[0,1,0,1]
 ; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm2
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm20
+; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm21
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm5[23],zero,ymm5[21,22,23,26],zero,ymm5[24],zero,ymm5[28,29,26,27]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -9446,7 +9396,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29]
 ; AVX512DQ-NEXT:    # ymm0 = mem[0,1,0,1]
 ; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm3, %ymm11
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm21
+; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm25
 ; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm2
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm12 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm11
@@ -9465,23 +9415,23 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm7, %ymm6
-; AVX512DQ-NEXT:    vmovdqa64 %ymm7, %ymm17
-; AVX512DQ-NEXT:    vmovdqa64 %ymm18, %ymm13
-; AVX512DQ-NEXT:    vmovdqa64 %ymm16, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm13, %ymm7
+; AVX512DQ-NEXT:    vmovdqa64 %ymm7, %ymm22
+; AVX512DQ-NEXT:    vmovdqa64 %ymm20, %ymm7
+; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm7, %ymm7
 ; AVX512DQ-NEXT:    vpor %ymm6, %ymm7, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm8, %ymm6
-; AVX512DQ-NEXT:    vmovdqa64 %ymm20, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm14, %ymm7
-; AVX512DQ-NEXT:    vmovdqa64 %ymm14, %ymm22
+; AVX512DQ-NEXT:    vmovdqa64 %ymm18, %ymm12
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm12, %ymm6
+; AVX512DQ-NEXT:    vmovdqa64 %ymm21, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm15, %ymm7
+; AVX512DQ-NEXT:    vmovdqa64 %ymm15, %ymm19
 ; AVX512DQ-NEXT:    vpor %ymm6, %ymm7, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 %ymm21, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm15, %ymm6
-; AVX512DQ-NEXT:    vmovdqa64 %ymm15, %ymm16
-; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm15
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm15, %ymm7
+; AVX512DQ-NEXT:    vmovdqa64 %ymm25, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm14, %ymm6
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm8, %ymm7
+; AVX512DQ-NEXT:    vmovdqa64 %ymm8, %ymm16
 ; AVX512DQ-NEXT:    vpor %ymm6, %ymm7, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm0
@@ -9490,379 +9440,348 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm11 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
 ; AVX512DQ-NEXT:    vpshufb %xmm11, %xmm1, %xmm6
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm12 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm0, %xmm7
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm7
+; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm21
 ; AVX512DQ-NEXT:    vpor %xmm6, %xmm7, %xmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm0
-; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm14
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm15
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm14, %xmm7
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm8
-; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm18
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm15, %xmm7
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm13 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm0, %xmm8
+; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm30
 ; AVX512DQ-NEXT:    vpor %xmm7, %xmm8, %xmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,2,3,0,1,0,18,0,19,18,0,19,0]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm28 = [0,0,0,0,2,3,0,1,0,18,0,19,18,0,19,0]
 ; AVX512DQ-NEXT:    vmovdqa 32(%rax), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm0[0,1,2,3,4,5,5,6]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm8 = xmm0[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-NEXT:    vpermi2d %zmm7, %zmm8, %zmm1
-; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpermi2d %zmm7, %zmm8, %zmm28
 ; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm0
-; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm1
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm8 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm0, %xmm7
-; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm20
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm14
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm7
+; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm24
+; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm18
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm30
-; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm29
-; AVX512DQ-NEXT:    vpor %xmm7, %xmm8, %xmm0
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 %ymm28, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm7
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm14, %xmm8
+; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm29
+; AVX512DQ-NEXT:    vporq %xmm7, %xmm8, %xmm31
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm23, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm7
+; AVX512DQ-NEXT:    vmovdqa64 %ymm26, %ymm0
 ; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm5, %ymm8
 ; AVX512DQ-NEXT:    vpor %ymm7, %ymm8, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20]
 ; AVX512DQ-NEXT:    # ymm1 = mem[0,1,0,1]
 ; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm21
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128]
-; AVX512DQ-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm5, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm4, %ymm19
+; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm23
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128]
+; AVX512DQ-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm5, %ymm1
 ; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 %ymm26, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm10, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 %ymm25, %ymm1
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm9, %ymm1
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm10[14],zero,zero,zero,zero,zero,zero,ymm10[15],zero,zero,zero,zero,zero,zero,ymm10[16],zero,zero,zero,zero,zero,zero,ymm10[17],zero,zero,zero,zero,zero,zero,ymm10[18]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm9[0,1,14],zero,ymm9[12,13,0,1,14,15],zero,ymm9[3,12,13,2,3,16],zero,ymm9[30,31,28,29,16,17],zero,ymm9[31,18,19,28,29,18],zero
 ; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
-; AVX512DQ-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm10, %ymm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [2,3,4,5,128,3,128,5,4,5,6,128,4,128,6,7,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm9, %ymm4
-; AVX512DQ-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22]
-; AVX512DQ-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm2, %ymm4
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128]
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
 ; AVX512DQ-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm3, %ymm5
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm10, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm8 = [2,3,4,5,128,3,128,5,4,5,6,128,4,128,6,7,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
+; AVX512DQ-NEXT:    vpshufb %ymm8, %ymm9, %ymm4
+; AVX512DQ-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512DQ-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22]
+; AVX512DQ-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm4
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128]
+; AVX512DQ-NEXT:    # ymm9 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm3, %ymm5
 ; AVX512DQ-NEXT:    vpor %ymm4, %ymm5, %ymm4
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 %ymm24, %ymm4
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm27, %ymm4
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vmovdqa64 %ymm17, %ymm4
 ; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpor %ymm3, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm3
 ; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm31
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm25
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm28
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm4, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm27
 ; AVX512DQ-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm3
 ; AVX512DQ-NEXT:    vpshufb %xmm11, %xmm3, %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm3, %xmm6
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm4, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm18
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm17
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm13
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm13, %xmm3
 ; AVX512DQ-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm3, %xmm8
-; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm3, %xmm9
-; AVX512DQ-NEXT:    vpor %xmm2, %xmm4, %xmm2
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm13[23],zero,zero,zero,zero,ymm13[26],zero,ymm13[24],zero,zero,zero,zero,ymm13[27],zero,ymm13[25]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm17, %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm3, %ymm4
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm4, %zmm2
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm3[23,u,u,u],zero,ymm3[26],zero,ymm3[24,u,u,u],zero,ymm3[27],zero
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm13, %ymm1
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 %ymm22, %ymm2
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa64 %ymm21, %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm2[23],zero,ymm2[21,22,23,26],zero,ymm2[24],zero,ymm2[28,29,26,27]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm30
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm15[25],zero,ymm15[23],zero,zero,zero,zero,ymm15[26],zero,ymm15[24],zero,zero
-; AVX512DQ-NEXT:    vmovdqa64 %ymm16, %ymm3
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm3[23],zero,ymm3[23,24,25,26],zero,ymm3[24],zero,ymm3[30,31]
-; AVX512DQ-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm15, %ymm1
+; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm11
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm2
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
+; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm10, %xmm3
+; AVX512DQ-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovdqa64 %ymm20, %ymm4
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[23],zero,zero,zero,zero,ymm4[26],zero,ymm4[24],zero,zero,zero,zero,ymm4[27],zero,ymm4[25]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm22, %ymm3
 ; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
-; AVX512DQ-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm4
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm4[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512DQ-NEXT:    vpermi2d %zmm0, %zmm1, %zmm15
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm9[8],xmm8[8],xmm9[9],xmm8[9],xmm9[10],xmm8[10],xmm9[11],xmm8[11],xmm9[12],xmm8[12],xmm9[13],xmm8[13],xmm9[14],xmm8[14],xmm9[15],xmm8[15]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm9, %xmm21
-; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm22
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm24
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm1[0,1,0,1],zmm0[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa (%rax), %xmm11
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm11[0,1,2,3,4,5,5,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
-; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm3[23,u,u,u],zero,ymm3[26],zero,ymm3[24,u,u,u],zero,ymm3[27],zero
+; AVX512DQ-NEXT:    vpshufb %ymm8, %ymm4, %ymm2
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm29
+; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm2
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero,zero,zero
+; AVX512DQ-NEXT:    vmovdqa64 %ymm23, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm26
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm12[23],zero,ymm12[21,22,23,26],zero,ymm12[24],zero,ymm12[28,29,26,27]
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm12, %ymm2
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm23
+; AVX512DQ-NEXT:    vmovdqa64 %ymm16, %ymm2
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm1[23],zero,ymm1[23,24,25,26],zero,ymm1[24],zero,ymm1[30,31]
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm8
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm8[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512DQ-NEXT:    vpermi2d %zmm1, %zmm2, %zmm16
+; AVX512DQ-NEXT:    vmovdqa (%rax), %xmm5
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,4,5,5,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
+; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [128,13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm4, %ymm2
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm16
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3],xmm10[4],xmm14[4],xmm10[5],xmm14[5],xmm10[6],xmm14[6],xmm10[7],xmm14[7]
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm8, %ymm3
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm19
+; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm4
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm4[0],xmm15[0],xmm4[1],xmm15[1],xmm4[2],xmm15[2],xmm4[3],xmm15[3],xmm4[4],xmm15[4],xmm4[5],xmm15[5],xmm4[6],xmm15[6],xmm4[7],xmm15[7]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm27
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm24
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm12 = zmm0[2,3,2,3],zmm2[0,1,0,1]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm0[2,3,2,3],zmm3[0,1,0,1]
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm7[0],xmm1[1],xmm7[1],xmm1[2],xmm7[2],xmm1[3],xmm7[3],xmm1[4],xmm7[4],xmm1[5],xmm7[5],xmm1[6],xmm7[6],xmm1[7],xmm7[7]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm26
+; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm21
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm17 = zmm0[2,3,2,3],zmm3[0,1,0,1]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm3
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm0[2,3,2,3],zmm3[0,1,0,1]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm2
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm12 = xmm14[0],xmm2[0],xmm14[1],xmm2[1],xmm14[2],xmm2[2],xmm14[3],xmm2[3],xmm14[4],xmm2[4],xmm14[5],xmm2[5],xmm14[6],xmm2[6],xmm14[7],xmm2[7]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm13, %xmm13
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm25
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm12, %xmm12
+; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm20
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm29 = zmm0[2,3,2,3],zmm13[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm20
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; AVX512DQ-NEXT:    vmovdqa %xmm6, %xmm8
-; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm7
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm30 = zmm0[2,3,2,3],zmm12[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm18
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm3
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm3[8],xmm13[8],xmm3[9],xmm13[9],xmm3[10],xmm13[10],xmm3[11],xmm13[11],xmm3[12],xmm13[12],xmm3[13],xmm13[13],xmm3[14],xmm13[14],xmm3[15],xmm13[15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm18
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm8, %xmm12
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm19
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm14[8],xmm10[8],xmm14[9],xmm10[9],xmm14[10],xmm10[10],xmm14[11],xmm10[11],xmm14[12],xmm10[12],xmm14[13],xmm10[13],xmm14[14],xmm10[14],xmm14[15],xmm10[15]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm6
-; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm4
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm8
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm4[8],xmm15[9],xmm4[9],xmm15[10],xmm4[10],xmm15[11],xmm4[11],xmm15[12],xmm4[12],xmm15[13],xmm4[13],xmm15[14],xmm4[14],xmm15[15],xmm4[15]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm7
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm4[8],xmm7[8],xmm4[9],xmm7[9],xmm4[10],xmm7[10],xmm4[11],xmm7[11],xmm4[12],xmm7[12],xmm4[13],xmm7[13],xmm4[14],xmm7[14],xmm4[15],xmm7[15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm14, %xmm13
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm15, %xmm15
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm14 = zmm2[0,1,0,1],zmm1[0,1,0,1]
-; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm10, %ymm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm10[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm17
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm14[8],xmm2[8],xmm14[9],xmm2[9],xmm14[10],xmm2[10],xmm14[11],xmm2[11],xmm14[12],xmm2[12],xmm14[13],xmm2[13],xmm14[14],xmm2[14],xmm14[15],xmm2[15]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm10[8],xmm11[8],xmm10[9],xmm11[9],xmm10[10],xmm11[10],xmm10[11],xmm11[11],xmm10[12],xmm11[12],xmm10[13],xmm11[13],xmm10[14],xmm11[14],xmm10[15],xmm11[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm14, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm0, %zmm31, %zmm31
+; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm0, %ymm1
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm0[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[0,1,1,3,4,5,5,7]
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm12, %zmm10, %zmm17
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm2 = mem[2,3,2,3]
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm28 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    # ymm28 = mem[2,3,2,3]
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm28, %zmm3, %zmm28
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm2, %zmm10, %zmm28
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm9[2,2,3,2]
-; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm10
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm27 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3],xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm3
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm8, %xmm8
-; AVX512DQ-NEXT:    vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm8 = zmm8[0,1,0,1],mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm9 = xmm11[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm11[1,1,0,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,3,2]
+; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm1, %zmm9
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm6, %zmm1, %zmm22
+; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm6 = mem[2,3,2,3]
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm6
+; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm27 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    # ymm27 = mem[2,3,2,3]
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm27, %zmm0, %zmm27
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm6, %zmm1, %zmm27
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm13[0],xmm3[0],xmm13[1],xmm3[1],xmm13[2],xmm3[2],xmm13[3],xmm3[3],xmm13[4],xmm3[4],xmm13[5],xmm3[5],xmm13[6],xmm3[6],xmm13[7],xmm3[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm6
+; AVX512DQ-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3],xmm10[4],xmm11[4],xmm10[5],xmm11[5],xmm10[6],xmm11[6],xmm10[7],xmm11[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm10, %xmm10
+; AVX512DQ-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm11 = xmm5[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[1,1,0,0,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[0,1,2,0]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm5, %zmm5
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm9 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm11 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vporq %zmm9, %zmm11, %zmm9
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm11 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm12 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vporq %zmm11, %zmm12, %zmm11
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm11
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm9 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm12 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vporq %zmm9, %zmm12, %zmm9
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
-; AVX512DQ-NEXT:    vpternlogq $184, %zmm11, %zmm12, %zmm9
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm13[0,1,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm18[0,1,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm20[2,3,2,3]
-; AVX512DQ-NEXT:    vpshuflw $5, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm1 = mem[1,1,0,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm13 = zmm19[0,1,0,1,4,5,4,5]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,1,0,1,4,5,4,5]
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm13, %zmm12, %zmm0
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm13 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm21 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vporq %zmm13, %zmm21, %zmm13
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm20 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm18 = zmm30[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vporq %zmm20, %zmm18, %zmm18
-; AVX512DQ-NEXT:    vpternlogq $226, %zmm13, %zmm12, %zmm18
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm4
-; AVX512DQ-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm23, %zmm16
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm16
-; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm4 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm4
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,0]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm29, %zmm1
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm17, %zmm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm5
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512DQ-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm13 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm11 = zmm11[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm13 = zmm13[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm13
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm11 = zmm11[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
+; AVX512DQ-NEXT:    vpternlogq $184, %zmm13, %zmm21, %zmm11
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[0,1,0,1,4,5,4,5]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm17[0,1,0,1,4,5,4,5]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm8, %zmm21, %zmm7
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm15[0,1,0,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,0,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm14[0,1,0,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm14 = ymm18[2,3,2,3]
+; AVX512DQ-NEXT:    vpshuflw $5, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm15 = mem[1,1,0,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[0,1,2,0]
+; AVX512DQ-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm20 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vporq %zmm26, %zmm23, %zmm17
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm18 = zmm20[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm17 = zmm17[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpternlogq $226, %zmm18, %zmm21, %zmm17
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm12 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm12
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm8 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm19
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm19
+; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm8 = mem[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm8
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm15[0,0,1,0]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm14, %zmm11
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm30, %zmm11
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm11
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm31[0,1,0,1,4,5,4,5]
+; AVX512DQ-NEXT:    vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm0
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm14
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm14
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[2,3,2,3]
-; AVX512DQ-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm10
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm10
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm27[0,1,0,1,4,5,4,5]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[0,1,0,1,4,5,4,5]
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm5[0,0,1,0,4,4,5,4]
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm0
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT:    vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm2 = zmm2[2,3,2,3],mem[2,3,2,3]
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm15
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm15
+; AVX512DQ-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    # zmm3 = zmm3[0,1,2,3],mem[2,3,2,3]
+; AVX512DQ-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm9
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm9
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm3 = zmm6[0,1,0,1,4,5,4,5]
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm3
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm1 = zmm10[0,1,0,1,4,5,4,5]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[0,0,1,0,4,4,5,4]
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm5
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5
+; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm16
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm17, %zmm16
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vmovdqa64 %zmm15, 128(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm10, 320(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm14, 256(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 192(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 384(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 64(%rax)
-; AVX512DQ-NEXT:    addq $1720, %rsp # imm = 0x6B8
+; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 128(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm9, 320(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 256(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm11, 192(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm8, 384(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm19, 64(%rax)
+; AVX512DQ-NEXT:    addq $1384, %rsp # imm = 0x568
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i8_stride7_vf64:
 ; AVX512DQ-FCP:       # %bb.0:
 ; AVX512DQ-FCP-NEXT:    subq $1432, %rsp # imm = 0x598
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm2[30],zero,ymm2[28],zero,zero,zero,zero,ymm2[31],zero,ymm2[29],zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
-; AVX512DQ-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm3, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm3[30],zero,ymm3[28],zero,zero,zero,zero,ymm3[31],zero,ymm3[29],zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
+; AVX512DQ-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm4, %ymm25
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm3[28],zero,ymm3[30,31,30,31],zero,ymm3[29],zero,ymm3[31,28,29]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm25
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [25,128,23,0,0,0,128,26,128,24,0,0,0,128,27,128,25,128,23,0,0,0,128,26,128,24,0,0,0,128,27,128]
-; AVX512DQ-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm2[28],zero,ymm2[30,31,30,31],zero,ymm2[29],zero,ymm2[31,28,29]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [25,128,23,0,0,0,128,26,128,24,0,0,0,128,27,128,25,128,23,0,0,0,128,26,128,24,0,0,0,128,27,128]
+; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm3, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm16
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm27
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm18
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[30],zero,ymm2[28],zero,zero,zero,zero,ymm2[31],zero,ymm2[29],zero,zero
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm3
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
-; AVX512DQ-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm4, %ymm28
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm1[30],zero,ymm1[28],zero,zero,zero,zero,ymm1[31],zero,ymm1[29],zero,zero
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm15
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
+; AVX512DQ-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm15, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm29
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u],zero,ymm3[30],zero,ymm3[28,u,u,u],zero,ymm3[31],zero,ymm3[29,u]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm19
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u],zero,ymm15[30],zero,ymm15[28,u,u,u],zero,ymm15[31],zero,ymm15[29,u]
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
 ; AVX512DQ-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm24
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm17
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm31
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm26
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm3[27],zero,zero,zero,zero,ymm3[30],zero,ymm3[28],zero,zero,zero,zero,ymm3[31],zero,ymm3[29]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[27],zero,zero,zero,zero,ymm2[30],zero,ymm2[28],zero,zero,zero,zero,ymm2[31],zero,ymm2[29]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
 ; AVX512DQ-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm4, %ymm23
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm4, %ymm21
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm2[27,u,u,u],zero,ymm2[30],zero,ymm2[28,u,u,u],zero,ymm2[31],zero
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
-; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm3, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm22
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm3[27,u,u,u],zero,ymm3[30],zero,ymm3[28,u,u,u],zero,ymm3[31],zero
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm23
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
+; AVX512DQ-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm20
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm19
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
@@ -9872,10 +9791,10 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm5
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm5[14],zero,zero,zero,zero,zero,zero,ymm5[15],zero,zero,zero,zero,zero,zero,ymm5[16],zero,zero,zero,zero,zero,zero,ymm5[17],zero,zero,zero,zero,zero,zero,ymm5[18]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm4
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm4[0,1,14],zero,ymm4[12,13,0,1,14,15],zero,ymm4[3,12,13,2,3,16],zero,ymm4[30,31,28,29,16,17],zero,ymm4[31,18,19,28,29,18],zero
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm6
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm6[14],zero,zero,zero,zero,zero,zero,ymm6[15],zero,zero,zero,zero,zero,zero,ymm6[16],zero,zero,zero,zero,zero,zero,ymm6[17],zero,zero,zero,zero,zero,zero,ymm6[18]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm5
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm5[0,1,14],zero,ymm5[12,13,0,1,14,15],zero,ymm5[3,12,13,2,3,16],zero,ymm5[30,31,28,29,16,17],zero,ymm5[31,18,19,28,29,18],zero
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm3
@@ -9885,327 +9804,316 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,ymm1[14],zero,zero,zero,zero,zero,zero,ymm1[15],zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,ymm1[17],zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm0[13,u,u,u,u,u],zero,ymm0[14,u,u,u,u,u],zero,ymm0[15,u,u,u,u,u],zero,ymm0[16,u,u,u,u,u],zero,ymm0[17,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpor %ymm6, %ymm7, %ymm6
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,ymm1[14],zero,zero,zero,zero,zero,zero,ymm1[15],zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,ymm1[17],zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm14
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm14[13,u,u,u,u,u],zero,ymm14[14,u,u,u,u,u],zero,ymm14[15,u,u,u,u,u],zero,ymm14[16,u,u,u,u,u],zero,ymm14[17,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpor %ymm4, %ymm7, %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm9
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm0, %xmm7
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm9, %xmm9
-; AVX512DQ-FCP-NEXT:    vpor %xmm7, %xmm9, %xmm7
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpor %xmm7, %xmm9, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm9, %xmm9
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm10 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm11, %xmm11
-; AVX512DQ-FCP-NEXT:    vpor %xmm9, %xmm11, %xmm9
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm11
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm28
+; AVX512DQ-FCP-NEXT:    vpor %xmm9, %xmm11, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm9
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm14
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm9, %xmm9
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm9, %xmm9
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm14, %xmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm14, %xmm30
-; AVX512DQ-FCP-NEXT:    vpor %xmm9, %xmm11, %xmm9
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm9
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm9, %xmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm9, %xmm26
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm9
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm9, %xmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm9, %xmm21
-; AVX512DQ-FCP-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm7
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm10, %xmm8
-; AVX512DQ-FCP-NEXT:    vpor %xmm6, %xmm8, %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm11
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm11, %xmm12
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm6
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm13
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, %xmm9
-; AVX512DQ-FCP-NEXT:    vporq %xmm12, %xmm13, %xmm31
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm4, %ymm12
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
-; AVX512DQ-FCP-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm5, %ymm13
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm6
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm27, %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm12
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
-; AVX512DQ-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm4, %ymm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm6, %ymm20
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm29
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm12
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20]
-; AVX512DQ-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm6, %ymm16
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm28
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm24, %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm2, %ymm13
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm0, %xmm12
+; AVX512DQ-FCP-NEXT:    vpor %xmm9, %xmm12, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm0
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm0, %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm24
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm10
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm17
+; AVX512DQ-FCP-NEXT:    vpor %xmm7, %xmm10, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm10
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm9
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm9, %xmm8
+; AVX512DQ-FCP-NEXT:    vpor %xmm4, %xmm8, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm8
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm8, %xmm11
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm7
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm7, %xmm13
+; AVX512DQ-FCP-NEXT:    vporq %xmm11, %xmm13, %xmm30
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm5, %ymm11
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
 ; AVX512DQ-FCP-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm2, %ymm15
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm15, %zmm27
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm23, %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm22, %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm15
-; AVX512DQ-FCP-NEXT:    vpor %ymm13, %ymm15, %ymm6
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22]
-; AVX512DQ-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm15
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm6, %ymm24
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128]
-; AVX512DQ-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm1, %ymm14
-; AVX512DQ-FCP-NEXT:    vpor %ymm15, %ymm14, %ymm6
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm5[30],zero,ymm5[28],zero,zero,zero,zero,ymm5[31],zero,ymm5[29],zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm4[28],zero,ymm4[30,31,30,31],zero,ymm4[29],zero,ymm4[31,28,29]
-; AVX512DQ-FCP-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm6, %ymm13
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm16, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm6, %ymm11
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
+; AVX512DQ-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm5, %ymm13
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm22
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm11
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20]
+; AVX512DQ-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm13
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm29
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm31, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm11
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128]
+; AVX512DQ-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm13
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm21, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm11
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22]
+; AVX512DQ-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm13
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm18
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm1, %ymm11
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128]
+; AVX512DQ-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm1, %ymm13
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm16
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm6[30],zero,ymm6[28],zero,zero,zero,zero,ymm6[31],zero,ymm6[29],zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm5[28],zero,ymm5[30,31,30,31],zero,ymm5[29],zero,ymm5[31,28,29]
+; AVX512DQ-FCP-NEXT:    vporq %ymm6, %ymm5, %ymm20
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[30],zero,ymm2[28],zero,zero,zero,zero,ymm2[31],zero,ymm2[29],zero,zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u],zero,ymm3[30],zero,ymm3[28,u,u,u],zero,ymm3[31],zero,ymm3[29,u]
-; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm23
+; AVX512DQ-FCP-NEXT:    vpor %ymm2, %ymm3, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[27],zero,zero,zero,zero,ymm1[30],zero,ymm1[28],zero,zero,zero,zero,ymm1[31],zero,ymm1[29]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm0[27,u,u,u],zero,ymm0[30],zero,ymm0[28,u,u,u],zero,ymm0[31],zero
-; AVX512DQ-FCP-NEXT:    vporq %ymm1, %ymm0, %ymm22
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm19, %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,ymm2[14],zero,zero,zero,zero,zero,zero,ymm2[15],zero,zero,zero,zero,zero,zero,ymm2[16],zero,zero,zero,zero,zero,zero,ymm2[17],zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm17, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm3[0,1,0,1,14],zero,ymm3[14,15,0,1,14,15],zero,ymm3[13,14,15,16,17,16],zero,ymm3[30,31,30,31,16,17],zero,ymm3[31,28,29,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm14[27,u,u,u],zero,ymm14[30],zero,ymm14[28,u,u,u],zero,ymm14[31],zero
+; AVX512DQ-FCP-NEXT:    vporq %ymm1, %ymm0, %ymm21
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,ymm15[14],zero,zero,zero,zero,zero,zero,ymm15[15],zero,zero,zero,zero,zero,zero,ymm15[16],zero,zero,zero,zero,zero,zero,ymm15[17],zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm26, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[0,1,0,1,14],zero,ymm2[14,15,0,1,14,15],zero,ymm2[13,14,15,16,17,16],zero,ymm2[30,31,30,31,16,17],zero,ymm2[31,28,29,30,31]
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm16, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm15, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm18, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm27, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero,zero,zero,ymm3[18]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[0,1,14],zero,ymm2[12,13,0,1,14,15],zero,ymm2[3,12,13,2,3,16],zero,ymm2[30,31,28,29,16,17],zero,ymm2[31,18,19,28,29,18],zero
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm3, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm22, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
-; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm24, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm29
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm23, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm18, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm19, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm16, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm3, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm2[14],zero,zero,zero,zero,zero,zero,ymm2[15],zero,zero,zero,zero,zero,zero,ymm2[16],zero,zero,zero,zero,zero,zero,ymm2[17],zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm3[13,u,u,u,u,u],zero,ymm3[14,u,u,u,u,u],zero,ymm3[15,u,u,u,u,u],zero,ymm3[16,u,u,u,u,u],zero,ymm3[17,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm9[8],xmm11[8],xmm9[9],xmm11[9],xmm9[10],xmm11[10],xmm9[11],xmm11[11],xmm9[12],xmm11[12],xmm9[13],xmm11[13],xmm9[14],xmm11[14],xmm9[15],xmm11[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm9, %xmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm11, %xmm17
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [2,2,3,3,2,2,3,3]
-; AVX512DQ-FCP-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5,5,6]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm19
-; AVX512DQ-FCP-NEXT:    vpermd %ymm1, %ymm6, %ymm1
-; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm12
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = zero,ymm12[13],zero,zero,zero,zero,zero,zero,ymm12[14],zero,zero,zero,zero,zero,zero,ymm12[15],zero,zero,zero,zero,zero,zero,ymm12[16],zero,zero,zero,zero,zero,zero,ymm12[17],zero,zero
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm20
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm7[8],xmm10[8],xmm7[9],xmm10[9],xmm7[10],xmm10[10],xmm7[11],xmm10[11],xmm7[12],xmm10[12],xmm7[13],xmm10[13],xmm7[14],xmm10[14],xmm7[15],xmm10[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm25
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %xmm1
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [2,2,3,3,2,2,3,3]
+; AVX512DQ-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5,5,6]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm23
+; AVX512DQ-FCP-NEXT:    vpermd %ymm0, %ymm3, %ymm0
+; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm15
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,ymm15[13],zero,zero,zero,zero,zero,zero,ymm15[14],zero,zero,zero,zero,zero,zero,ymm15[15],zero,zero,zero,zero,zero,zero,ymm15[16],zero,zero,zero,zero,zero,zero,ymm15[17],zero,zero
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm18
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm27
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm16 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm16, %xmm2
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm11[8],xmm2[8],xmm11[9],xmm2[9],xmm11[10],xmm2[10],xmm11[11],xmm2[11],xmm11[12],xmm2[12],xmm11[13],xmm2[13],xmm11[14],xmm2[14],xmm11[15],xmm2[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm16, %xmm1
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm12[8],xmm1[8],xmm12[9],xmm1[9],xmm12[10],xmm1[10],xmm12[11],xmm1[11],xmm12[12],xmm1[12],xmm12[13],xmm1[13],xmm12[14],xmm1[14],xmm12[15],xmm1[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm19
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm17, %xmm1
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm24
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm5
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm13[8],xmm5[8],xmm13[9],xmm5[9],xmm13[10],xmm5[10],xmm13[11],xmm5[11],xmm13[12],xmm5[12],xmm13[13],xmm5[13],xmm13[14],xmm5[14],xmm13[15],xmm5[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm2, %zmm18
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm0
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm21
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm13 = xmm14[8],xmm4[8],xmm14[9],xmm4[9],xmm14[10],xmm4[10],xmm14[11],xmm4[11],xmm14[12],xmm4[12],xmm14[13],xmm4[13],xmm14[14],xmm4[14],xmm14[15],xmm4[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm13, %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm3, %zmm13
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm30, %xmm5
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm5[8],xmm15[8],xmm5[9],xmm15[9],xmm5[10],xmm15[10],xmm5[11],xmm15[11],xmm5[12],xmm15[12],xmm5[13],xmm15[13],xmm5[14],xmm15[14],xmm5[15],xmm15[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm2, %zmm17
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %xmm6
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm14 = xmm6[0,1,2,3,4,5,5,6]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm14, %ymm3, %ymm31
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm7[8],xmm8[8],xmm7[9],xmm8[9],xmm7[10],xmm8[10],xmm7[11],xmm8[11],xmm7[12],xmm8[12],xmm7[13],xmm8[13],xmm7[14],xmm8[14],xmm7[15],xmm8[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm3, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm22
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm11, %xmm11
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,0,1],zmm1[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %xmm3
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5,5,6]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm1, %ymm6, %ymm26
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm24 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3],xmm10[4],xmm7[4],xmm10[5],xmm7[5],xmm10[6],xmm7[6],xmm10[7],xmm7[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm9, %xmm9
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm11, %zmm3, %zmm28
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm11, %xmm11
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm26 # 16-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3],xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm9, %xmm9
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm9 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm17, %xmm2
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm2[0],xmm8[1],xmm2[1],xmm8[2],xmm2[2],xmm8[3],xmm2[3],xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm8, %xmm8
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,0,1],zmm31[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3],xmm4[4],xmm14[4],xmm4[5],xmm14[5],xmm4[6],xmm14[6],xmm4[7],xmm14[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm14, %xmm1
-; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm14 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm31 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vporq %zmm14, %zmm31, %zmm14
-; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm31 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm30 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vporq %zmm31, %zmm30, %zmm30
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm30
-; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm14 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm31 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vporq %zmm14, %zmm31, %zmm14
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
-; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm30, %zmm31, %zmm14
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm18[0,1,0,1,4,5,4,5]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm13 = zmm13[0,1,0,1,4,5,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm7, %zmm31, %zmm13
-; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm7 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm29 = zmm29[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vporq %zmm7, %zmm29, %zmm7
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm28 = zmm28[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm27 = zmm27[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vporq %zmm28, %zmm27, %zmm27
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm7, %zmm31, %zmm27
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[2,3,2,3],zmm1[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm16, %xmm2
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm2[0],xmm11[0],xmm2[1],xmm11[1],xmm2[2],xmm11[2],xmm2[3],xmm11[3],xmm2[4],xmm11[4],xmm2[5],xmm11[5],xmm2[6],xmm11[6],xmm2[7],xmm11[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm7, %xmm7
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm23[2,3,2,3],zmm7[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm5[0],xmm15[0],xmm5[1],xmm15[1],xmm5[2],xmm15[2],xmm5[3],xmm15[3],xmm5[4],xmm15[4],xmm5[5],xmm15[5],xmm5[6],xmm15[6],xmm5[7],xmm15[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm10, %xmm10
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm22[2,3,2,3],zmm10[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm19, %xmm2
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm2[1,1,0,0,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [0,1,0,1,2,0,0,1]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm11, %ymm17, %ymm28
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm2, %xmm6
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm3[1,1,0,0,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm4, %ymm17, %ymm17
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm12[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm23 = [4,5,4,5,5,7,4,5]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm2, %ymm23, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = zero,ymm15[13],zero,zero,zero,zero,zero,zero,ymm15[14],zero,zero,zero,zero,zero,zero,ymm15[15],zero,zero,zero,zero,zero,zero,ymm15[16],zero,zero,zero,zero,zero,zero,ymm15[17],zero,zero
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm15 = ymm15[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm15, %ymm23, %ymm15
-; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm15, %ymm15
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm11, %zmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm1, %zmm15, %zmm7
-; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # ymm1 = mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3],xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm7, %xmm7
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm30, %zmm7, %zmm7
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm5[0],xmm13[0],xmm5[1],xmm13[1],xmm5[2],xmm13[2],xmm5[3],xmm13[3],xmm5[4],xmm13[4],xmm5[5],xmm13[5],xmm5[6],xmm13[6],xmm5[7],xmm13[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm8, %xmm8
+; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm14 = zmm20[2,3,2,3],zmm8[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm20 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,3,2,3,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm20 = zmm20[2,3,2,3,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm20
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm30 = zmm8[2,3,2,3,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, %zmm20, %zmm8, %zmm30
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm19[0,1,0,1,4,5,4,5]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm20 = zmm17[0,1,0,1,4,5,4,5]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm5, %zmm8, %zmm20
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm5 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm25 = zmm1[2,3,2,3,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm5[2,3,2,3,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm25, %zmm8, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm16, %xmm4
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3],xmm4[4],xmm12[4],xmm4[5],xmm12[5],xmm4[6],xmm12[6],xmm4[7],xmm12[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm5, %xmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
-; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # ymm18 = mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm3, %zmm18
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm21[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm19 = ymm25[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,1,0]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,3,2,3]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm1, %zmm15, %zmm18
-; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm1 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm1
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm14 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm14
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm20
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm20
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm26, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm24[0,1,0,1,4,5,4,5]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[0,1,0,1,4,5,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm9
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm28, %zmm5
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm5
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm6 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm6 = zmm3[2,3,2,3],mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm2
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm4
+; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm10 = zmm3[2,3,2,3],zmm5[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm5, %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm14, %zmm8, %zmm10
+; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm21[2,3,2,3],zmm5[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # ymm11 = mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm11
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm29[2,3,2,3]
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %zmm11, %zmm8, %zmm14
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm23, %xmm0
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm0[1,1,0,0,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [0,1,0,1,2,0,0,1]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm11, %ymm16
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm12
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm6[1,1,0,0,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm0, %ymm11, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm15 = ymm15[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm17 = [4,5,4,5,5,7,4,5]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm15, %ymm17, %ymm15
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm13[13],zero,zero,zero,zero,zero,zero,ymm13[14],zero,zero,zero,zero,zero,zero,ymm13[15],zero,zero,zero,zero,zero,zero,ymm13[16],zero,zero,zero,zero,zero,zero,ymm13[17],zero,zero
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm13 = ymm13[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm13, %ymm17, %ymm13
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm24[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm17 = ymm27[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm22[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,1,0]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[0,0,1,0]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,3,2,3]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm8, %zmm8
+; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # zmm13 = mem[2,3,2,3,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm30, %zmm13
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm17 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm17
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm18
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm17, %zmm18
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm31, %zmm5, %zmm3
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm28[0,1,0,1,4,5,4,5]
+; AVX512DQ-FCP-NEXT:    vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm5
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm26[0,1,0,1,4,5,4,5]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm6 = zmm9[0,1,0,1,4,5,4,5]
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm6
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm16, %zmm3
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm4 = zmm7[0,1,0,1,4,5,4,5]
+; AVX512DQ-FCP-NEXT:    vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm4
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm4
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm6 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm6 = zmm3[0,1,2,3],mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm11
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm11
+; AVX512DQ-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm15, %zmm6
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[2,3,2,3,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm6
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm6
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm21, %zmm0
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    # zmm1 = zmm1[0,1,2,3],mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm8
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm8
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, 320(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 192(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 256(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm20, 64(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 384(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm8, 320(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 192(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 128(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 256(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm18, 64(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, 384(%rax)
 ; AVX512DQ-FCP-NEXT:    addq $1432, %rsp # imm = 0x598
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
index 1771b53a0f835..f5a6b9f59aacf 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
@@ -2104,16 +2104,14 @@ define void @store_i8_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    kmovd %ecx, %k3
 ; AVX512BW-NEXT:    vmovdqa32 %zmm4, %zmm5 {%k3}
 ; AVX512BW-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[1,3,1,3,5,7,5,7]
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,2,10,2,10,u,u,u,u,3,11,3,11,u,u,u,u,4,12,4,12,u,u,u,u,5,13,5,13,u,u,u,u,6,14,6,14,u,u,u,u,7,15,7,15]
-; AVX512BW-NEXT:    vpshufb %zmm4, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u]
 ; AVX512BW-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[1,3,1,3,5,7,5,7]
-; AVX512BW-NEXT:    vpshufb %zmm4, %zmm3, %zmm3
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63]
 ; AVX512BW-NEXT:    vmovdqu16 %zmm3, %zmm2 {%k1}
 ; AVX512BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[1,3,1,3,5,7,5,7]
-; AVX512BW-NEXT:    vpmovsxdq {{.*#+}} zmm3 = [134219776,151062785,167905794,184748803,201591812,218434821,235277830,252120839]
-; AVX512BW-NEXT:    vpshufb %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[1,3,1,3,5,7,5,7]
-; AVX512BW-NEXT:    vpshufb %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u,u,u]
 ; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k2}
 ; AVX512BW-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k3}
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
@@ -2159,16 +2157,14 @@ define void @store_i8_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [5,7,5,7,5,7,5,7]
 ; AVX512BW-FCP-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-FCP-NEXT:    vpermq %zmm5, %zmm1, %zmm5
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,2,10,2,10,u,u,u,u,3,11,3,11,u,u,u,u,4,12,4,12,u,u,u,u,5,13,5,13,u,u,u,u,6,14,6,14,u,u,u,u,7,15,7,15]
-; AVX512BW-FCP-NEXT:    vpshufb %zmm6, %zmm5, %zmm5
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm5 = zmm5[u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63]
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[1,3,1,3,5,7,5,7]
-; AVX512BW-FCP-NEXT:    vpshufb %zmm6, %zmm2, %zmm2
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u]
 ; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm5, %zmm2 {%k1}
 ; AVX512BW-FCP-NEXT:    vpermq %zmm4, %zmm1, %zmm1
-; AVX512BW-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm4 = [134219776,151062785,167905794,184748803,201591812,218434821,235277830,252120839]
-; AVX512BW-FCP-NEXT:    vpshufb %zmm4, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[1,3,1,3,5,7,5,7]
-; AVX512BW-FCP-NEXT:    vpshufb %zmm4, %zmm0, %zmm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k2}
 ; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k3}
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
@@ -2213,16 +2209,14 @@ define void @store_i8_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-NEXT:    kmovd %ecx, %k3
 ; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm4, %zmm5 {%k3}
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[1,3,1,3,5,7,5,7]
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,2,10,2,10,u,u,u,u,3,11,3,11,u,u,u,u,4,12,4,12,u,u,u,u,5,13,5,13,u,u,u,u,6,14,6,14,u,u,u,u,7,15,7,15]
-; AVX512DQ-BW-NEXT:    vpshufb %zmm4, %zmm2, %zmm2
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u]
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[1,3,1,3,5,7,5,7]
-; AVX512DQ-BW-NEXT:    vpshufb %zmm4, %zmm3, %zmm3
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63]
 ; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm3, %zmm2 {%k1}
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[1,3,1,3,5,7,5,7]
-; AVX512DQ-BW-NEXT:    vpmovsxdq {{.*#+}} zmm3 = [134219776,151062785,167905794,184748803,201591812,218434821,235277830,252120839]
-; AVX512DQ-BW-NEXT:    vpshufb %zmm3, %zmm0, %zmm0
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u,u,u,u,u]
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[1,3,1,3,5,7,5,7]
-; AVX512DQ-BW-NEXT:    vpshufb %zmm3, %zmm1, %zmm1
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u,u,u]
 ; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k2}
 ; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k3}
 ; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
@@ -2268,16 +2262,14 @@ define void @store_i8_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [5,7,5,7,5,7,5,7]
 ; AVX512DQ-BW-FCP-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm5, %zmm1, %zmm5
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,2,10,2,10,u,u,u,u,3,11,3,11,u,u,u,u,4,12,4,12,u,u,u,u,5,13,5,13,u,u,u,u,6,14,6,14,u,u,u,u,7,15,7,15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm6, %zmm5, %zmm5
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm5 = zmm5[u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63]
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[1,3,1,3,5,7,5,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm6, %zmm2, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm5, %zmm2 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm4, %zmm1, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm4 = [134219776,151062785,167905794,184748803,201591812,218434821,235277830,252120839]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm4, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[1,3,1,3,5,7,5,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm4, %zmm0, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[0,8,0,8,u,u,u,u,1,9,1,9,u,u,u,u,18,26,18,26,u,u,u,u,19,27,19,27,u,u,u,u,36,44,u,u,u,u,u,u,37,45,u,u,u,u,u,u,54,62,u,u,u,u,u,u,55,63,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k3}
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
@@ -8620,230 +8612,336 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512BW-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512BW-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vmovdqa 16(%r10), %xmm13
-; AVX512BW-NEXT:    vmovdqa64 32(%r10), %xmm22
-; AVX512BW-NEXT:    vmovdqa64 48(%r10), %xmm19
-; AVX512BW-NEXT:    vmovdqa (%rax), %xmm0
-; AVX512BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 16(%rax), %xmm16
-; AVX512BW-NEXT:    vmovdqa64 32(%rax), %xmm23
-; AVX512BW-NEXT:    vmovdqa64 48(%rax), %xmm20
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqa (%r9), %xmm5
-; AVX512BW-NEXT:    vmovdqa64 16(%r9), %xmm17
-; AVX512BW-NEXT:    vmovdqa64 32(%r9), %xmm25
-; AVX512BW-NEXT:    vmovdqa64 48(%r9), %xmm21
-; AVX512BW-NEXT:    vmovdqa (%r8), %xmm7
-; AVX512BW-NEXT:    vmovdqa64 16(%r8), %xmm18
-; AVX512BW-NEXT:    vmovdqa64 32(%r8), %xmm26
-; AVX512BW-NEXT:    vmovdqa64 48(%r8), %xmm24
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm7[8],xmm5[8],xmm7[9],xmm5[9],xmm7[10],xmm5[10],xmm7[11],xmm5[11],xmm7[12],xmm5[12],xmm7[13],xmm5[13],xmm7[14],xmm5[14],xmm7[15],xmm5[15]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,1,0,1,4,5,1,3,2,1,2,1,4,5,3,3,16,17,20,21,20,21,21,23,16,17,22,21,22,21,23,23]
-; AVX512BW-NEXT:    vpermw %zmm1, %zmm10, %zmm4
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,4,5,2,1,0,2,0,2,4,5,2,3,16,17,20,20,20,20,22,21,16,17,20,22,20,22,22,23]
+; AVX512BW-NEXT:    vmovdqa (%r10), %xmm0
+; AVX512BW-NEXT:    vmovdqa 16(%r10), %xmm12
+; AVX512BW-NEXT:    vmovdqa64 32(%r10), %xmm16
+; AVX512BW-NEXT:    vmovdqa 48(%r10), %xmm15
+; AVX512BW-NEXT:    vmovdqa (%rax), %xmm2
+; AVX512BW-NEXT:    vmovdqa 16(%rax), %xmm13
+; AVX512BW-NEXT:    vmovdqa64 32(%rax), %xmm17
+; AVX512BW-NEXT:    vmovdqa64 48(%rax), %xmm18
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm1[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [0,0,0,0,4,5,2,1,0,2,0,2,4,5,2,3]
+; AVX512BW-NEXT:    vpermw %ymm1, %ymm3, %ymm1
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm1
+; AVX512BW-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512BW-NEXT:    vmovdqa64 48(%r9), %xmm19
+; AVX512BW-NEXT:    vmovdqa (%r8), %xmm5
+; AVX512BW-NEXT:    vmovdqa64 48(%r8), %xmm21
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm7[0,1,2,3,4,5,5,7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm7[0,1,2,3,6,5,7,7]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm8, %ymm6, %ymm6
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm8 = ymm6[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm6 = [0,1,0,1,4,5,1,3,2,1,2,1,4,5,3,3]
+; AVX512BW-NEXT:    vpermw %ymm7, %ymm6, %ymm7
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm7, %zmm14
 ; AVX512BW-NEXT:    movl $-2004318072, %eax # imm = 0x88888888
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm9, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm0
-; AVX512BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 48(%rsi), %xmm28
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm6
-; AVX512BW-NEXT:    vmovdqa64 48(%rdi), %xmm30
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm14 {%k1}
+; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm7
+; AVX512BW-NEXT:    vmovdqa64 48(%rsi), %xmm24
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm8
+; AVX512BW-NEXT:    vmovdqa64 48(%rdi), %xmm27
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm10 = xmm1[3,3,3,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm9, %ymm9
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm10 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[3,3,3,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti128 $1, %xmm8, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm8 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm8, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa (%rcx), %xmm8
-; AVX512BW-NEXT:    vmovdqa 48(%rcx), %xmm1
-; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm11
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm11[8],xmm8[8],xmm11[9],xmm8[9],xmm11[10],xmm8[10],xmm11[11],xmm8[11],xmm11[12],xmm8[12],xmm11[13],xmm8[13],xmm11[14],xmm8[14],xmm11[15],xmm8[15]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm12, %zmm12, %zmm14
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,2,1,2,1,6,7,0,2,2,3,2,3,6,7,20,20,18,19,22,21,22,21,20,22,18,19,22,23,22,23]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm10, %ymm1
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm9, %zmm1, %zmm1
+; AVX512BW-NEXT:    vmovdqa (%rcx), %xmm9
+; AVX512BW-NEXT:    vmovdqa64 48(%rcx), %xmm28
+; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm10
+; AVX512BW-NEXT:    vmovdqa64 48(%rdx), %xmm29
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm11 = xmm20[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm22 = xmm20[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm22, %ymm11, %ymm11
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm22 = ymm11[2,1,3,3,6,5,7,7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,0,2,1,2,1,6,7,0,2,2,3,2,3,6,7]
+; AVX512BW-NEXT:    vpermw %ymm20, %ymm11, %ymm20
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm22, %zmm20, %zmm20
 ; AVX512BW-NEXT:    movl $572662306, %eax # imm = 0x22222222
 ; AVX512BW-NEXT:    kmovd %eax, %k2
-; AVX512BW-NEXT:    vpermw %zmm14, %zmm12, %zmm3 {%k2}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm20[0],xmm19[0],xmm20[1],xmm19[1],xmm20[2],xmm19[2],xmm20[3],xmm19[3],xmm20[4],xmm19[4],xmm20[5],xmm19[5],xmm20[6],xmm19[6],xmm20[7],xmm19[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm14, %zmm14, %zmm15
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm24[0],xmm21[0],xmm24[1],xmm21[1],xmm24[2],xmm21[2],xmm24[3],xmm21[3],xmm24[4],xmm21[4],xmm24[5],xmm21[5],xmm24[6],xmm21[6],xmm24[7],xmm21[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm14, %zmm14, %zmm14
-; AVX512BW-NEXT:    vpermw %zmm14, %zmm10, %zmm14
-; AVX512BW-NEXT:    vpermw %zmm15, %zmm9, %zmm14 {%k1}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm30[0],xmm28[0],xmm30[1],xmm28[1],xmm30[2],xmm28[2],xmm30[3],xmm28[3],xmm30[4],xmm28[4],xmm30[5],xmm28[5],xmm30[6],xmm28[6],xmm30[7],xmm28[7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm27 = xmm15[2,3,2,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm27 = xmm27[0],zero,zero,zero,xmm27[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm29 = xmm15[3,3,3,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm29 = xmm29[0],zero,zero,zero,xmm29[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm29, %ymm27, %ymm27
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm29 = xmm15[0],zero,zero,zero,xmm15[1],zero,zero,zero
+; AVX512BW-NEXT:    vmovdqu16 %zmm20, %zmm1 {%k2}
+; AVX512BW-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; AVX512BW-NEXT:    kmovd %eax, %k3
+; AVX512BW-NEXT:    vmovdqa32 %zmm14, %zmm1 {%k3}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm18[0],xmm15[0],xmm18[1],xmm15[1],xmm18[2],xmm15[2],xmm18[3],xmm15[3],xmm18[4],xmm15[4],xmm18[5],xmm15[5],xmm18[6],xmm15[6],xmm18[7],xmm15[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm14[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm22 = xmm14[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm22, %ymm20, %ymm20
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm20 = ymm20[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm14, %ymm3, %ymm14
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm14, %zmm14
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm21[0],xmm19[0],xmm21[1],xmm19[1],xmm21[2],xmm19[2],xmm21[3],xmm19[3],xmm21[4],xmm19[4],xmm21[5],xmm19[5],xmm21[6],xmm19[6],xmm21[7],xmm19[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm22 = xmm20[0,1,2,3,4,5,5,7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm20[0,1,2,3,6,5,7,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm22, %ymm22
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm22 = ymm22[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm20, %ymm6, %ymm20
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm22, %zmm20, %zmm23
+; AVX512BW-NEXT:    vmovdqu16 %zmm14, %zmm23 {%k1}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm27[0],xmm24[0],xmm27[1],xmm24[1],xmm27[2],xmm24[2],xmm27[3],xmm24[3],xmm27[4],xmm24[4],xmm27[5],xmm24[5],xmm27[6],xmm24[6],xmm27[7],xmm24[7]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm20 = xmm14[2,3,2,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm20 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm22 = xmm14[3,3,3,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm22 = xmm22[0],zero,zero,zero,xmm22[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm22, %ymm20, %ymm20
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm22 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm14 = xmm14[1,1,1,1]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm14 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm14, %ymm22, %ymm14
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm14, %zmm14
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm29[0],xmm28[0],xmm29[1],xmm28[1],xmm29[2],xmm28[2],xmm29[3],xmm28[3],xmm29[4],xmm28[4],xmm29[5],xmm28[5],xmm29[6],xmm28[6],xmm29[7],xmm28[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm22 = xmm20[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm25 = xmm20[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm25, %ymm22, %ymm25
+; AVX512BW-NEXT:    vmovdqa64 32(%r9), %xmm22
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm25 = ymm25[2,1,3,3,6,5,7,7]
+; AVX512BW-NEXT:    vpermw %ymm20, %ymm11, %ymm20
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm25, %zmm20, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 32(%r8), %xmm25
+; AVX512BW-NEXT:    vmovdqu16 %zmm20, %zmm14 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 32(%rsi), %xmm20
+; AVX512BW-NEXT:    vmovdqa32 %zmm23, %zmm14 {%k3}
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm18[8],xmm15[8],xmm18[9],xmm15[9],xmm18[10],xmm15[10],xmm18[11],xmm15[11],xmm18[12],xmm15[12],xmm18[13],xmm15[13],xmm18[14],xmm15[14],xmm18[15],xmm15[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm18 = xmm15[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm15[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm18, %ymm18
+; AVX512BW-NEXT:    vmovdqa64 32(%rdi), %xmm23
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm18 = ymm18[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm15, %ymm3, %ymm15
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm18, %zmm15, %zmm15
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm21[8],xmm19[8],xmm21[9],xmm19[9],xmm21[10],xmm19[10],xmm21[11],xmm19[11],xmm21[12],xmm19[12],xmm21[13],xmm19[13],xmm21[14],xmm19[14],xmm21[15],xmm19[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm19 = xmm18[0,1,2,3,4,5,5,7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm21 = xmm18[0,1,2,3,6,5,7,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm21, %ymm19, %ymm21
+; AVX512BW-NEXT:    vmovdqa64 32(%rcx), %xmm19
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm21 = ymm21[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm18, %ymm6, %ymm18
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm18, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 32(%rdx), %xmm26
+; AVX512BW-NEXT:    vmovdqu16 %zmm15, %zmm18 {%k1}
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm27[8],xmm24[8],xmm27[9],xmm24[9],xmm27[10],xmm24[10],xmm27[11],xmm24[11],xmm27[12],xmm24[12],xmm27[13],xmm24[13],xmm27[14],xmm24[14],xmm27[15],xmm24[15]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm21 = xmm15[2,3,2,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm21 = xmm21[0],zero,zero,zero,xmm21[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm24 = xmm15[3,3,3,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm24[0],zero,zero,zero,xmm24[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm21, %ymm21
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm15[0],zero,zero,zero,xmm15[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[1,1,1,1]
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm15 = xmm15[0],zero,zero,zero,xmm15[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm15, %ymm29, %ymm15
-; AVX512BW-NEXT:    vmovdqa 48(%rdx), %xmm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm27, %zmm15, %zmm15
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm27 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm27, %zmm27, %zmm27
-; AVX512BW-NEXT:    vpermw %zmm27, %zmm12, %zmm15 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 32(%rsi), %xmm29
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm20[8],xmm19[8],xmm20[9],xmm19[9],xmm20[10],xmm19[10],xmm20[11],xmm19[11],xmm20[12],xmm19[12],xmm20[13],xmm19[13],xmm20[14],xmm19[14],xmm20[15],xmm19[15]
-; AVX512BW-NEXT:    vmovdqa64 32(%rdi), %xmm31
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm24[8],xmm21[8],xmm24[9],xmm21[9],xmm24[10],xmm21[10],xmm24[11],xmm21[11],xmm24[12],xmm21[12],xmm24[13],xmm21[13],xmm24[14],xmm21[14],xmm24[15],xmm21[15]
-; AVX512BW-NEXT:    vmovdqa64 32(%rcx), %xmm27
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm19, %zmm19, %zmm21
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm20, %zmm20, %zmm19
-; AVX512BW-NEXT:    vpermw %zmm19, %zmm10, %zmm19
-; AVX512BW-NEXT:    vpermw %zmm21, %zmm9, %zmm19 {%k1}
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm30[8],xmm28[8],xmm30[9],xmm28[9],xmm30[10],xmm28[10],xmm30[11],xmm28[11],xmm30[12],xmm28[12],xmm30[13],xmm28[13],xmm30[14],xmm28[14],xmm30[15],xmm28[15]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm21 = xmm20[2,3,2,3]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm15, %ymm24, %ymm15
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm15, %zmm15
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm21 = xmm29[8],xmm28[8],xmm29[9],xmm28[9],xmm29[10],xmm28[10],xmm29[11],xmm28[11],xmm29[12],xmm28[12],xmm29[13],xmm28[13],xmm29[14],xmm28[14],xmm29[15],xmm28[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm24 = xmm21[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm27 = xmm21[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm24, %ymm24
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm24 = ymm24[2,1,3,3,6,5,7,7]
+; AVX512BW-NEXT:    vpermw %ymm21, %ymm11, %ymm21
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm24, %zmm21, %zmm21
+; AVX512BW-NEXT:    vmovdqu16 %zmm21, %zmm15 {%k2}
+; AVX512BW-NEXT:    vmovdqa32 %zmm18, %zmm15 {%k3}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm18 = xmm17[0],xmm16[0],xmm17[1],xmm16[1],xmm17[2],xmm16[2],xmm17[3],xmm16[3],xmm17[4],xmm16[4],xmm17[5],xmm16[5],xmm17[6],xmm16[6],xmm17[7],xmm16[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm21 = xmm18[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm24 = xmm18[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm21, %ymm21
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm21 = ymm21[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm18, %ymm3, %ymm18
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm18, %zmm18
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm21 = xmm25[0],xmm22[0],xmm25[1],xmm22[1],xmm25[2],xmm22[2],xmm25[3],xmm22[3],xmm25[4],xmm22[4],xmm25[5],xmm22[5],xmm25[6],xmm22[6],xmm25[7],xmm22[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm24 = xmm21[0,1,2,3,4,5,5,7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm27 = xmm21[0,1,2,3,6,5,7,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm24, %ymm24
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm24 = ymm24[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm21, %ymm6, %ymm21
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm24, %zmm21, %zmm27
+; AVX512BW-NEXT:    vmovdqu16 %zmm18, %zmm27 {%k1}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm18 = xmm23[0],xmm20[0],xmm23[1],xmm20[1],xmm23[2],xmm20[2],xmm23[3],xmm20[3],xmm23[4],xmm20[4],xmm23[5],xmm20[5],xmm23[6],xmm20[6],xmm23[7],xmm20[7]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm21 = xmm18[2,3,2,3]
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm21 = xmm21[0],zero,zero,zero,xmm21[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm24 = xmm20[3,3,3,3]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm24 = xmm18[3,3,3,3]
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm24[0],zero,zero,zero,xmm24[1],zero,zero,zero
 ; AVX512BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm21, %ymm21
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm20 = xmm20[1,1,1,1]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm18[0],zero,zero,zero,xmm18[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm18 = xmm18[1,1,1,1]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm18 = xmm18[0],zero,zero,zero,xmm18[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm18, %ymm24, %ymm18
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm18, %zmm18
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm21 = xmm26[0],xmm19[0],xmm26[1],xmm19[1],xmm26[2],xmm19[2],xmm26[3],xmm19[3],xmm26[4],xmm19[4],xmm26[5],xmm19[5],xmm26[6],xmm19[6],xmm26[7],xmm19[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm24 = xmm21[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm28 = xmm21[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm28, %ymm24, %ymm28
+; AVX512BW-NEXT:    vmovdqa64 16(%r9), %xmm24
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm28 = ymm28[2,1,3,3,6,5,7,7]
+; AVX512BW-NEXT:    vpermw %ymm21, %ymm11, %ymm21
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm28, %zmm21, %zmm21
+; AVX512BW-NEXT:    vmovdqa64 16(%r8), %xmm28
+; AVX512BW-NEXT:    vmovdqu16 %zmm21, %zmm18 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 16(%rsi), %xmm21
+; AVX512BW-NEXT:    vmovdqa32 %zmm27, %zmm18 {%k3}
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm17[8],xmm16[8],xmm17[9],xmm16[9],xmm17[10],xmm16[10],xmm17[11],xmm16[11],xmm17[12],xmm16[12],xmm17[13],xmm16[13],xmm17[14],xmm16[14],xmm17[15],xmm16[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm17 = xmm16[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm27 = xmm16[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm17, %ymm17
+; AVX512BW-NEXT:    vmovdqa64 16(%rdi), %xmm27
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm17 = ymm17[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm16, %ymm3, %ymm16
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm17, %zmm16, %zmm16
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm22 = xmm25[8],xmm22[8],xmm25[9],xmm22[9],xmm25[10],xmm22[10],xmm25[11],xmm22[11],xmm25[12],xmm22[12],xmm25[13],xmm22[13],xmm25[14],xmm22[14],xmm25[15],xmm22[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm17 = xmm22[0,1,2,3,4,5,5,7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm25 = xmm22[0,1,2,3,6,5,7,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm25, %ymm17, %ymm25
+; AVX512BW-NEXT:    vmovdqa64 16(%rcx), %xmm17
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm25 = ymm25[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm22, %ymm6, %ymm22
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm25, %zmm22, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 16(%rdx), %xmm22
+; AVX512BW-NEXT:    vmovdqu16 %zmm16, %zmm25 {%k1}
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm23[8],xmm20[8],xmm23[9],xmm20[9],xmm23[10],xmm20[10],xmm23[11],xmm20[11],xmm23[12],xmm20[12],xmm23[13],xmm20[13],xmm23[14],xmm20[14],xmm23[15],xmm20[15]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm20 = xmm16[2,3,2,3]
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm20 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm20, %ymm24, %ymm20
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm20, %zmm20
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm12, %zmm20 {%k2}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm23[0],xmm22[0],xmm23[1],xmm22[1],xmm23[2],xmm22[2],xmm23[3],xmm22[3],xmm23[4],xmm22[4],xmm23[5],xmm22[5],xmm23[6],xmm22[6],xmm23[7],xmm22[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm26[0],xmm25[0],xmm26[1],xmm25[1],xmm26[2],xmm25[2],xmm26[3],xmm25[3],xmm26[4],xmm25[4],xmm26[5],xmm25[5],xmm26[6],xmm25[6],xmm26[7],xmm25[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpermw %zmm1, %zmm10, %zmm21
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm9, %zmm21 {%k1}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm31[0],xmm29[0],xmm31[1],xmm29[1],xmm31[2],xmm29[2],xmm31[3],xmm29[3],xmm31[4],xmm29[4],xmm31[5],xmm29[5],xmm31[6],xmm29[6],xmm31[7],xmm29[7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm24 = xmm0[3,3,3,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm24[0],zero,zero,zero,xmm24[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm0, %ymm24, %ymm0
-; AVX512BW-NEXT:    vmovdqa 32(%rdx), %xmm2
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm24
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm27[0],xmm2[1],xmm27[1],xmm2[2],xmm27[2],xmm2[3],xmm27[3],xmm2[4],xmm27[4],xmm2[5],xmm27[5],xmm2[6],xmm27[6],xmm2[7],xmm27[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm12, %zmm24 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 16(%rsi), %xmm28
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm23[8],xmm22[8],xmm23[9],xmm22[9],xmm23[10],xmm22[10],xmm23[11],xmm22[11],xmm23[12],xmm22[12],xmm23[13],xmm22[13],xmm23[14],xmm22[14],xmm23[15],xmm22[15]
-; AVX512BW-NEXT:    vmovdqa64 16(%rdi), %xmm30
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
-; AVX512BW-NEXT:    vmovdqa64 16(%rcx), %xmm25
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpermw %zmm1, %zmm10, %zmm22
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm9, %zmm22 {%k1}
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm31[8],xmm29[8],xmm31[9],xmm29[9],xmm31[10],xmm29[10],xmm31[11],xmm29[11],xmm31[12],xmm29[12],xmm31[13],xmm29[13],xmm31[14],xmm29[14],xmm31[15],xmm29[15]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm23 = xmm0[3,3,3,3]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm23 = xmm16[3,3,3,3]
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm23 = xmm23[0],zero,zero,zero,xmm23[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm23 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm0, %ymm23, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm23
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm2[8],xmm27[8],xmm2[9],xmm27[9],xmm2[10],xmm27[10],xmm2[11],xmm27[11],xmm2[12],xmm27[12],xmm2[13],xmm27[13],xmm2[14],xmm27[14],xmm2[15],xmm27[15]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm12, %zmm23 {%k2}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm16[0],xmm13[0],xmm16[1],xmm13[1],xmm16[2],xmm13[2],xmm16[3],xmm13[3],xmm16[4],xmm13[4],xmm16[5],xmm13[5],xmm16[6],xmm13[6],xmm16[7],xmm13[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm18[0],xmm17[0],xmm18[1],xmm17[1],xmm18[2],xmm17[2],xmm18[3],xmm17[3],xmm18[4],xmm17[4],xmm18[5],xmm17[5],xmm18[6],xmm17[6],xmm18[7],xmm17[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpermw %zmm1, %zmm10, %zmm26
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm9, %zmm26 {%k1}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm30[0],xmm28[0],xmm30[1],xmm28[1],xmm30[2],xmm28[2],xmm30[3],xmm28[3],xmm30[4],xmm28[4],xmm30[5],xmm28[5],xmm30[6],xmm28[6],xmm30[7],xmm28[7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm20, %ymm20
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm23 = xmm16[0],zero,zero,zero,xmm16[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm16 = xmm16[1,1,1,1]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm16 = xmm16[0],zero,zero,zero,xmm16[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm16, %ymm23, %ymm16
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm16, %zmm16
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm26[8],xmm19[8],xmm26[9],xmm19[9],xmm26[10],xmm19[10],xmm26[11],xmm19[11],xmm26[12],xmm19[12],xmm26[13],xmm19[13],xmm26[14],xmm19[14],xmm26[15],xmm19[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm19[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm19[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm20, %ymm20
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm20 = ymm20[2,1,3,3,6,5,7,7]
+; AVX512BW-NEXT:    vpermw %ymm19, %ymm11, %ymm19
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm19, %zmm19
+; AVX512BW-NEXT:    vmovdqu16 %zmm19, %zmm16 {%k2}
+; AVX512BW-NEXT:    vmovdqa32 %zmm25, %zmm16 {%k3}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3],xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm19[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm19[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm20, %ymm20
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm20 = ymm20[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm19, %ymm3, %ymm19
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm19, %zmm19
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm28[0],xmm24[0],xmm28[1],xmm24[1],xmm28[2],xmm24[2],xmm28[3],xmm24[3],xmm28[4],xmm24[4],xmm28[5],xmm24[5],xmm28[6],xmm24[6],xmm28[7],xmm24[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm20[0,1,2,3,4,5,5,7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm25 = xmm20[0,1,2,3,6,5,7,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm25, %ymm23, %ymm23
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm23 = ymm23[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm20, %ymm6, %ymm20
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm23, %zmm20, %zmm20
+; AVX512BW-NEXT:    vmovdqu16 %zmm19, %zmm20 {%k1}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm27[0],xmm21[0],xmm27[1],xmm21[1],xmm27[2],xmm21[2],xmm27[3],xmm21[3],xmm27[4],xmm21[4],xmm27[5],xmm21[5],xmm27[6],xmm21[6],xmm27[7],xmm21[7]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm23 = xmm19[2,3,2,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm23 = xmm23[0],zero,zero,zero,xmm23[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm25 = xmm19[3,3,3,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm25[0],zero,zero,zero,xmm25[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm25, %ymm23, %ymm23
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm19[0],zero,zero,zero,xmm19[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm19 = xmm19[1,1,1,1]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm19 = xmm19[0],zero,zero,zero,xmm19[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm19, %ymm25, %ymm19
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm23, %zmm19, %zmm19
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm22[0],xmm17[0],xmm22[1],xmm17[1],xmm22[2],xmm17[2],xmm22[3],xmm17[3],xmm22[4],xmm17[4],xmm22[5],xmm17[5],xmm22[6],xmm17[6],xmm22[7],xmm17[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm25 = xmm23[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm26 = xmm23[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm26, %ymm25, %ymm25
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm25 = ymm25[2,1,3,3,6,5,7,7]
+; AVX512BW-NEXT:    vpermw %ymm23, %ymm11, %ymm23
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm25, %zmm23, %zmm23
+; AVX512BW-NEXT:    vmovdqu16 %zmm23, %zmm19 {%k2}
+; AVX512BW-NEXT:    vmovdqa32 %zmm20, %zmm19 {%k3}
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm13 = xmm12[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm12[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm20, %ymm13, %ymm13
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm13 = ymm13[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm12, %ymm3, %ymm12
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm13 = xmm28[8],xmm24[8],xmm28[9],xmm24[9],xmm28[10],xmm24[10],xmm28[11],xmm24[11],xmm28[12],xmm24[12],xmm28[13],xmm24[13],xmm28[14],xmm24[14],xmm28[15],xmm24[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm13[0,1,2,3,4,5,5,7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm13[0,1,2,3,6,5,7,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm20, %ymm20
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm20 = ymm20[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vpermw %ymm13, %ymm6, %ymm13
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm13, %zmm13
+; AVX512BW-NEXT:    vmovdqu16 %zmm12, %zmm13 {%k1}
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm27[8],xmm21[8],xmm27[9],xmm21[9],xmm27[10],xmm21[10],xmm27[11],xmm21[11],xmm27[12],xmm21[12],xmm27[13],xmm21[13],xmm27[14],xmm21[14],xmm27[15],xmm21[15]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm20 = xmm12[2,3,2,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm20 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm21 = xmm12[3,3,3,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm21 = xmm21[0],zero,zero,zero,xmm21[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm21, %ymm20, %ymm20
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm21 = xmm12[0],zero,zero,zero,xmm12[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[1,1,1,1]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm12 = xmm12[0],zero,zero,zero,xmm12[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm12, %ymm21, %ymm12
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm12, %zmm12
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm22[8],xmm17[8],xmm22[9],xmm17[9],xmm22[10],xmm17[10],xmm22[11],xmm17[11],xmm22[12],xmm17[12],xmm22[13],xmm17[13],xmm22[14],xmm17[14],xmm22[15],xmm17[15]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm17[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm21 = xmm17[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm21, %ymm20, %ymm20
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm20 = ymm20[2,1,3,3,6,5,7,7]
+; AVX512BW-NEXT:    vpermw %ymm17, %ymm11, %ymm17
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm17, %zmm17
+; AVX512BW-NEXT:    vmovdqu16 %zmm17, %zmm12 {%k2}
+; AVX512BW-NEXT:    vmovdqa32 %zmm13, %zmm12 {%k3}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm13 = xmm0[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm2, %ymm2
+; AVX512BW-NEXT:    vpermw %ymm0, %ymm3, %ymm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm2[0,1,2,3,4,5,5,7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,6,5,7,7]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
+; AVX512BW-NEXT:    vpermw %ymm2, %ymm6, %ymm2
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[0,2,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdx), %xmm2
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm27
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm25[0],xmm2[1],xmm25[1],xmm2[2],xmm25[2],xmm2[3],xmm25[3],xmm2[4],xmm25[4],xmm2[5],xmm25[5],xmm2[6],xmm25[6],xmm2[7],xmm25[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm12, %zmm27 {%k2}
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm16[8],xmm13[8],xmm16[9],xmm13[9],xmm16[10],xmm13[10],xmm16[11],xmm13[11],xmm16[12],xmm13[12],xmm16[13],xmm13[13],xmm16[14],xmm13[14],xmm16[15],xmm13[15]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm18[8],xmm17[8],xmm18[9],xmm17[9],xmm18[10],xmm17[10],xmm18[11],xmm17[11],xmm18[12],xmm17[12],xmm18[13],xmm17[13],xmm18[14],xmm17[14],xmm18[15],xmm17[15]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm13
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm10, %zmm0
-; AVX512BW-NEXT:    vpermw %zmm13, %zmm9, %zmm0 {%k1}
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm30[8],xmm28[8],xmm30[9],xmm28[9],xmm30[10],xmm28[10],xmm30[11],xmm28[11],xmm30[12],xmm28[12],xmm30[13],xmm28[13],xmm30[14],xmm28[14],xmm30[15],xmm28[15]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX512BW-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm13 # 16-byte Folded Reload
-; AVX512BW-NEXT:    # xmm13 = xmm13[0],mem[0],xmm13[1],mem[1],xmm13[2],mem[2],xmm13[3],mem[3],xmm13[4],mem[4],xmm13[5],mem[5],xmm13[6],mem[6],xmm13[7],mem[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3],xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm5
-; AVX512BW-NEXT:    vpermw %zmm5, %zmm10, %zmm5
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm13, %zmm13, %zmm7
-; AVX512BW-NEXT:    vpermw %zmm7, %zmm9, %zmm5 {%k1}
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm9 = xmm1[3,3,3,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti128 $1, %xmm9, %ymm7, %ymm7
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm9 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm9, %ymm1
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm25[8],xmm2[9],xmm25[9],xmm2[10],xmm25[10],xmm2[11],xmm25[11],xmm2[12],xmm25[12],xmm2[13],xmm25[13],xmm2[14],xmm25[14],xmm2[15],xmm25[15]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpermw %zmm2, %zmm12, %zmm1 {%k2}
-; AVX512BW-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm2 # 16-byte Folded Reload
-; AVX512BW-NEXT:    # xmm2 = xmm6[0],mem[0],xmm6[1],mem[1],xmm6[2],mem[2],xmm6[3],mem[3],xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[3,3,3,3]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti128 $1, %xmm7, %ymm6, %ymm6
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm7, %ymm2
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm11[0],xmm8[0],xmm11[1],xmm8[1],xmm11[2],xmm8[2],xmm11[3],xmm8[3],xmm11[4],xmm8[4],xmm11[5],xmm8[5],xmm11[6],xmm8[6],xmm11[7],xmm8[7]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm6, %zmm6, %zmm6
-; AVX512BW-NEXT:    vpermw %zmm6, %zmm12, %zmm2 {%k2}
-; AVX512BW-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa32 %zmm4, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm14, %zmm15 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm19, %zmm20 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm21, %zmm24 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm22, %zmm23 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm26, %zmm27 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm5, %zmm2 {%k1}
+; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm4, %ymm0
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm3[0,1,2,3,4,4,6,5]
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm3[0,1,2,3,4,6,6,7]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm4
+; AVX512BW-NEXT:    vpermw %ymm3, %ymm11, %ymm3
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,1,3,3,6,5,7,7]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
+; AVX512BW-NEXT:    vmovdqu16 %zmm3, %zmm0 {%k2}
+; AVX512BW-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k3}
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, 320(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, 256(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, 448(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, 384(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 320(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, 256(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, 448(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, 384(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, 64(%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -9024,230 +9122,336 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW:       # %bb.0:
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-BW-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vmovdqa 16(%r10), %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r10), %xmm22
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%r10), %xmm19
-; AVX512DQ-BW-NEXT:    vmovdqa (%rax), %xmm0
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rax), %xmm16
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rax), %xmm23
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rax), %xmm20
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %xmm5
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%r9), %xmm17
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r9), %xmm25
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%r9), %xmm21
-; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm7
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%r8), %xmm18
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r8), %xmm26
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%r8), %xmm24
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm7[8],xmm5[8],xmm7[9],xmm5[9],xmm7[10],xmm5[10],xmm7[11],xmm5[11],xmm7[12],xmm5[12],xmm7[13],xmm5[13],xmm7[14],xmm5[14],xmm7[15],xmm5[15]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm1
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,1,0,1,4,5,1,3,2,1,2,1,4,5,3,3,16,17,20,21,20,21,21,23,16,17,22,21,22,21,23,23]
-; AVX512DQ-BW-NEXT:    vpermw %zmm1, %zmm10, %zmm4
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,4,5,2,1,0,2,0,2,4,5,2,3,16,17,20,20,20,20,22,21,16,17,20,22,20,22,22,23]
+; AVX512DQ-BW-NEXT:    vmovdqa (%r10), %xmm0
+; AVX512DQ-BW-NEXT:    vmovdqa 16(%r10), %xmm12
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r10), %xmm16
+; AVX512DQ-BW-NEXT:    vmovdqa 48(%r10), %xmm15
+; AVX512DQ-BW-NEXT:    vmovdqa (%rax), %xmm2
+; AVX512DQ-BW-NEXT:    vmovdqa 16(%rax), %xmm13
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rax), %xmm17
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rax), %xmm18
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm1[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [0,0,0,0,4,5,2,1,0,2,0,2,4,5,2,3]
+; AVX512DQ-BW-NEXT:    vpermw %ymm1, %ymm3, %ymm1
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm1
+; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%r9), %xmm19
+; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%r8), %xmm21
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm7[0,1,2,3,4,5,5,7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm7[0,1,2,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm8, %ymm6, %ymm6
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm8 = ymm6[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm6 = [0,1,0,1,4,5,1,3,2,1,2,1,4,5,3,3]
+; AVX512DQ-BW-NEXT:    vpermw %ymm7, %ymm6, %ymm7
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm7, %zmm14
 ; AVX512DQ-BW-NEXT:    movl $-2004318072, %eax # imm = 0x88888888
 ; AVX512DQ-BW-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm9, %zmm4 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %xmm0
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rsi), %xmm28
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm6
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rdi), %xmm30
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm1, %zmm14 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %xmm7
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rsi), %xmm24
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm8
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rdi), %xmm27
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm10 = xmm1[3,3,3,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm10, %ymm9, %ymm9
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm10 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[3,3,3,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm8, %ymm1, %ymm1
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm8 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm0, %ymm8, %ymm0
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm3
-; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %xmm8
-; AVX512DQ-BW-NEXT:    vmovdqa 48(%rcx), %xmm1
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm11
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm11[8],xmm8[8],xmm11[9],xmm8[9],xmm11[10],xmm8[10],xmm11[11],xmm8[11],xmm11[12],xmm8[12],xmm11[13],xmm8[13],xmm11[14],xmm8[14],xmm11[15],xmm8[15]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm12, %zmm12, %zmm14
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,2,1,2,1,6,7,0,2,2,3,2,3,6,7,20,20,18,19,22,21,22,21,20,22,18,19,22,23,22,23]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm10, %ymm1
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm9, %zmm1, %zmm1
+; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %xmm9
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rcx), %xmm28
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm10
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rdx), %xmm29
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm11 = xmm20[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm22 = xmm20[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm22, %ymm11, %ymm11
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm22 = ymm11[2,1,3,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,0,2,1,2,1,6,7,0,2,2,3,2,3,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm20, %ymm11, %ymm20
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm22, %zmm20, %zmm20
 ; AVX512DQ-BW-NEXT:    movl $572662306, %eax # imm = 0x22222222
 ; AVX512DQ-BW-NEXT:    kmovd %eax, %k2
-; AVX512DQ-BW-NEXT:    vpermw %zmm14, %zmm12, %zmm3 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm20[0],xmm19[0],xmm20[1],xmm19[1],xmm20[2],xmm19[2],xmm20[3],xmm19[3],xmm20[4],xmm19[4],xmm20[5],xmm19[5],xmm20[6],xmm19[6],xmm20[7],xmm19[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm14, %zmm14, %zmm15
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm24[0],xmm21[0],xmm24[1],xmm21[1],xmm24[2],xmm21[2],xmm24[3],xmm21[3],xmm24[4],xmm21[4],xmm24[5],xmm21[5],xmm24[6],xmm21[6],xmm24[7],xmm21[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm14, %zmm14, %zmm14
-; AVX512DQ-BW-NEXT:    vpermw %zmm14, %zmm10, %zmm14
-; AVX512DQ-BW-NEXT:    vpermw %zmm15, %zmm9, %zmm14 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm30[0],xmm28[0],xmm30[1],xmm28[1],xmm30[2],xmm28[2],xmm30[3],xmm28[3],xmm30[4],xmm28[4],xmm30[5],xmm28[5],xmm30[6],xmm28[6],xmm30[7],xmm28[7]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm27 = xmm15[2,3,2,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm27 = xmm27[0],zero,zero,zero,xmm27[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm29 = xmm15[3,3,3,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm29 = xmm29[0],zero,zero,zero,xmm29[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm29, %ymm27, %ymm27
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm29 = xmm15[0],zero,zero,zero,xmm15[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm20, %zmm1 {%k2}
+; AVX512DQ-BW-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; AVX512DQ-BW-NEXT:    kmovd %eax, %k3
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm14, %zmm1 {%k3}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm18[0],xmm15[0],xmm18[1],xmm15[1],xmm18[2],xmm15[2],xmm18[3],xmm15[3],xmm18[4],xmm15[4],xmm18[5],xmm15[5],xmm18[6],xmm15[6],xmm18[7],xmm15[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm14[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm22 = xmm14[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm22, %ymm20, %ymm20
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm20 = ymm20[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm14, %ymm3, %ymm14
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm14, %zmm14
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm21[0],xmm19[0],xmm21[1],xmm19[1],xmm21[2],xmm19[2],xmm21[3],xmm19[3],xmm21[4],xmm19[4],xmm21[5],xmm19[5],xmm21[6],xmm19[6],xmm21[7],xmm19[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm22 = xmm20[0,1,2,3,4,5,5,7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm20[0,1,2,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm22, %ymm22
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm22 = ymm22[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm20, %ymm6, %ymm20
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm22, %zmm20, %zmm23
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm14, %zmm23 {%k1}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm27[0],xmm24[0],xmm27[1],xmm24[1],xmm27[2],xmm24[2],xmm27[3],xmm24[3],xmm27[4],xmm24[4],xmm27[5],xmm24[5],xmm27[6],xmm24[6],xmm27[7],xmm24[7]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm20 = xmm14[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm20 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm22 = xmm14[3,3,3,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm22 = xmm22[0],zero,zero,zero,xmm22[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm22, %ymm20, %ymm20
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm22 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm14 = xmm14[1,1,1,1]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm14 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm14, %ymm22, %ymm14
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm14, %zmm14
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm29[0],xmm28[0],xmm29[1],xmm28[1],xmm29[2],xmm28[2],xmm29[3],xmm28[3],xmm29[4],xmm28[4],xmm29[5],xmm28[5],xmm29[6],xmm28[6],xmm29[7],xmm28[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm22 = xmm20[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm25 = xmm20[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm25, %ymm22, %ymm25
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r9), %xmm22
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm25 = ymm25[2,1,3,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm20, %ymm11, %ymm20
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm25, %zmm20, %zmm20
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r8), %xmm25
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm20, %zmm14 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rsi), %xmm20
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm23, %zmm14 {%k3}
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm18[8],xmm15[8],xmm18[9],xmm15[9],xmm18[10],xmm15[10],xmm18[11],xmm15[11],xmm18[12],xmm15[12],xmm18[13],xmm15[13],xmm18[14],xmm15[14],xmm18[15],xmm15[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm18 = xmm15[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm15[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm18, %ymm18
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdi), %xmm23
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm18 = ymm18[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm15, %ymm3, %ymm15
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm18, %zmm15, %zmm15
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm21[8],xmm19[8],xmm21[9],xmm19[9],xmm21[10],xmm19[10],xmm21[11],xmm19[11],xmm21[12],xmm19[12],xmm21[13],xmm19[13],xmm21[14],xmm19[14],xmm21[15],xmm19[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm19 = xmm18[0,1,2,3,4,5,5,7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm21 = xmm18[0,1,2,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm21, %ymm19, %ymm21
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rcx), %xmm19
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm21 = ymm21[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm18, %ymm6, %ymm18
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm18, %zmm18
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdx), %xmm26
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm15, %zmm18 {%k1}
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm27[8],xmm24[8],xmm27[9],xmm24[9],xmm27[10],xmm24[10],xmm27[11],xmm24[11],xmm27[12],xmm24[12],xmm27[13],xmm24[13],xmm27[14],xmm24[14],xmm27[15],xmm24[15]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm21 = xmm15[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm21 = xmm21[0],zero,zero,zero,xmm21[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm24 = xmm15[3,3,3,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm24[0],zero,zero,zero,xmm24[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm21, %ymm21
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm15[0],zero,zero,zero,xmm15[1],zero,zero,zero
 ; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[1,1,1,1]
 ; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm15 = xmm15[0],zero,zero,zero,xmm15[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm15, %ymm29, %ymm15
-; AVX512DQ-BW-NEXT:    vmovdqa 48(%rdx), %xmm0
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm27, %zmm15, %zmm15
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm27 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm27, %zmm27, %zmm27
-; AVX512DQ-BW-NEXT:    vpermw %zmm27, %zmm12, %zmm15 {%k2}
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rsi), %xmm29
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm20[8],xmm19[8],xmm20[9],xmm19[9],xmm20[10],xmm19[10],xmm20[11],xmm19[11],xmm20[12],xmm19[12],xmm20[13],xmm19[13],xmm20[14],xmm19[14],xmm20[15],xmm19[15]
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdi), %xmm31
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm24[8],xmm21[8],xmm24[9],xmm21[9],xmm24[10],xmm21[10],xmm24[11],xmm21[11],xmm24[12],xmm21[12],xmm24[13],xmm21[13],xmm24[14],xmm21[14],xmm24[15],xmm21[15]
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rcx), %xmm27
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm19, %zmm19, %zmm21
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm20, %zmm20, %zmm19
-; AVX512DQ-BW-NEXT:    vpermw %zmm19, %zmm10, %zmm19
-; AVX512DQ-BW-NEXT:    vpermw %zmm21, %zmm9, %zmm19 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm30[8],xmm28[8],xmm30[9],xmm28[9],xmm30[10],xmm28[10],xmm30[11],xmm28[11],xmm30[12],xmm28[12],xmm30[13],xmm28[13],xmm30[14],xmm28[14],xmm30[15],xmm28[15]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm21 = xmm20[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm15, %ymm24, %ymm15
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm15, %zmm15
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm21 = xmm29[8],xmm28[8],xmm29[9],xmm28[9],xmm29[10],xmm28[10],xmm29[11],xmm28[11],xmm29[12],xmm28[12],xmm29[13],xmm28[13],xmm29[14],xmm28[14],xmm29[15],xmm28[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm24 = xmm21[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm27 = xmm21[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm24, %ymm24
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm24 = ymm24[2,1,3,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm21, %ymm11, %ymm21
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm24, %zmm21, %zmm21
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm21, %zmm15 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm18, %zmm15 {%k3}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm18 = xmm17[0],xmm16[0],xmm17[1],xmm16[1],xmm17[2],xmm16[2],xmm17[3],xmm16[3],xmm17[4],xmm16[4],xmm17[5],xmm16[5],xmm17[6],xmm16[6],xmm17[7],xmm16[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm21 = xmm18[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm24 = xmm18[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm21, %ymm21
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm21 = ymm21[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm18, %ymm3, %ymm18
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm18, %zmm18
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm21 = xmm25[0],xmm22[0],xmm25[1],xmm22[1],xmm25[2],xmm22[2],xmm25[3],xmm22[3],xmm25[4],xmm22[4],xmm25[5],xmm22[5],xmm25[6],xmm22[6],xmm25[7],xmm22[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm24 = xmm21[0,1,2,3,4,5,5,7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm27 = xmm21[0,1,2,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm24, %ymm24
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm24 = ymm24[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm21, %ymm6, %ymm21
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm24, %zmm21, %zmm27
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm18, %zmm27 {%k1}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm18 = xmm23[0],xmm20[0],xmm23[1],xmm20[1],xmm23[2],xmm20[2],xmm23[3],xmm20[3],xmm23[4],xmm20[4],xmm23[5],xmm20[5],xmm23[6],xmm20[6],xmm23[7],xmm20[7]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm21 = xmm18[2,3,2,3]
 ; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm21 = xmm21[0],zero,zero,zero,xmm21[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm24 = xmm20[3,3,3,3]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm24 = xmm18[3,3,3,3]
 ; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm24[0],zero,zero,zero,xmm24[1],zero,zero,zero
 ; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm21, %ymm21
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm20 = xmm20[1,1,1,1]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm18[0],zero,zero,zero,xmm18[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm18 = xmm18[1,1,1,1]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm18 = xmm18[0],zero,zero,zero,xmm18[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm18, %ymm24, %ymm18
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm18, %zmm18
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm21 = xmm26[0],xmm19[0],xmm26[1],xmm19[1],xmm26[2],xmm19[2],xmm26[3],xmm19[3],xmm26[4],xmm19[4],xmm26[5],xmm19[5],xmm26[6],xmm19[6],xmm26[7],xmm19[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm24 = xmm21[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm28 = xmm21[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm28, %ymm24, %ymm28
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%r9), %xmm24
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm28 = ymm28[2,1,3,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm21, %ymm11, %ymm21
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm28, %zmm21, %zmm21
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%r8), %xmm28
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm21, %zmm18 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rsi), %xmm21
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm27, %zmm18 {%k3}
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm17[8],xmm16[8],xmm17[9],xmm16[9],xmm17[10],xmm16[10],xmm17[11],xmm16[11],xmm17[12],xmm16[12],xmm17[13],xmm16[13],xmm17[14],xmm16[14],xmm17[15],xmm16[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm17 = xmm16[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm27 = xmm16[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm17, %ymm17
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rdi), %xmm27
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm17 = ymm17[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm16, %ymm3, %ymm16
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm17, %zmm16, %zmm16
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm22 = xmm25[8],xmm22[8],xmm25[9],xmm22[9],xmm25[10],xmm22[10],xmm25[11],xmm22[11],xmm25[12],xmm22[12],xmm25[13],xmm22[13],xmm25[14],xmm22[14],xmm25[15],xmm22[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm17 = xmm22[0,1,2,3,4,5,5,7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm25 = xmm22[0,1,2,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm25, %ymm17, %ymm25
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rcx), %xmm17
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm25 = ymm25[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm22, %ymm6, %ymm22
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm25, %zmm22, %zmm25
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rdx), %xmm22
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm16, %zmm25 {%k1}
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm23[8],xmm20[8],xmm23[9],xmm20[9],xmm23[10],xmm20[10],xmm23[11],xmm20[11],xmm23[12],xmm20[12],xmm23[13],xmm20[13],xmm23[14],xmm20[14],xmm23[15],xmm20[15]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm20 = xmm16[2,3,2,3]
 ; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm20 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm20, %ymm24, %ymm20
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm20, %zmm20
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm12, %zmm20 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm23[0],xmm22[0],xmm23[1],xmm22[1],xmm23[2],xmm22[2],xmm23[3],xmm22[3],xmm23[4],xmm22[4],xmm23[5],xmm22[5],xmm23[6],xmm22[6],xmm23[7],xmm22[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm26[0],xmm25[0],xmm26[1],xmm25[1],xmm26[2],xmm25[2],xmm26[3],xmm25[3],xmm26[4],xmm25[4],xmm26[5],xmm25[5],xmm26[6],xmm25[6],xmm26[7],xmm25[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm1
-; AVX512DQ-BW-NEXT:    vpermw %zmm1, %zmm10, %zmm21
-; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm9, %zmm21 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm31[0],xmm29[0],xmm31[1],xmm29[1],xmm31[2],xmm29[2],xmm31[3],xmm29[3],xmm31[4],xmm29[4],xmm31[5],xmm29[5],xmm31[6],xmm29[6],xmm31[7],xmm29[7]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm24 = xmm0[3,3,3,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm24[0],zero,zero,zero,xmm24[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm1, %ymm1
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm0, %ymm24, %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%rdx), %xmm2
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm24
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm27[0],xmm2[1],xmm27[1],xmm2[2],xmm27[2],xmm2[3],xmm27[3],xmm2[4],xmm27[4],xmm2[5],xmm27[5],xmm2[6],xmm27[6],xmm2[7],xmm27[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm12, %zmm24 {%k2}
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rsi), %xmm28
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm23[8],xmm22[8],xmm23[9],xmm22[9],xmm23[10],xmm22[10],xmm23[11],xmm22[11],xmm23[12],xmm22[12],xmm23[13],xmm22[13],xmm23[14],xmm22[14],xmm23[15],xmm22[15]
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rdi), %xmm30
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rcx), %xmm25
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm1
-; AVX512DQ-BW-NEXT:    vpermw %zmm1, %zmm10, %zmm22
-; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm9, %zmm22 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm31[8],xmm29[8],xmm31[9],xmm29[9],xmm31[10],xmm29[10],xmm31[11],xmm29[11],xmm31[12],xmm29[12],xmm31[13],xmm29[13],xmm31[14],xmm29[14],xmm31[15],xmm29[15]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm23 = xmm0[3,3,3,3]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm23 = xmm16[3,3,3,3]
 ; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm23 = xmm23[0],zero,zero,zero,xmm23[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm1, %ymm1
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm23 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm0, %ymm23, %ymm0
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm23
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm2[8],xmm27[8],xmm2[9],xmm27[9],xmm2[10],xmm27[10],xmm2[11],xmm27[11],xmm2[12],xmm27[12],xmm2[13],xmm27[13],xmm2[14],xmm27[14],xmm2[15],xmm27[15]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm12, %zmm23 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm16[0],xmm13[0],xmm16[1],xmm13[1],xmm16[2],xmm13[2],xmm16[3],xmm13[3],xmm16[4],xmm13[4],xmm16[5],xmm13[5],xmm16[6],xmm13[6],xmm16[7],xmm13[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm18[0],xmm17[0],xmm18[1],xmm17[1],xmm18[2],xmm17[2],xmm18[3],xmm17[3],xmm18[4],xmm17[4],xmm18[5],xmm17[5],xmm18[6],xmm17[6],xmm18[7],xmm17[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm1
-; AVX512DQ-BW-NEXT:    vpermw %zmm1, %zmm10, %zmm26
-; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm9, %zmm26 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm30[0],xmm28[0],xmm30[1],xmm28[1],xmm30[2],xmm28[2],xmm30[3],xmm28[3],xmm30[4],xmm28[4],xmm30[5],xmm28[5],xmm30[6],xmm28[6],xmm30[7],xmm28[7]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm20, %ymm20
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm23 = xmm16[0],zero,zero,zero,xmm16[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm16 = xmm16[1,1,1,1]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm16 = xmm16[0],zero,zero,zero,xmm16[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm16, %ymm23, %ymm16
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm16, %zmm16
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm26[8],xmm19[8],xmm26[9],xmm19[9],xmm26[10],xmm19[10],xmm26[11],xmm19[11],xmm26[12],xmm19[12],xmm26[13],xmm19[13],xmm26[14],xmm19[14],xmm26[15],xmm19[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm19[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm19[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm20, %ymm20
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm20 = ymm20[2,1,3,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm19, %ymm11, %ymm19
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm19, %zmm19
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm19, %zmm16 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm25, %zmm16 {%k3}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3],xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm19[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm19[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm20, %ymm20
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm20 = ymm20[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm19, %ymm3, %ymm19
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm19, %zmm19
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm28[0],xmm24[0],xmm28[1],xmm24[1],xmm28[2],xmm24[2],xmm28[3],xmm24[3],xmm28[4],xmm24[4],xmm28[5],xmm24[5],xmm28[6],xmm24[6],xmm28[7],xmm24[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm20[0,1,2,3,4,5,5,7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm25 = xmm20[0,1,2,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm25, %ymm23, %ymm23
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm23 = ymm23[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm20, %ymm6, %ymm20
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm23, %zmm20, %zmm20
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm19, %zmm20 {%k1}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm27[0],xmm21[0],xmm27[1],xmm21[1],xmm27[2],xmm21[2],xmm27[3],xmm21[3],xmm27[4],xmm21[4],xmm27[5],xmm21[5],xmm27[6],xmm21[6],xmm27[7],xmm21[7]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm23 = xmm19[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm23 = xmm23[0],zero,zero,zero,xmm23[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm25 = xmm19[3,3,3,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm25[0],zero,zero,zero,xmm25[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm25, %ymm23, %ymm23
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm19[0],zero,zero,zero,xmm19[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm19 = xmm19[1,1,1,1]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm19 = xmm19[0],zero,zero,zero,xmm19[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm19, %ymm25, %ymm19
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm23, %zmm19, %zmm19
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm22[0],xmm17[0],xmm22[1],xmm17[1],xmm22[2],xmm17[2],xmm22[3],xmm17[3],xmm22[4],xmm17[4],xmm22[5],xmm17[5],xmm22[6],xmm17[6],xmm22[7],xmm17[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm25 = xmm23[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm26 = xmm23[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm26, %ymm25, %ymm25
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm25 = ymm25[2,1,3,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm23, %ymm11, %ymm23
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm25, %zmm23, %zmm23
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm23, %zmm19 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm20, %zmm19 {%k3}
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm13 = xmm12[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm12[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm20, %ymm13, %ymm13
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm13 = ymm13[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm12, %ymm3, %ymm12
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm13 = xmm28[8],xmm24[8],xmm28[9],xmm24[9],xmm28[10],xmm24[10],xmm28[11],xmm24[11],xmm28[12],xmm24[12],xmm28[13],xmm24[13],xmm28[14],xmm24[14],xmm28[15],xmm24[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm13[0,1,2,3,4,5,5,7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm23 = xmm13[0,1,2,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm23, %ymm20, %ymm20
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm20 = ymm20[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm13, %ymm6, %ymm13
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm13, %zmm13
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm12, %zmm13 {%k1}
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm27[8],xmm21[8],xmm27[9],xmm21[9],xmm27[10],xmm21[10],xmm27[11],xmm21[11],xmm27[12],xmm21[12],xmm27[13],xmm21[13],xmm27[14],xmm21[14],xmm27[15],xmm21[15]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm20 = xmm12[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm20 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm21 = xmm12[3,3,3,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm21 = xmm21[0],zero,zero,zero,xmm21[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm21, %ymm20, %ymm20
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm21 = xmm12[0],zero,zero,zero,xmm12[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[1,1,1,1]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm12 = xmm12[0],zero,zero,zero,xmm12[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm12, %ymm21, %ymm12
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm12, %zmm12
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm22[8],xmm17[8],xmm22[9],xmm17[9],xmm22[10],xmm17[10],xmm22[11],xmm17[11],xmm22[12],xmm17[12],xmm22[13],xmm17[13],xmm22[14],xmm17[14],xmm22[15],xmm17[15]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm20 = xmm17[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm21 = xmm17[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm21, %ymm20, %ymm20
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm20 = ymm20[2,1,3,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm17, %ymm11, %ymm17
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm17, %zmm17
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm17, %zmm12 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm13, %zmm12 {%k3}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm13 = xmm0[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm2, %ymm2
+; AVX512DQ-BW-NEXT:    vpermw %ymm0, %ymm3, %ymm0
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm2[0,1,2,3,4,5,5,7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
+; AVX512DQ-BW-NEXT:    vpermw %ymm2, %ymm6, %ymm2
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[0,2,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm0, %zmm2 {%k1}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
+; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa 16(%rdx), %xmm2
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm27
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm25[0],xmm2[1],xmm25[1],xmm2[2],xmm25[2],xmm2[3],xmm25[3],xmm2[4],xmm25[4],xmm2[5],xmm25[5],xmm2[6],xmm25[6],xmm2[7],xmm25[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm12, %zmm27 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm16[8],xmm13[8],xmm16[9],xmm13[9],xmm16[10],xmm13[10],xmm16[11],xmm13[11],xmm16[12],xmm13[12],xmm16[13],xmm13[13],xmm16[14],xmm13[14],xmm16[15],xmm13[15]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm18[8],xmm17[8],xmm18[9],xmm17[9],xmm18[10],xmm17[10],xmm18[11],xmm17[11],xmm18[12],xmm17[12],xmm18[13],xmm17[13],xmm18[14],xmm17[14],xmm18[15],xmm17[15]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm13
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm0
-; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm10, %zmm0
-; AVX512DQ-BW-NEXT:    vpermw %zmm13, %zmm9, %zmm0 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm30[8],xmm28[8],xmm30[9],xmm28[9],xmm30[10],xmm28[10],xmm30[11],xmm28[11],xmm30[12],xmm28[12],xmm30[13],xmm28[13],xmm30[14],xmm28[14],xmm30[15],xmm28[15]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm13 # 16-byte Folded Reload
-; AVX512DQ-BW-NEXT:    # xmm13 = xmm13[0],mem[0],xmm13[1],mem[1],xmm13[2],mem[2],xmm13[3],mem[3],xmm13[4],mem[4],xmm13[5],mem[5],xmm13[6],mem[6],xmm13[7],mem[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3],xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm5
-; AVX512DQ-BW-NEXT:    vpermw %zmm5, %zmm10, %zmm5
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm13, %zmm13, %zmm7
-; AVX512DQ-BW-NEXT:    vpermw %zmm7, %zmm9, %zmm5 {%k1}
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm9 = xmm1[3,3,3,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm9, %ymm7, %ymm7
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm9 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm9, %ymm1
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm1, %zmm1
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm25[8],xmm2[9],xmm25[9],xmm2[10],xmm25[10],xmm2[11],xmm25[11],xmm2[12],xmm25[12],xmm2[13],xmm25[13],xmm2[14],xmm25[14],xmm2[15],xmm25[15]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm2, %zmm2
-; AVX512DQ-BW-NEXT:    vpermw %zmm2, %zmm12, %zmm1 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm2 # 16-byte Folded Reload
-; AVX512DQ-BW-NEXT:    # xmm2 = xmm6[0],mem[0],xmm6[1],mem[1],xmm6[2],mem[2],xmm6[3],mem[3],xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[3,3,3,3]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm7, %ymm6, %ymm6
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm7, %ymm2
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm2
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm11[0],xmm8[0],xmm11[1],xmm8[1],xmm11[2],xmm8[2],xmm11[3],xmm8[3],xmm11[4],xmm8[4],xmm11[5],xmm8[5],xmm11[6],xmm8[6],xmm11[7],xmm8[7]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm6, %zmm6, %zmm6
-; AVX512DQ-BW-NEXT:    vpermw %zmm6, %zmm12, %zmm2 {%k2}
-; AVX512DQ-BW-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; AVX512DQ-BW-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm4, %zmm3 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm14, %zmm15 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm19, %zmm20 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm21, %zmm24 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm22, %zmm23 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm26, %zmm27 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm5, %zmm2 {%k1}
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm0, %ymm4, %ymm0
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm3[0,1,2,3,4,4,6,5]
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm3[0,1,2,3,4,6,6,7]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm4
+; AVX512DQ-BW-NEXT:    vpermw %ymm3, %ymm11, %ymm3
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[2,1,3,3,6,5,7,7]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm3, %zmm0 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k3}
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, 192(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm27, 128(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm23, 320(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm24, 256(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm20, 448(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm15, 384(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, 64(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm12, 192(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm19, 128(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, 320(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm18, 256(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm15, 448(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm14, 384(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, 64(%rax)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/widen_bitcnt.ll b/llvm/test/CodeGen/X86/widen_bitcnt.ll
index 18cb5e1b86ec3..541dfb54e96d2 100644
--- a/llvm/test/CodeGen/X86/widen_bitcnt.ll
+++ b/llvm/test/CodeGen/X86/widen_bitcnt.ll
@@ -629,71 +629,45 @@ define <8 x i32> @widen_ctlz_v2i32_v8i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
 ; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX2-NEXT:    vmovq {{.*#+}} xmm5 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; AVX2-NEXT:    vpshufb %xmm0, %xmm5, %xmm4
-; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm6
-; AVX2-NEXT:    vpbroadcastb {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm7, %xmm6, %xmm8
-; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm8, %xmm9
-; AVX2-NEXT:    vpand %xmm4, %xmm9, %xmm4
-; AVX2-NEXT:    vpshufb %xmm8, %xmm5, %xmm8
-; AVX2-NEXT:    vpaddb %xmm4, %xmm8, %xmm4
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm0, %xmm8
-; AVX2-NEXT:    vpsrlw $8, %xmm8, %xmm8
-; AVX2-NEXT:    vpand %xmm4, %xmm8, %xmm8
-; AVX2-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX2-NEXT:    vpaddw %xmm4, %xmm8, %xmm4
-; AVX2-NEXT:    vpshufb %xmm1, %xmm5, %xmm8
-; AVX2-NEXT:    vpsrlw $4, %xmm1, %xmm9
-; AVX2-NEXT:    vpand %xmm7, %xmm9, %xmm9
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm9, %xmm10
-; AVX2-NEXT:    vpand %xmm10, %xmm8, %xmm8
-; AVX2-NEXT:    vpshufb %xmm9, %xmm5, %xmm9
-; AVX2-NEXT:    vpaddb %xmm9, %xmm8, %xmm8
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm1, %xmm9
-; AVX2-NEXT:    vpsrlw $8, %xmm9, %xmm9
-; AVX2-NEXT:    vpand %xmm9, %xmm8, %xmm9
-; AVX2-NEXT:    vpsrlw $8, %xmm8, %xmm8
-; AVX2-NEXT:    vpaddw %xmm9, %xmm8, %xmm8
-; AVX2-NEXT:    vpshufb %xmm2, %xmm5, %xmm9
-; AVX2-NEXT:    vpsrlw $4, %xmm2, %xmm10
-; AVX2-NEXT:    vpand %xmm7, %xmm10, %xmm10
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm10, %xmm11
-; AVX2-NEXT:    vpand %xmm11, %xmm9, %xmm9
-; AVX2-NEXT:    vpshufb %xmm10, %xmm5, %xmm10
-; AVX2-NEXT:    vpaddb %xmm10, %xmm9, %xmm9
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm10
-; AVX2-NEXT:    vpsrlw $8, %xmm10, %xmm10
-; AVX2-NEXT:    vpand %xmm10, %xmm9, %xmm10
-; AVX2-NEXT:    vpsrlw $8, %xmm9, %xmm9
-; AVX2-NEXT:    vpaddw %xmm10, %xmm9, %xmm9
-; AVX2-NEXT:    vpshufb %xmm3, %xmm5, %xmm10
-; AVX2-NEXT:    vpsrlw $4, %xmm3, %xmm11
-; AVX2-NEXT:    vpand %xmm7, %xmm11, %xmm7
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm7, %xmm11
-; AVX2-NEXT:    vpand %xmm11, %xmm10, %xmm10
-; AVX2-NEXT:    vpshufb %xmm7, %xmm5, %xmm5
-; AVX2-NEXT:    vpaddb %xmm5, %xmm10, %xmm5
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm3, %xmm6
-; AVX2-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX2-NEXT:    vpand %xmm6, %xmm5, %xmm6
-; AVX2-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX2-NEXT:    vpaddw %xmm6, %xmm5, %xmm5
-; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm8, %ymm5
 ; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX2-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX2-NEXT:    vpshufb %ymm1, %ymm3, %ymm4
+; AVX2-NEXT:    vpsrlw $4, %ymm1, %ymm5
+; AVX2-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpxor %xmm7, %xmm7, %xmm7
+; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm5, %ymm8
+; AVX2-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; AVX2-NEXT:    vpshufb %ymm5, %ymm3, %ymm5
+; AVX2-NEXT:    vpaddb %ymm5, %ymm4, %ymm4
+; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm1, %ymm5
+; AVX2-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX2-NEXT:    vpand %ymm5, %ymm4, %ymm5
+; AVX2-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX2-NEXT:    vpaddw %ymm5, %ymm4, %ymm4
+; AVX2-NEXT:    vpcmpeqw %ymm7, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsrld $16, %ymm1, %ymm1
-; AVX2-NEXT:    vpand %ymm1, %ymm5, %ymm1
-; AVX2-NEXT:    vpsrld $16, %ymm5, %ymm5
-; AVX2-NEXT:    vpaddd %ymm1, %ymm5, %ymm1
-; AVX2-NEXT:    vinserti128 $1, %xmm9, %ymm4, %ymm4
+; AVX2-NEXT:    vpand %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpsrld $16, %ymm4, %ymm4
+; AVX2-NEXT:    vpaddd %ymm1, %ymm4, %ymm1
 ; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqw %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm4
+; AVX2-NEXT:    vpand %ymm6, %ymm4, %ymm4
+; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm4, %ymm5
+; AVX2-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX2-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpaddb %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm0, %ymm3
+; AVX2-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm3
+; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX2-NEXT:    vpaddw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpeqw %ymm7, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
-; AVX2-NEXT:    vpand %ymm0, %ymm4, %ymm0
-; AVX2-NEXT:    vpsrld $16, %ymm4, %ymm2
+; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
 ; AVX2-NEXT:    retq
@@ -1008,71 +982,45 @@ define <8 x i32> @widen_ctlz_undef_v2i32_v8i32(<2 x i32> %a0, <2 x i32> %a1, <2
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
 ; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX2-NEXT:    vmovq {{.*#+}} xmm5 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; AVX2-NEXT:    vpshufb %xmm0, %xmm5, %xmm4
-; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm6
-; AVX2-NEXT:    vpbroadcastb {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm7, %xmm6, %xmm8
-; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm8, %xmm9
-; AVX2-NEXT:    vpand %xmm4, %xmm9, %xmm4
-; AVX2-NEXT:    vpshufb %xmm8, %xmm5, %xmm8
-; AVX2-NEXT:    vpaddb %xmm4, %xmm8, %xmm4
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm0, %xmm8
-; AVX2-NEXT:    vpsrlw $8, %xmm8, %xmm8
-; AVX2-NEXT:    vpand %xmm4, %xmm8, %xmm8
-; AVX2-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX2-NEXT:    vpaddw %xmm4, %xmm8, %xmm4
-; AVX2-NEXT:    vpshufb %xmm1, %xmm5, %xmm8
-; AVX2-NEXT:    vpsrlw $4, %xmm1, %xmm9
-; AVX2-NEXT:    vpand %xmm7, %xmm9, %xmm9
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm9, %xmm10
-; AVX2-NEXT:    vpand %xmm10, %xmm8, %xmm8
-; AVX2-NEXT:    vpshufb %xmm9, %xmm5, %xmm9
-; AVX2-NEXT:    vpaddb %xmm9, %xmm8, %xmm8
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm1, %xmm9
-; AVX2-NEXT:    vpsrlw $8, %xmm9, %xmm9
-; AVX2-NEXT:    vpand %xmm9, %xmm8, %xmm9
-; AVX2-NEXT:    vpsrlw $8, %xmm8, %xmm8
-; AVX2-NEXT:    vpaddw %xmm9, %xmm8, %xmm8
-; AVX2-NEXT:    vpshufb %xmm2, %xmm5, %xmm9
-; AVX2-NEXT:    vpsrlw $4, %xmm2, %xmm10
-; AVX2-NEXT:    vpand %xmm7, %xmm10, %xmm10
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm10, %xmm11
-; AVX2-NEXT:    vpand %xmm11, %xmm9, %xmm9
-; AVX2-NEXT:    vpshufb %xmm10, %xmm5, %xmm10
-; AVX2-NEXT:    vpaddb %xmm10, %xmm9, %xmm9
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm10
-; AVX2-NEXT:    vpsrlw $8, %xmm10, %xmm10
-; AVX2-NEXT:    vpand %xmm10, %xmm9, %xmm10
-; AVX2-NEXT:    vpsrlw $8, %xmm9, %xmm9
-; AVX2-NEXT:    vpaddw %xmm10, %xmm9, %xmm9
-; AVX2-NEXT:    vpshufb %xmm3, %xmm5, %xmm10
-; AVX2-NEXT:    vpsrlw $4, %xmm3, %xmm11
-; AVX2-NEXT:    vpand %xmm7, %xmm11, %xmm7
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm7, %xmm11
-; AVX2-NEXT:    vpand %xmm11, %xmm10, %xmm10
-; AVX2-NEXT:    vpshufb %xmm7, %xmm5, %xmm5
-; AVX2-NEXT:    vpaddb %xmm5, %xmm10, %xmm5
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm3, %xmm6
-; AVX2-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX2-NEXT:    vpand %xmm6, %xmm5, %xmm6
-; AVX2-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX2-NEXT:    vpaddw %xmm6, %xmm5, %xmm5
-; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm8, %ymm5
 ; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX2-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX2-NEXT:    vpshufb %ymm1, %ymm3, %ymm4
+; AVX2-NEXT:    vpsrlw $4, %ymm1, %ymm5
+; AVX2-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpxor %xmm7, %xmm7, %xmm7
+; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm5, %ymm8
+; AVX2-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; AVX2-NEXT:    vpshufb %ymm5, %ymm3, %ymm5
+; AVX2-NEXT:    vpaddb %ymm5, %ymm4, %ymm4
+; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm1, %ymm5
+; AVX2-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX2-NEXT:    vpand %ymm5, %ymm4, %ymm5
+; AVX2-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX2-NEXT:    vpaddw %ymm5, %ymm4, %ymm4
+; AVX2-NEXT:    vpcmpeqw %ymm7, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsrld $16, %ymm1, %ymm1
-; AVX2-NEXT:    vpand %ymm1, %ymm5, %ymm1
-; AVX2-NEXT:    vpsrld $16, %ymm5, %ymm5
-; AVX2-NEXT:    vpaddd %ymm1, %ymm5, %ymm1
-; AVX2-NEXT:    vinserti128 $1, %xmm9, %ymm4, %ymm4
+; AVX2-NEXT:    vpand %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpsrld $16, %ymm4, %ymm4
+; AVX2-NEXT:    vpaddd %ymm1, %ymm4, %ymm1
 ; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqw %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm4
+; AVX2-NEXT:    vpand %ymm6, %ymm4, %ymm4
+; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm4, %ymm5
+; AVX2-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX2-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpaddb %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm0, %ymm3
+; AVX2-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm3
+; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX2-NEXT:    vpaddw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpeqw %ymm7, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
-; AVX2-NEXT:    vpand %ymm0, %ymm4, %ymm0
-; AVX2-NEXT:    vpsrld $16, %ymm4, %ymm2
+; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
 ; AVX2-NEXT:    retq

From 05671cec6f736c65b7df29234db828116e6d95dd Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 22 Jul 2024 14:24:55 +0100
Subject: [PATCH 410/486] [X86] Add tests showing failure to push freeze
 through SSE PMADD nodes

PMADD guarantee inbounds/saturated ext-multiply-add results

Test to help with regression identified on #84924
---
 llvm/test/CodeGen/X86/combine-pmadd.ll | 62 ++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/llvm/test/CodeGen/X86/combine-pmadd.ll b/llvm/test/CodeGen/X86/combine-pmadd.ll
index 565d9ef6eb3e6..8ba5a25678122 100644
--- a/llvm/test/CodeGen/X86/combine-pmadd.ll
+++ b/llvm/test/CodeGen/X86/combine-pmadd.ll
@@ -63,6 +63,37 @@ define <8 x i32> @combine_pmaddwd_concat(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>
   ret <8 x i32> %3
 }
 
+define <8 x i32> @combine_pmaddwd_concat_freeze(<8 x i16> %a0, <8 x i16> %a1) {
+; SSE-LABEL: combine_pmaddwd_concat_freeze:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmovsxbw {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
+; SSE-NEXT:    pmaddwd %xmm2, %xmm0
+; SSE-NEXT:    pmaddwd %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: combine_pmaddwd_concat_freeze:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpmaddwd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmaddwd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: combine_pmaddwd_concat_freeze:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
+; AVX2-NEXT:    vpmaddwd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmaddwd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %lo = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+  %hi = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+  %flo = freeze <4 x i32> %lo
+  %fhi = freeze <4 x i32> %hi
+  %res = shufflevector <4 x i32> %flo, <4 x i32> %fhi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i32> %res
+}
+
 define <4 x i32> @combine_pmaddwd_demandedelts(<8 x i16> %a0, <8 x i16> %a1) {
 ; SSE-LABEL: combine_pmaddwd_demandedelts:
 ; SSE:       # %bb.0:
@@ -178,6 +209,37 @@ define <16 x i16> @combine_pmaddubsw_concat(<16 x i8> %a0, <16 x i8> %a1, <16 x
   ret <16 x i16> %3
 }
 
+define <16 x i16> @combine_pmaddubsw_concat_freeze(<16 x i8> %a0, <16 x i8> %a1) {
+; SSE-LABEL: combine_pmaddubsw_concat_freeze:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SSE-NEXT:    pmaddubsw %xmm2, %xmm0
+; SSE-NEXT:    pmaddubsw %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: combine_pmaddubsw_concat_freeze:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmaddubsw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: combine_pmaddubsw_concat_freeze:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastb {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX2-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmaddubsw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %lo = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
+  %hi = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a1, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
+  %flo = freeze <8 x i16> %lo
+  %fhi = freeze <8 x i16> %hi
+  %res = shufflevector <8 x i16> %flo, <8 x i16> %fhi, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i16> %res
+}
+
 define <8 x i16> @combine_pmaddubsw_demandedelts(<16 x i8> %a0, <16 x i8> %a1) {
 ; SSE-LABEL: combine_pmaddubsw_demandedelts:
 ; SSE:       # %bb.0:

From e9974f02a30280eab868cdab0a90a5372b1799cd Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 22 Jul 2024 14:30:46 +0100
Subject: [PATCH 411/486] [X86] canCreateUndefOrPoisonForTargetNode -
 PMADDWD/PMADDUBSW nodes don't create poison

Help with regression identified on #84924
---
 llvm/lib/Target/X86/X86ISelLowering.cpp |  3 +++
 llvm/test/CodeGen/X86/combine-pmadd.ll  | 10 ++++++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index f06b962b74df8..5a9d679d7002c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -43315,6 +43315,9 @@ bool X86TargetLowering::canCreateUndefOrPoisonForTargetNode(
     bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const {
 
   switch (Op.getOpcode()) {
+  // SSE vector multiplies are either inbounds or saturate.
+  case X86ISD::VPMADDUBSW:
+  case X86ISD::VPMADDWD:
   // SSE vector shifts handle out of bounds shift amounts.
   case X86ISD::VSHLI:
   case X86ISD::VSRLI:
diff --git a/llvm/test/CodeGen/X86/combine-pmadd.ll b/llvm/test/CodeGen/X86/combine-pmadd.ll
index 8ba5a25678122..f18497065c943 100644
--- a/llvm/test/CodeGen/X86/combine-pmadd.ll
+++ b/llvm/test/CodeGen/X86/combine-pmadd.ll
@@ -81,10 +81,11 @@ define <8 x i32> @combine_pmaddwd_concat_freeze(<8 x i16> %a0, <8 x i16> %a1) {
 ;
 ; AVX2-LABEL: combine_pmaddwd_concat_freeze:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX2-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; AVX2-NEXT:    vpmaddwd %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpmaddwd %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm2, %ymm1
+; AVX2-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
   %lo = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
   %hi = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
@@ -227,10 +228,11 @@ define <16 x i16> @combine_pmaddubsw_concat_freeze(<16 x i8> %a0, <16 x i8> %a1)
 ;
 ; AVX2-LABEL: combine_pmaddubsw_concat_freeze:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX2-NEXT:    vpbroadcastb {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX2-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpmaddubsw %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm2, %ymm1
+; AVX2-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
   %lo = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
   %hi = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a1, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)

From cb528e59de6bb16493e7e2314d2a9d0486b91152 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 22 Jul 2024 09:38:54 -0400
Subject: [PATCH 412/486] [gn] port a41a4ac78294

---
 llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
index 3005a2a8a1852..e1bb52cafa6ab 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
@@ -571,6 +571,7 @@ static_library("builtins") {
 
   if (current_cpu == "riscv" || current_cpu == "riscv64") {
     sources += [
+      "riscv/feature_bits.c",
       "riscv/fp_mode.c",
       "riscv/restore.S",
       "riscv/save.S",

From aa99192bc65071c0a008faefedd90874fbe053dd Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Mon, 22 Jul 2024 14:40:57 +0100
Subject: [PATCH 413/486] [flang][OpenMP] Add TODO messages for partially
 implemented atomic types (#99817)

There is ongoing work for fir.complex but this looks unlikely to land
before the LLVM branch. Adding a TODO message gives cleaner output to
users. Currently fir.complex leads to a compiler assertion failure.

Some uses of character types lead to invalid fir.convert type
conversions, others make it far enough to hit the same assertion failure
as for fir.complex.
---
 flang/lib/Lower/DirectivesCommon.h            | 23 +++++++++++++++
 .../Lower/OpenMP/Todo/atomic-character.f90    |  8 ++++++
 .../test/Lower/OpenMP/Todo/atomic-complex.f90 |  8 ++++++
 flang/test/Lower/OpenMP/atomic-read.f90       | 28 ++++++++-----------
 4 files changed, 51 insertions(+), 16 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/Todo/atomic-character.f90
 create mode 100644 flang/test/Lower/OpenMP/Todo/atomic-complex.f90

diff --git a/flang/lib/Lower/DirectivesCommon.h b/flang/lib/Lower/DirectivesCommon.h
index e8d6bb591e361..5b8d1f2359c5b 100644
--- a/flang/lib/Lower/DirectivesCommon.h
+++ b/flang/lib/Lower/DirectivesCommon.h
@@ -33,6 +33,7 @@
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/HLFIRTools.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/openmp-directive-sets.h"
@@ -135,6 +136,22 @@ static inline void genOmpAtomicHintAndMemoryOrderClauses(
   }
 }
 
+template <typename AtomicListT>
+static void processOmpAtomicTODO(mlir::Type elementType, mlir::Location loc) {
+  if (!elementType)
+    return;
+  if constexpr (std::is_same<AtomicListT,
+                             Fortran::parser::OmpAtomicClauseList>()) {
+    // Based on assertion for supported element types in OMPIRBuilder.cpp
+    // createAtomicRead
+    mlir::Type unwrappedEleTy = fir::unwrapRefType(elementType);
+    bool supportedAtomicType =
+        (!fir::isa_complex(unwrappedEleTy) && fir::isa_trivial(unwrappedEleTy));
+    if (!supportedAtomicType)
+      TODO(loc, "Unsupported atomic type");
+  }
+}
+
 /// Used to generate atomic.read operation which is created in existing
 /// location set by builder.
 template <typename AtomicListT>
@@ -147,6 +164,8 @@ static inline void genOmpAccAtomicCaptureStatement(
   // Generate `atomic.read` operation for atomic assigment statements
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
+  processOmpAtomicTODO<AtomicListT>(elementType, loc);
+
   if constexpr (std::is_same<AtomicListT,
                              Fortran::parser::OmpAtomicClauseList>()) {
     // If no hint clause is specified, the effect is as if
@@ -183,6 +202,8 @@ static inline void genOmpAccAtomicWriteStatement(
   mlir::Type varType = fir::unwrapRefType(lhsAddr.getType());
   rhsExpr = firOpBuilder.createConvert(loc, varType, rhsExpr);
 
+  processOmpAtomicTODO<AtomicListT>(varType, loc);
+
   if constexpr (std::is_same<AtomicListT,
                              Fortran::parser::OmpAtomicClauseList>()) {
     // If no hint clause is specified, the effect is as if
@@ -323,6 +344,8 @@ static inline void genOmpAccAtomicUpdateStatement(
         currentLocation, lhsAddr);
   }
 
+  processOmpAtomicTODO<AtomicListT>(varType, loc);
+
   llvm::SmallVector<mlir::Type> varTys = {varType};
   llvm::SmallVector<mlir::Location> locs = {currentLocation};
   firOpBuilder.createBlock(&atomicUpdateOp->getRegion(0), {}, varTys, locs);
diff --git a/flang/test/Lower/OpenMP/Todo/atomic-character.f90 b/flang/test/Lower/OpenMP/Todo/atomic-character.f90
new file mode 100644
index 0000000000000..88effa4a2a515
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/atomic-character.f90
@@ -0,0 +1,8 @@
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Unsupported atomic type
+subroutine character_atomic
+  character :: l, r
+  !$omp atomic read
+    l = r
+end subroutine
diff --git a/flang/test/Lower/OpenMP/Todo/atomic-complex.f90 b/flang/test/Lower/OpenMP/Todo/atomic-complex.f90
new file mode 100644
index 0000000000000..6d6e4399ee192
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/atomic-complex.f90
@@ -0,0 +1,8 @@
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Unsupported atomic type
+subroutine complex_atomic
+  complex :: l, r
+  !$omp atomic read
+    l = r
+end subroutine
diff --git a/flang/test/Lower/OpenMP/atomic-read.f90 b/flang/test/Lower/OpenMP/atomic-read.f90
index 8c3f37c94975e..9559df171111d 100644
--- a/flang/test/Lower/OpenMP/atomic-read.f90
+++ b/flang/test/Lower/OpenMP/atomic-read.f90
@@ -5,22 +5,18 @@
 ! This test checks the lowering of atomic read
 
 !CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomic"} {
-!CHECK:    %[[A_C1:.*]] = arith.constant 1 : index
-!CHECK:    %[[A_REF:.*]] = fir.alloca !fir.char<1> {bindc_name = "a", uniq_name = "_QFEa"}
-!CHECK:    %[[A_DECL:.*]]:2 = hlfir.declare %[[A_REF]] typeparams %[[A_C1]] {uniq_name = "_QFEa"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
-!CHECK:    %[[B_C1:.*]] = arith.constant 1 : index
-!CHECK:    %[[B_REF:.*]] = fir.alloca !fir.char<1> {bindc_name = "b", uniq_name = "_QFEb"}
-!CHECK:    %[[B_DECL:.*]]:2 = hlfir.declare %[[B_REF]] typeparams %[[B_C1]] {uniq_name = "_QFEb"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
+!CHECK:    %[[A_REF:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
+!CHECK:    %[[A_DECL:.*]]:2 = hlfir.declare %[[A_REF]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[B_REF:.*]] = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}
+!CHECK:    %[[B_DECL:.*]]:2 = hlfir.declare %[[B_REF]] {uniq_name = "_QFEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C_REF:.*]] = fir.alloca !fir.logical<4> {bindc_name = "c", uniq_name = "_QFEc"}
 !CHECK:    %[[C_DECL:.*]]:2 = hlfir.declare %[[C_REF]] {uniq_name = "_QFEc"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK:    %[[D_REF:.*]] = fir.alloca !fir.logical<4> {bindc_name = "d", uniq_name = "_QFEd"}
 !CHECK:    %[[D_DECL:.*]]:2 = hlfir.declare %[[D_REF]] {uniq_name = "_QFEd"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-!CHECK:    %[[E_C8:.*]] = arith.constant 8 : index
-!CHECK:    %[[E_REF:.*]] = fir.alloca !fir.char<1,8> {bindc_name = "e", uniq_name = "_QFEe"}
-!CHECK:    %[[E_DECL:.*]]:2 = hlfir.declare %[[E_REF]] typeparams %[[E_C8]] {uniq_name = "_QFEe"} : (!fir.ref<!fir.char<1,8>>, index) -> (!fir.ref<!fir.char<1,8>>, !fir.ref<!fir.char<1,8>>)
-!CHECK:    %[[F_C8:.*]] = arith.constant 8 : index
-!CHECK:    %[[F_REF:.*]] = fir.alloca !fir.char<1,8> {bindc_name = "f", uniq_name = "_QFEf"}
-!CHECK:    %[[F_DECL:.*]]:2 = hlfir.declare %[[F_REF]] typeparams %[[F_C8]] {uniq_name = "_QFEf"} : (!fir.ref<!fir.char<1,8>>, index) -> (!fir.ref<!fir.char<1,8>>, !fir.ref<!fir.char<1,8>>)
+!CHECK:    %[[E_REF:.*]] = fir.alloca i32 {bindc_name = "e", uniq_name = "_QFEe"}
+!CHECK:    %[[E_DECL:.*]]:2 = hlfir.declare %[[E_REF]] {uniq_name = "_QFEe"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[F_REF:.*]] = fir.alloca i32 {bindc_name = "f", uniq_name = "_QFEf"}
+!CHECK:    %[[F_DECL:.*]]:2 = hlfir.declare %[[F_REF]] {uniq_name = "_QFEf"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[G_REF:.*]] = fir.alloca f32 {bindc_name = "g", uniq_name = "_QFEg"}
 !CHECK:    %[[G_DECL:.*]]:2 = hlfir.declare %[[G_REF]] {uniq_name = "_QFEg"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:    %[[H_REF:.*]] = fir.alloca f32 {bindc_name = "h", uniq_name = "_QFEh"}
@@ -30,9 +26,9 @@
 !CHECK:    %[[Y_REF:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
 !CHECK:    %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_REF]] {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    omp.atomic.read %[[X_DECL]]#1 = %[[Y_DECL]]#1   memory_order(acquire) hint(uncontended) : !fir.ref<i32>, i32
-!CHECK:    omp.atomic.read %[[A_DECL]]#1 = %[[B_DECL]]#1   memory_order(relaxed) : !fir.ref<!fir.char<1>>, !fir.char<1>
+!CHECK:    omp.atomic.read %[[A_DECL]]#1 = %[[B_DECL]]#1   memory_order(relaxed) : !fir.ref<i32>, i32
 !CHECK:    omp.atomic.read %[[C_DECL]]#1 = %[[D_DECL]]#1   memory_order(seq_cst) hint(contended) : !fir.ref<!fir.logical<4>>, !fir.logical<4>
-!CHECK:    omp.atomic.read %[[E_DECL]]#1 = %[[F_DECL]]#1   hint(speculative) : !fir.ref<!fir.char<1,8>>, !fir.char<1,8>
+!CHECK:    omp.atomic.read %[[E_DECL]]#1 = %[[F_DECL]]#1   hint(speculative) : !fir.ref<i32>, i32
 !CHECK:    omp.atomic.read %[[G_DECL]]#1 = %[[H_DECL]]#1   hint(nonspeculative) : !fir.ref<f32>, f32
 !CHECK:    omp.atomic.read %[[G_DECL]]#1 = %[[H_DECL]]#1   : !fir.ref<f32>, f32
 
@@ -40,9 +36,9 @@ program OmpAtomic
 
     use omp_lib
     integer :: x, y
-    character :: a, b
+    integer :: a, b
     logical :: c, d
-    character(8) :: e, f
+    integer :: e, f
     real g, h
     !$omp atomic acquire read hint(omp_sync_hint_uncontended)
        x = y

From bc4c3bf1b75fec183e2616d6688aa155d6aada74 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 22 Jul 2024 08:43:17 -0500
Subject: [PATCH 414/486] [libc] Remove special case handing around test case
 that was fixed

Summary:
I needed to hack something around this for a previous backend bug, it's
since been fixed as far as I'm aware.
---
 libc/test/src/__support/blockstore_test.cpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/libc/test/src/__support/blockstore_test.cpp b/libc/test/src/__support/blockstore_test.cpp
index dd74ea18f2c02..de7bd72b96c80 100644
--- a/libc/test/src/__support/blockstore_test.cpp
+++ b/libc/test/src/__support/blockstore_test.cpp
@@ -183,11 +183,10 @@ TEST_F(LlvmLibcBlockStoreTest, PopulateAndIterateReverse10) {
   populate_and_iterate<4, 10, true>();
 }
 
-TEST_F(LlvmLibcBlockStoreTest, Back) { back_test<false>(); }
-
-// FIXME: Combing this test with the above test makes the AMDGPU backend
-// generate code which hangs. This should be fixed in the clang compiler.
-TEST_F(LlvmLibcBlockStoreTest, BackReverse) { back_test<true>(); }
+TEST_F(LlvmLibcBlockStoreTest, Back) {
+  back_test<false>();
+  back_test<true>();
+}
 
 TEST_F(LlvmLibcBlockStoreTest, Empty) {
   empty_test<false>();

From 3c459cfcaebdaf7cabac33a0e18bf6588cef4cdb Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Mon, 22 Jul 2024 15:57:22 +0200
Subject: [PATCH 415/486] [Clang] Fix handling of qualified id-expressions in
 unevaluated contexts (#99807)

In #89713, we made qualified, parenthesized id-expression ill-formed in
and address of expressions.

The expected behavior should instead be to form a pointer (rather than a
pointer to member)

The fix has been suggested by @zwuis and the tests by
@hubert-reinterpretcast.

It is worth pointing out that some of these tests seem rejected by all
compilers, however the tests do seem correct.

Fixes #89713
Fixes #40906

---------

Co-authored-by: YanzuoLiu <zwuis@outlook.com>
---
 clang/docs/ReleaseNotes.rst                   |  3 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |  3 --
 clang/lib/Sema/SemaExpr.cpp                   | 25 ++++-------
 .../CXX/expr/expr.unary/expr.unary.op/p4.cpp  | 42 +++++++++++++------
 4 files changed, 40 insertions(+), 33 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 7ac6ed934290d..61629ff30aeeb 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -58,7 +58,8 @@ C++ Specific Potentially Breaking Changes
   versions of clang. The deprecation warning for the negative spelling can be
   disabled with `-Wno-deprecated-no-relaxed-template-template-args`.
 
-- Clang now rejects pointer to member from parenthesized expression in unevaluated context such as ``decltype(&(foo::bar))``. (#GH40906).
+- Clang no longer tries to form pointer-to-members from qualified and parenthesized unevaluated expressions
+  such as ``decltype(&(foo::bar))``. (#GH40906).
 
 - Clang now performs semantic analysis for unary operators with dependent operands
   that are known to be of non-class non-enumeration type prior to instantiation.
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index dd9dc912ef4ca..c4a4e8064fac2 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -7596,9 +7596,6 @@ def err_nested_non_static_member_use : Error<
 def warn_cxx98_compat_non_static_member_use : Warning<
   "use of non-static data member %0 in an unevaluated context is "
   "incompatible with C++98">, InGroup<CXX98Compat>, DefaultIgnore;
-def err_form_ptr_to_member_from_parenthesized_expr : Error<
-  "cannot form pointer to member from a parenthesized expression; "
-  "did you mean to remove the parentheses?">;
 def err_invalid_incomplete_type_use : Error<
   "invalid use of incomplete type %0">;
 def err_builtin_func_cast_more_than_one_arg : Error<
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 8d24e34520e77..1a441d99515f4 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -14117,7 +14117,14 @@ QualType Sema::CheckAddressOfOperand(ExprResult &OrigOp, SourceLocation OpLoc) {
       // Okay: we can take the address of a field.
       // Could be a pointer to member, though, if there is an explicit
       // scope qualifier for the class.
-      if (isa<DeclRefExpr>(op) && cast<DeclRefExpr>(op)->getQualifier()) {
+
+      // [C++26] [expr.prim.id.general]
+      // If an id-expression E denotes a non-static non-type member
+      // of some class C [...] and if E is a qualified-id, E is
+      // not the un-parenthesized operand of the unary & operator [...]
+      // the id-expression is transformed into a class member access expression.
+      if (isa<DeclRefExpr>(op) && cast<DeclRefExpr>(op)->getQualifier() &&
+          !isa<ParenExpr>(OrigOp.get())) {
         DeclContext *Ctx = dcl->getDeclContext();
         if (Ctx && Ctx->isRecord()) {
           if (dcl->getType()->isReferenceType()) {
@@ -14127,22 +14134,6 @@ QualType Sema::CheckAddressOfOperand(ExprResult &OrigOp, SourceLocation OpLoc) {
             return QualType();
           }
 
-          // C++11 [expr.unary.op] p4:
-          // A pointer to member is only formed when an explicit & is used and
-          // its operand is a qualified-id not enclosed in parentheses.
-          if (isa<ParenExpr>(OrigOp.get())) {
-            SourceLocation LeftParenLoc = OrigOp.get()->getBeginLoc(),
-                           RightParenLoc = OrigOp.get()->getEndLoc();
-
-            Diag(LeftParenLoc,
-                 diag::err_form_ptr_to_member_from_parenthesized_expr)
-                << SourceRange(OpLoc, RightParenLoc)
-                << FixItHint::CreateRemoval(LeftParenLoc)
-                << FixItHint::CreateRemoval(RightParenLoc);
-
-            // Continuing might lead to better error recovery.
-          }
-
           while (cast<RecordDecl>(Ctx)->isAnonymousStructOrUnion())
             Ctx = Ctx->getParent();
 
diff --git a/clang/test/CXX/expr/expr.unary/expr.unary.op/p4.cpp b/clang/test/CXX/expr/expr.unary/expr.unary.op/p4.cpp
index 162d59439d08e..170ca0a3f1c6b 100644
--- a/clang/test/CXX/expr/expr.unary/expr.unary.op/p4.cpp
+++ b/clang/test/CXX/expr/expr.unary/expr.unary.op/p4.cpp
@@ -43,18 +43,36 @@ namespace test2 {
 }
 
 namespace GH40906 {
-  struct A {
-    int val;
-    void func() {}
-  };
+struct S {
+    int x;
+    void func();
+    static_assert(__is_same_as(decltype((S::x)), int&), "");
+    static_assert(__is_same_as(decltype(&(S::x)), int*), "");
 
-  void test() {
-    decltype(&(A::val)) ptr1; // expected-error {{cannot form pointer to member from a parenthesized expression; did you mean to remove the parentheses?}}
-    int A::* ptr2 = &(A::val); // expected-error {{invalid use of non-static data member 'val'}}
+    // FIXME: provide better error messages
+    static_assert(__is_same_as(decltype((S::func)), int&), ""); // expected-error {{call to non-static member function without an object argument}}
+    static_assert(__is_same_as(decltype(&(S::func)), int*), ""); // expected-error {{call to non-static member function without an object argument}}
+};
+static_assert(__is_same_as(decltype((S::x)), int&), "");
+static_assert(__is_same_as(decltype(&(S::x)), int*), "");
+static_assert(__is_same_as(decltype((S::func)), int&), ""); // expected-error {{call to non-static member function without an object argument}}
+static_assert(__is_same_as(decltype(&(S::func)), int*), ""); // expected-error {{call to non-static member function without an object argument}}
+
+struct A { int x;};
+
+char q(int *);
+short q(int A::*);
+
+template <typename T>
+constexpr int f(char (*)[sizeof(q(&T::x))]) { return 1; }
+
+template <typename T>
+constexpr int f(char (*)[sizeof(q(&(T::x)))]) { return 2; }
+
+constexpr int g(char (*p)[sizeof(char)] = 0) { return f<A>(p); }
+constexpr int h(char (*p)[sizeof(short)] = 0) { return f<A>(p); }
+
+static_assert(g() == 2);
+static_assert(h() == 1);
 
-    // FIXME: Error messages in these cases are less than clear, we can do
-    // better.
-    int size = sizeof(&(A::func)); // expected-error {{call to non-static member function without an object argument}}
-    void (A::* ptr3)() = &(A::func); // expected-error {{call to non-static member function without an object argument}}
-  }
 }

From 6ae4014befbc16c63c31922d8462afcaae51e260 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 22 Jul 2024 15:01:21 +0100
Subject: [PATCH 416/486] [AArch64] Add a test for stack-frame-layout analysis
 of Scalable objects. NFC

---
 .../CodeGen/AArch64/sve-stack-frame-layout.ll | 135 ++++++++++++++++++
 1 file changed, 135 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll

diff --git a/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll b/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll
new file mode 100644
index 0000000000000..67c1486b7a8c0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 | FileCheck %s --check-prefixes=CHECK
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 -pass-remarks-analysis=stack-frame-layout 2>&1 >/dev/null | FileCheck %s --check-prefixes=CHECK-FRAMELAYOUT
+
+; CHECK-FRAMELAYOUT-LABEL: Function: csr_d8_allocnxv4i32i32f64
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-8], Type: Spill, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Variable, Align: 16, Size: 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-20], Type: Variable, Align: 4, Size: 4
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Variable, Align: 8, Size: 8
+
+define i32 @csr_d8_allocnxv4i32i32f64(double %d) "aarch64_pstate_sm_compatible" {
+; CHECK-LABEL: csr_d8_allocnxv4i32i32f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str d8, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #8] // 8-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -8
+; CHECK-NEXT:    .cfi_offset b8, -16
+; CHECK-NEXT:    mov z1.s, #0 // =0x0
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    add x8, sp, #16
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    str wzr, [sp, #12]
+; CHECK-NEXT:    str d0, [sp]
+; CHECK-NEXT:    st1w { z1.s }, p0, [x8]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ldr x29, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr d8, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %a = alloca <vscale x 4 x i32>
+  %b = alloca i32
+  %c = alloca double
+  tail call void asm sideeffect "", "~{d8}"() #1
+  store <vscale x 4 x i32> zeroinitializer, ptr %a
+  store i32 zeroinitializer, ptr %b
+  store double %d, ptr %c
+  ret i32 0
+}
+
+; CHECK-FRAMELAYOUT-LABEL: Function: csr_d8_allocnxv4i32i32f64_fp
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-8], Type: Spill, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Variable, Align: 16, Size: 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-20], Type: Variable, Align: 4, Size: 4
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Spill, Align: 16, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-40], Type: Variable, Align: 8, Size: 8
+
+define i32 @csr_d8_allocnxv4i32i32f64_fp(double %d) "aarch64_pstate_sm_compatible" "frame-pointer"="all" {
+; CHECK-LABEL: csr_d8_allocnxv4i32i32f64_fp:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x29, sp, #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_offset b8, -32
+; CHECK-NEXT:    mov z1.s, #0 // =0x0
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    addvl x8, sp, #1
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    str wzr, [x8, #28]
+; CHECK-NEXT:    sub x8, x29, #16
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    str d0, [sp, #8]
+; CHECK-NEXT:    st1w { z1.s }, p0, [x8, #-1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %a = alloca <vscale x 4 x i32>
+  %b = alloca i32
+  %c = alloca double
+  tail call void asm sideeffect "", "~{d8}"() #1
+  store <vscale x 4 x i32> zeroinitializer, ptr %a
+  store i32 zeroinitializer, ptr %b
+  store double %d, ptr %c
+  ret i32 0
+}
+
+; CHECK-FRAMELAYOUT-LABEL: Function: svecc_z8_allocnxv4i32i32f64_fp
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-8], Type: Spill, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 16, Size: 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-20], Type: Variable, Align: 4, Size: 4
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Variable, Align: 16, Size: 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Variable, Align: 8, Size: 8
+
+define i32 @svecc_z8_allocnxv4i32i32f64_fp(double %d, <vscale x 4 x i32> %v) "aarch64_pstate_sm_compatible" "frame-pointer"="all" {
+; CHECK-LABEL: svecc_z8_allocnxv4i32i32f64_fp:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str z8, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    str wzr, [sp, #12]
+; CHECK-NEXT:    st1w { z1.s }, p0, [x29, #-2, mul vl]
+; CHECK-NEXT:    str d0, [sp], #16
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %a = alloca <vscale x 4 x i32>
+  %b = alloca i32
+  %c = alloca double
+  tail call void asm sideeffect "", "~{d8}"() #1
+  store <vscale x 4 x i32> %v, ptr %a
+  store i32 zeroinitializer, ptr %b
+  store double %d, ptr %c
+  ret i32 0
+}

From 8ebe7e60f50c234f5d187fcc470d92f4d2a6a925 Mon Sep 17 00:00:00 2001
From: Bjorn Pettersson <bjorn.a.pettersson@ericsson.com>
Date: Fri, 5 Apr 2024 10:57:08 +0200
Subject: [PATCH 417/486] [DAGCombiner] Push freeze through SETCC and SELECT_CC
 (#64718)

Allow pushing freeze through SETCC and SELECT_CC even if there are
multiple "maybe poison" operands. In the past we have limited it to
a single "maybe poison" operand, but it seems profitable to also
allow the multiple operand scenario.

One goal here is to avoid some regressions seen in review of
  https://github.com/llvm/llvm-project/pull/84924
when solving the select->and miscompiles described in
  https://github.com/llvm/llvm-project/issues/84653
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |   2 +
 llvm/test/CodeGen/RISCV/double-convert.ll     |   3 +-
 .../CodeGen/RISCV/double-round-conv-sat.ll    | 174 +++++++++---------
 3 files changed, 87 insertions(+), 92 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 8595f466d51e1..dff9049dacf3c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -15653,6 +15653,8 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
     return SDValue();
 
   bool AllowMultipleMaybePoisonOperands =
+      N0.getOpcode() == ISD::SELECT_CC ||
+      N0.getOpcode() == ISD::SETCC ||
       N0.getOpcode() == ISD::BUILD_VECTOR ||
       N0.getOpcode() == ISD::BUILD_PAIR ||
       N0.getOpcode() == ISD::VECTOR_SHUFFLE ||
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index 6024a29da33d2..feea4f19720b0 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -742,9 +742,8 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI12_0)(a2)
 ; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a4, 1
 ; RV32IZFINXZDINX-NEXT:    lui a3, 524288
-; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB12_2
+; RV32IZFINXZDINX-NEXT:    beqz a2, .LBB12_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1: # %start
 ; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB12_2: # %start
diff --git a/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll
index 927eee2e9e545..0839f61b2d793 100644
--- a/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll
+++ b/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll
@@ -102,30 +102,29 @@ define i64 @test_floor_si64(double %x) nounwind {
 ; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI1_0)
 ; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI1_0+4)(a2)
 ; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
-; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
+; RV32IZFINXZDINX-NEXT:    lui a4, %hi(.LCPI1_1)
+; RV32IZFINXZDINX-NEXT:    lw a5, %lo(.LCPI1_1+4)(a4)
+; RV32IZFINXZDINX-NEXT:    lw a4, %lo(.LCPI1_1)(a4)
+; RV32IZFINXZDINX-NEXT:    fle.d a6, a2, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a6
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a4, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a4, 1
 ; RV32IZFINXZDINX-NEXT:    lui a3, 524288
-; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB1_2
+; RV32IZFINXZDINX-NEXT:    beqz a6, .LBB1_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
 ; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB1_2:
-; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI1_1)
-; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI1_1)(a1)
-; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI1_1+4)(a1)
-; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
 ; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB1_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
 ; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB1_4:
-; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a5, a1
-; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    neg a2, a4
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    and a1, a2, a3
 ; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -347,30 +346,29 @@ define i64 @test_ceil_si64(double %x) nounwind {
 ; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI5_0)
 ; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI5_0+4)(a2)
 ; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI5_0)(a2)
-; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
+; RV32IZFINXZDINX-NEXT:    lui a4, %hi(.LCPI5_1)
+; RV32IZFINXZDINX-NEXT:    lw a5, %lo(.LCPI5_1+4)(a4)
+; RV32IZFINXZDINX-NEXT:    lw a4, %lo(.LCPI5_1)(a4)
+; RV32IZFINXZDINX-NEXT:    fle.d a6, a2, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a6
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a4, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a4, 1
 ; RV32IZFINXZDINX-NEXT:    lui a3, 524288
-; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB5_2
+; RV32IZFINXZDINX-NEXT:    beqz a6, .LBB5_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
 ; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB5_2:
-; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI5_1)
-; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI5_1)(a1)
-; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI5_1+4)(a1)
-; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
 ; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB5_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
 ; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB5_4:
-; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a5, a1
-; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    neg a2, a4
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    and a1, a2, a3
 ; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -592,30 +590,29 @@ define i64 @test_trunc_si64(double %x) nounwind {
 ; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI9_0)
 ; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI9_0+4)(a2)
 ; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI9_0)(a2)
-; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
+; RV32IZFINXZDINX-NEXT:    lui a4, %hi(.LCPI9_1)
+; RV32IZFINXZDINX-NEXT:    lw a5, %lo(.LCPI9_1+4)(a4)
+; RV32IZFINXZDINX-NEXT:    lw a4, %lo(.LCPI9_1)(a4)
+; RV32IZFINXZDINX-NEXT:    fle.d a6, a2, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a6
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a4, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a4, 1
 ; RV32IZFINXZDINX-NEXT:    lui a3, 524288
-; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB9_2
+; RV32IZFINXZDINX-NEXT:    beqz a6, .LBB9_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
 ; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB9_2:
-; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI9_1)
-; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI9_1)(a1)
-; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI9_1+4)(a1)
-; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
 ; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB9_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
 ; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB9_4:
-; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a5, a1
-; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    neg a2, a4
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    and a1, a2, a3
 ; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -837,30 +834,29 @@ define i64 @test_round_si64(double %x) nounwind {
 ; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI13_0)
 ; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI13_0+4)(a2)
 ; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI13_0)(a2)
-; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
+; RV32IZFINXZDINX-NEXT:    lui a4, %hi(.LCPI13_1)
+; RV32IZFINXZDINX-NEXT:    lw a5, %lo(.LCPI13_1+4)(a4)
+; RV32IZFINXZDINX-NEXT:    lw a4, %lo(.LCPI13_1)(a4)
+; RV32IZFINXZDINX-NEXT:    fle.d a6, a2, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a6
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a4, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a4, 1
 ; RV32IZFINXZDINX-NEXT:    lui a3, 524288
-; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB13_2
+; RV32IZFINXZDINX-NEXT:    beqz a6, .LBB13_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
 ; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB13_2:
-; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI13_1)
-; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI13_1)(a1)
-; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI13_1+4)(a1)
-; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
 ; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB13_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
 ; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB13_4:
-; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a5, a1
-; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    neg a2, a4
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    and a1, a2, a3
 ; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -1082,30 +1078,29 @@ define i64 @test_roundeven_si64(double %x) nounwind {
 ; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI17_0)
 ; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI17_0+4)(a2)
 ; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI17_0)(a2)
-; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
+; RV32IZFINXZDINX-NEXT:    lui a4, %hi(.LCPI17_1)
+; RV32IZFINXZDINX-NEXT:    lw a5, %lo(.LCPI17_1+4)(a4)
+; RV32IZFINXZDINX-NEXT:    lw a4, %lo(.LCPI17_1)(a4)
+; RV32IZFINXZDINX-NEXT:    fle.d a6, a2, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a6
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a4, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a4, 1
 ; RV32IZFINXZDINX-NEXT:    lui a3, 524288
-; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB17_2
+; RV32IZFINXZDINX-NEXT:    beqz a6, .LBB17_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
 ; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB17_2:
-; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI17_1)
-; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI17_1)(a1)
-; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI17_1+4)(a1)
-; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
 ; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB17_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
 ; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB17_4:
-; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a5, a1
-; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    neg a2, a4
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    and a1, a2, a3
 ; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -1327,30 +1322,29 @@ define i64 @test_rint_si64(double %x) nounwind {
 ; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI21_0)
 ; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI21_0+4)(a2)
 ; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI21_0)(a2)
-; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
+; RV32IZFINXZDINX-NEXT:    lui a4, %hi(.LCPI21_1)
+; RV32IZFINXZDINX-NEXT:    lw a5, %lo(.LCPI21_1+4)(a4)
+; RV32IZFINXZDINX-NEXT:    lw a4, %lo(.LCPI21_1)(a4)
+; RV32IZFINXZDINX-NEXT:    fle.d a6, a2, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a6
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a4, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a4, 1
 ; RV32IZFINXZDINX-NEXT:    lui a3, 524288
-; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB21_2
+; RV32IZFINXZDINX-NEXT:    beqz a6, .LBB21_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
 ; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB21_2:
-; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI21_1)
-; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI21_1)(a1)
-; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI21_1+4)(a1)
-; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
 ; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB21_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
 ; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB21_4:
-; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a5, a1
-; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    neg a2, a4
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    and a1, a2, a3
 ; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload

From 8417f490be0343ea4a183a9a9e02c6015275b291 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 22 Jul 2024 14:05:06 +0000
Subject: [PATCH 418/486] [gn build] Port 04760bfadb39

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 27f29c3344bfa..1e42aef80ee5b 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -168,6 +168,7 @@ if (current_toolchain == default_toolchain) {
       "__algorithm/ranges_find_first_of.h",
       "__algorithm/ranges_find_if.h",
       "__algorithm/ranges_find_if_not.h",
+      "__algorithm/ranges_find_last.h",
       "__algorithm/ranges_for_each.h",
       "__algorithm/ranges_for_each_n.h",
       "__algorithm/ranges_generate.h",

From 6f915f48c1362fa3273e57e4a152830e3c8dd77e Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 22 Jul 2024 14:05:07 +0000
Subject: [PATCH 419/486] [gn build] Port af0d731b1298

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 1e42aef80ee5b..cc759d2337516 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -580,6 +580,7 @@ if (current_toolchain == default_toolchain) {
       "__math/remainder.h",
       "__math/roots.h",
       "__math/rounding_functions.h",
+      "__math/special_functions.h",
       "__math/traits.h",
       "__math/trigonometric_functions.h",
       "__mbstate_t.h",

From 091ec15bbebf5be42d10f05a393489732ce651e4 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 22 Jul 2024 14:05:08 +0000
Subject: [PATCH 420/486] [gn build] Port b15aa7f88c21

---
 .../gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn
index 3e05b9b1f8c3f..357e5d91ee19e 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn
@@ -26,6 +26,7 @@ unittest("OrcJITTests") {
     "LookupAndRecordAddrsTest.cpp",
     "MachOPlatformTest.cpp",
     "MapperJITLinkMemoryManagerTest.cpp",
+    "MemoryFlagsTest.cpp",
     "MemoryMapperTest.cpp",
     "ObjectFormatsTest.cpp",
     "ObjectLinkingLayerTest.cpp",

From 32cd18975d2e1de5a783e9b1c3c21a234d5723b4 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 22 Jul 2024 16:22:01 +0200
Subject: [PATCH 421/486] [GVN] Look through select/phi when determining
 underlying object (#99509)

This addresses an optimization regression in Rust we have observed after
https://github.com/llvm/llvm-project/pull/82458. We now only perform
pointer replacement if they have the same underlying object. However,
getUnderlyingObject() by default only looks through linear chains, not
selects/phis. In particular, this means that we miss cases involving
involving pointer induction variables.

This patch fixes this by introducing a new helper
getUnderlyingObjectAggressive() which basically does what
getUnderlyingObjects() does, just specialized to the case where we must
arrive at a single underlying object in the end, and with a limit on the
number of inspected values.

Doing this more expensive underlying object check has no measurable
compile-time impact on CTMark.
---
 llvm/include/llvm/Analysis/ValueTracking.h |  4 +++
 llvm/lib/Analysis/Loads.cpp                |  5 ++-
 llvm/lib/Analysis/ValueTracking.cpp        | 42 ++++++++++++++++++++++
 llvm/test/Transforms/GVN/condprop.ll       |  6 ++--
 4 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 44e27234aa90d..8e5a8dc483d27 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -736,6 +736,10 @@ inline Value *getUnderlyingObject(Value *V, unsigned MaxLookup = 6) {
   return const_cast<Value *>(getUnderlyingObject(VConst, MaxLookup));
 }
 
+/// Like getUnderlyingObject(), but will try harder to find a single underlying
+/// object. In particular, this function also looks through selects and phis.
+const Value *getUnderlyingObjectAggressive(const Value *V);
+
 /// This method is similar to getUnderlyingObject except that it can
 /// look through phi and select instructions and return multiple objects.
 ///
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 4f0b92f51777b..61c6aa5e5a3eb 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -750,9 +750,8 @@ static bool isPointerAlwaysReplaceable(const Value *From, const Value *To,
   if (isa<Constant>(To) &&
       isDereferenceablePointer(To, Type::getInt8Ty(To->getContext()), DL))
     return true;
-  if (getUnderlyingObject(From) == getUnderlyingObject(To))
-    return true;
-  return false;
+  return getUnderlyingObjectAggressive(From) ==
+         getUnderlyingObjectAggressive(To);
 }
 
 bool llvm::canReplacePointersInUseIfEqual(const Use &U, const Value *To,
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 8f1c42b7ede44..40fe1ffe13f1b 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -6620,6 +6620,48 @@ void llvm::getUnderlyingObjects(const Value *V,
   } while (!Worklist.empty());
 }
 
+const Value *llvm::getUnderlyingObjectAggressive(const Value *V) {
+  const unsigned MaxVisited = 8;
+
+  SmallPtrSet<const Value *, 8> Visited;
+  SmallVector<const Value *, 8> Worklist;
+  Worklist.push_back(V);
+  const Value *Object = nullptr;
+  // Used as fallback if we can't find a common underlying object through
+  // recursion.
+  bool First = true;
+  const Value *FirstObject = getUnderlyingObject(V);
+  do {
+    const Value *P = Worklist.pop_back_val();
+    P = First ? FirstObject : getUnderlyingObject(P);
+    First = false;
+
+    if (!Visited.insert(P).second)
+      continue;
+
+    if (Visited.size() == MaxVisited)
+      return FirstObject;
+
+    if (auto *SI = dyn_cast<SelectInst>(P)) {
+      Worklist.push_back(SI->getTrueValue());
+      Worklist.push_back(SI->getFalseValue());
+      continue;
+    }
+
+    if (auto *PN = dyn_cast<PHINode>(P)) {
+      append_range(Worklist, PN->incoming_values());
+      continue;
+    }
+
+    if (!Object)
+      Object = P;
+    else if (Object != P)
+      return FirstObject;
+  } while (!Worklist.empty());
+
+  return Object;
+}
+
 /// This is the function that does the work of looking through basic
 /// ptrtoint+arithmetic+inttoptr sequences.
 static const Value *getUnderlyingObjectFromInt(const Value *V) {
diff --git a/llvm/test/Transforms/GVN/condprop.ll b/llvm/test/Transforms/GVN/condprop.ll
index 02f4bb97d7ebd..3babdd8173a21 100644
--- a/llvm/test/Transforms/GVN/condprop.ll
+++ b/llvm/test/Transforms/GVN/condprop.ll
@@ -813,7 +813,7 @@ define void @select_same_obj(i1 %c, ptr %p, i64 %x) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[EXIT:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
-; CHECK-NEXT:    call void @use_ptr(ptr [[P3]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -902,7 +902,7 @@ define void @phi_same_obj(i1 %c, ptr %p, i64 %x) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF2:%.*]], label [[EXIT:%.*]]
 ; CHECK:       if2:
 ; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
-; CHECK-NEXT:    call void @use_ptr(ptr [[P3]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -975,7 +975,7 @@ define void @phi_same_obj_cycle(i1 %c, ptr %p, i64 %x) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[P_IV]], [[P]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[LOOP_LATCH]]
 ; CHECK:       if:
-; CHECK-NEXT:    call void @use_ptr(ptr [[P_IV]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
 ; CHECK-NEXT:    call void @use_ptr(ptr [[P]])
 ; CHECK-NEXT:    br label [[LOOP_LATCH]]
 ; CHECK:       loop.latch:

From 83c2bfdacb0593b3a72e93098a55afdcd93d865f Mon Sep 17 00:00:00 2001
From: Samira Bazuzi <bazuzi@google.com>
Date: Mon, 22 Jul 2024 10:22:23 -0400
Subject: [PATCH 422/486] [clang][dataflow] Handle this-capturing lambdas in
 field initializers. (#99519)

We previously would assume these lambdas appeared inside a method
definition and end up crashing.
---
 .../FlowSensitive/DataflowEnvironment.cpp     | 21 ++++++++++----
 .../FlowSensitive/DataflowEnvironmentTest.cpp | 28 +++++++++++++++++++
 2 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index f734168e647bd..8d7fe18488217 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -524,12 +524,21 @@ void Environment::initialize() {
           assert(VarDecl != nullptr);
           setStorageLocation(*VarDecl, createObject(*VarDecl, nullptr));
         } else if (Capture.capturesThis()) {
-          const auto *SurroundingMethodDecl =
-              cast<CXXMethodDecl>(InitialTargetFunc->getNonClosureAncestor());
-          QualType ThisPointeeType =
-              SurroundingMethodDecl->getFunctionObjectParameterType();
-          setThisPointeeStorageLocation(
-              cast<RecordStorageLocation>(createObject(ThisPointeeType)));
+          if (auto *Ancestor = InitialTargetFunc->getNonClosureAncestor()) {
+            const auto *SurroundingMethodDecl = cast<CXXMethodDecl>(Ancestor);
+            QualType ThisPointeeType =
+                SurroundingMethodDecl->getFunctionObjectParameterType();
+            setThisPointeeStorageLocation(
+                cast<RecordStorageLocation>(createObject(ThisPointeeType)));
+          } else if (auto *FieldBeingInitialized =
+                         dyn_cast<FieldDecl>(Parent->getLambdaContextDecl())) {
+            // This is in a field initializer, rather than a method.
+            setThisPointeeStorageLocation(
+                cast<RecordStorageLocation>(createObject(QualType(
+                    FieldBeingInitialized->getParent()->getTypeForDecl(), 0))));
+          } else {
+            assert(false && "Unexpected this-capturing lambda context.");
+          }
         }
       }
     } else if (MethodDecl->isImplicitObjectMemberFunction()) {
diff --git a/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp b/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp
index a4ac597bb06d6..41fca6bf6eba2 100644
--- a/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp
@@ -473,4 +473,32 @@ TEST_F(EnvironmentTest, Stmt) {
   Env.getResultObjectLocation(*Init);
 }
 
+// This is a crash repro.
+TEST_F(EnvironmentTest, LambdaCapturingThisInFieldInitializer) {
+  using namespace ast_matchers;
+  std::string Code = R"cc(
+      struct S {
+        int f{[this]() { return 1; }()};
+      };
+    )cc";
+
+  auto Unit =
+      tooling::buildASTFromCodeWithArgs(Code, {"-fsyntax-only", "-std=c++11"});
+  auto &Context = Unit->getASTContext();
+
+  ASSERT_EQ(Context.getDiagnostics().getClient()->getNumErrors(), 0U);
+
+  auto *LambdaCallOperator = selectFirst<CXXMethodDecl>(
+      "method", match(cxxMethodDecl(hasName("operator()"),
+                                    ofClass(cxxRecordDecl(isLambda())))
+                          .bind("method"),
+                      Context));
+
+  Environment Env(DAContext, *LambdaCallOperator);
+  // Don't crash when initializing.
+  Env.initialize();
+  // And initialize the captured `this` pointee.
+  ASSERT_NE(nullptr, Env.getThisPointeeStorageLocation());
+}
+
 } // namespace

From d8e0b0d6854757c4fc0c120995cf76457c347a1e Mon Sep 17 00:00:00 2001
From: premanandrao <premanand.m.rao@intel.com>
Date: Mon, 22 Jul 2024 10:57:28 -0400
Subject: [PATCH 423/486] [clang] Diagnose use of deprecated template alias
 (#97619)

Issue a warning diagnostic when a template alias with a deprecated
attribute is used.
---
 clang/docs/ReleaseNotes.rst                   | 13 +++
 clang/lib/Sema/SemaAvailability.cpp           |  6 ++
 clang/lib/Sema/SemaTemplate.cpp               |  4 +
 .../alias-template-deprecated.cpp             | 83 +++++++++++++++++++
 4 files changed, 106 insertions(+)
 create mode 100644 clang/test/SemaTemplate/alias-template-deprecated.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 61629ff30aeeb..8e24087b3dcdb 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -744,6 +744,19 @@ Improvements to Clang's diagnostics
 
 - Clang now diagnoses dangling assignments for pointer-like objects (annotated with `[[gsl::Pointer]]`) under `-Wdangling-assignment-gsl` (off by default)
   Fixes #GH63310.
+  
+- Clang now diagnoses uses of alias templates with a deprecated attribute. (Fixes #GH18236).
+
+  .. code-block:: c++
+
+     template <typename T>
+     struct NoAttr {
+     };
+
+     template <typename T>
+     using UsingWithAttr __attribute__((deprecated)) = NoAttr<T>;
+
+     UsingWithAttr<int> objUsingWA; // warning: 'UsingWithAttr' is deprecated
 
 Improvements to Clang's time-trace
 ----------------------------------
diff --git a/clang/lib/Sema/SemaAvailability.cpp b/clang/lib/Sema/SemaAvailability.cpp
index df83bbfb7aac8..17566c226ec80 100644
--- a/clang/lib/Sema/SemaAvailability.cpp
+++ b/clang/lib/Sema/SemaAvailability.cpp
@@ -107,6 +107,12 @@ ShouldDiagnoseAvailabilityOfDecl(Sema &S, const NamedDecl *D,
     break;
   }
 
+  // For alias templates, get the underlying declaration.
+  if (const auto *ADecl = dyn_cast<TypeAliasTemplateDecl>(D)) {
+    D = ADecl->getTemplatedDecl();
+    Result = D->getAvailability(Message);
+  }
+
   // Forward class declarations get their attributes from their definition.
   if (const auto *IDecl = dyn_cast<ObjCInterfaceDecl>(D)) {
     if (IDecl->getDefinition()) {
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 9d96201625389..87b1f98bbe5ac 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -3344,6 +3344,10 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
         *this, /*PointOfInstantiation=*/TemplateLoc,
         /*Entity=*/AliasTemplate,
         /*TemplateArgs=*/TemplateArgLists.getInnermost());
+
+    // Diagnose uses of this alias.
+    (void)DiagnoseUseOfDecl(AliasTemplate, TemplateLoc);
+
     if (Inst.isInvalid())
       return QualType();
 
diff --git a/clang/test/SemaTemplate/alias-template-deprecated.cpp b/clang/test/SemaTemplate/alias-template-deprecated.cpp
new file mode 100644
index 0000000000000..7418e222bcfc5
--- /dev/null
+++ b/clang/test/SemaTemplate/alias-template-deprecated.cpp
@@ -0,0 +1,83 @@
+// RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify %s
+//
+// This test checks that a deprecated attribute on an alias
+// template triggers a warning diagnostic when it is used.
+
+template <typename T>
+struct NoAttr {
+  void foo() {}
+};
+
+// expected-note@+2 7{{'UsingWithAttr' has been explicitly marked deprecated here}}
+template <typename T>
+using UsingWithAttr __attribute__((deprecated)) = NoAttr<T>;
+
+// expected-note@+1 {{'UsingInstWithAttr' has been explicitly marked deprecated here}}
+using UsingInstWithAttr __attribute__((deprecated)) = NoAttr<int>;
+
+// expected-note@+1 {{'TDWithAttr' has been explicitly marked deprecated here}}
+typedef NoAttr<int> TDWithAttr __attribute__((deprecated));
+
+// expected-warning@+1 {{'UsingWithAttr' is deprecated}}
+typedef UsingWithAttr<int> TDUsingWithAttr;
+
+typedef NoAttr<int> TDNoAttr;
+
+// expected-note@+1 {{'UsingTDWithAttr' has been explicitly marked deprecated here}}
+using UsingTDWithAttr __attribute__((deprecated)) = TDNoAttr;
+
+struct S {
+  NoAttr<float> f1;
+  // expected-warning@+1 {{'UsingWithAttr' is deprecated}}
+  UsingWithAttr<float> f2;
+};
+
+// expected-warning@+1 {{'UsingWithAttr' is deprecated}}
+void foo(NoAttr<short> s1, UsingWithAttr<short> s2) {
+}
+
+// expected-note@+2 {{'UsingWithCPPAttr' has been explicitly marked deprecated here}}
+template <typename T>
+using UsingWithCPPAttr [[deprecated]] = NoAttr<T>;
+
+// expected-note@+1 {{'UsingInstWithCPPAttr' has been explicitly marked deprecated here}}
+using UsingInstWithCPPAttr [[deprecated("Do not use this")]] = NoAttr<int>;
+
+void bar() {
+  NoAttr<int> obj; // Okay
+
+  // expected-warning@+2 {{'UsingWithAttr' is deprecated}}
+  // expected-note@+1 {{in instantiation of template type alias 'UsingWithAttr' requested here}}
+  UsingWithAttr<int> objUsingWA;
+
+  // expected-warning@+2 {{'UsingWithAttr' is deprecated}}
+  // expected-note@+1 {{in instantiation of template type alias 'UsingWithAttr' requested here}}
+  NoAttr<UsingWithAttr<int>> s;
+
+  // expected-note@+1 {{'DepInt' has been explicitly marked deprecated here}}
+  using DepInt [[deprecated]] = int;
+  // expected-warning@+3 {{'UsingWithAttr' is deprecated}}
+  // expected-warning@+2 {{'DepInt' is deprecated}}
+  // expected-note@+1 {{in instantiation of template type alias 'UsingWithAttr' requested here}}
+  using X = UsingWithAttr<DepInt>;
+
+  // expected-warning@+2 {{'UsingWithAttr' is deprecated}}
+  // expected-note@+1 {{in instantiation of template type alias 'UsingWithAttr' requested here}}
+  UsingWithAttr<int>().foo();
+
+  // expected-warning@+1 {{'UsingInstWithAttr' is deprecated}}
+  UsingInstWithAttr objUIWA;
+
+  // expected-warning@+1 {{'TDWithAttr' is deprecated}}
+  TDWithAttr objTDWA;
+
+  // expected-warning@+1 {{'UsingTDWithAttr' is deprecated}}
+  UsingTDWithAttr objUTDWA;
+
+  // expected-warning@+2 {{'UsingWithCPPAttr' is deprecated}}
+  // expected-note@+1 {{in instantiation of template type alias 'UsingWithCPPAttr' requested here}}
+  UsingWithCPPAttr<int> objUsingWCPPA;
+
+  // expected-warning@+1 {{'UsingInstWithCPPAttr' is deprecated: Do not use this}}
+  UsingInstWithCPPAttr objUICPPWA;
+}

From 613d2c393992eee470405f1859aaf5fd1837e36c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sat, 20 Jul 2024 20:22:38 +0200
Subject: [PATCH 424/486] [clang][Interp][NFC] Avoid hitting an assertion in
 invalid code

---
 clang/lib/AST/Interp/Pointer.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp
index 229007c6d720a..3324691cdb7b7 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/Interp/Pointer.cpp
@@ -149,6 +149,10 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
   CharUnits Offset = CharUnits::Zero();
 
   auto getFieldOffset = [&](const FieldDecl *FD) -> CharUnits {
+    // This shouldn't happen, but if it does, don't crash inside
+    // getASTRecordLayout.
+    if (FD->getParent()->isInvalidDecl())
+      return CharUnits::Zero();
     const ASTRecordLayout &Layout = ASTCtx.getASTRecordLayout(FD->getParent());
     unsigned FieldIndex = FD->getFieldIndex();
     return ASTCtx.toCharUnitsFromBits(Layout.getFieldOffset(FieldIndex));

From ca3d4dfe0c8844a8010751c51600a3bd3f6f80a4 Mon Sep 17 00:00:00 2001
From: AtariDreams <gfunni234@gmail.com>
Date: Mon, 22 Jul 2024 11:02:05 -0400
Subject: [PATCH 425/486] [Metadata] Make range boundary variables unsigned
 (NFC) (#99338)

They should be unsigned because the source and target value are too.
---
 llvm/lib/IR/Metadata.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index 3aec7140510a6..ae5f5de142328 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -1287,12 +1287,12 @@ MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) {
     return A;
 
   // First, walk both lists in order of the lower boundary of each interval.
-  // At each step, try to merge the new interval to the last one we adedd.
+  // At each step, try to merge the new interval to the last one we added.
   SmallVector<ConstantInt *, 4> EndPoints;
-  int AI = 0;
-  int BI = 0;
-  int AN = A->getNumOperands() / 2;
-  int BN = B->getNumOperands() / 2;
+  unsigned AI = 0;
+  unsigned BI = 0;
+  unsigned AN = A->getNumOperands() / 2;
+  unsigned BN = B->getNumOperands() / 2;
   while (AI < AN && BI < BN) {
     ConstantInt *ALow = mdconst::extract<ConstantInt>(A->getOperand(2 * AI));
     ConstantInt *BLow = mdconst::extract<ConstantInt>(B->getOperand(2 * BI));

From 2e6558b8bcdaa4c0924f1f49a9200cb2dea44bd4 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Mon, 22 Jul 2024 16:05:17 +0100
Subject: [PATCH 426/486] [flang][OpenMP] fix lastprivate for allocatables
 (#99686)

Don't use `copyHostAssociateVar` for allocatable variables. It isn't
clear to me whether or not this should be addressed in
`copyHostAssociateVar` instead of inside OpenMP. I opted for OpenMP
to minimise how many things I effected. `copyHostAssociateVar` will
not update the destination variable if the destination variable
was unallocated. This is incorrect because assignment inside of the
openmp block can cause the allocation status of the variable to
change. Furthermore, `copyHostAssociateVar` seems to only copy the
variable address not other metadata like the size of the allocation.
Reallocation by assignment could cause this to change.
---
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp | 49 ++++++++++++++++++-
 .../Lower/OpenMP/lastprivate-allocatable.f90  | 38 ++++++++++++++
 2 files changed, 85 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/lastprivate-allocatable.f90

diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index c1d4c089df3b2..7e76a81e0df92 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -17,6 +17,7 @@
 #include "flang/Lower/SymbolMap.h"
 #include "flang/Optimizer/Builder/HLFIRTools.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Semantics/tools.h"
 
 namespace Fortran {
@@ -127,8 +128,52 @@ void DataSharingProcessor::copyFirstPrivateSymbol(
 void DataSharingProcessor::copyLastPrivateSymbol(
     const semantics::Symbol *sym,
     [[maybe_unused]] mlir::OpBuilder::InsertPoint *lastPrivIP) {
-  if (sym->test(semantics::Symbol::Flag::OmpLastPrivate))
-    converter.copyHostAssociateVar(*sym, lastPrivIP);
+  if (sym->test(semantics::Symbol::Flag::OmpLastPrivate)) {
+    bool allocatable = semantics::IsAllocatable(sym->GetUltimate());
+    if (!allocatable) {
+      converter.copyHostAssociateVar(*sym, lastPrivIP);
+      return;
+    }
+
+    // copyHostAssociateVar doesn't work properly if the privatised copy was
+    // reallocated (e.g. by assignment): it will only copy if the ultimate
+    // symbol was already allocated, and it only copies data so any reallocated
+    // lengths etc are lost
+
+    // 1) Fetch the original copy of the variable.
+    assert(sym->has<Fortran::semantics::HostAssocDetails>() &&
+           "No host-association found");
+    const Fortran::semantics::Symbol &hsym = sym->GetUltimate();
+    Fortran::lower::SymbolBox hsb = symTable->lookupOneLevelUpSymbol(hsym);
+    assert(hsb && "Host symbol box not found");
+
+    // 2) Fetch the copied one that will mask the original.
+    Fortran::lower::SymbolBox sb = symTable->shallowLookupSymbol(sym);
+    assert(sb && "Host-associated symbol box not found");
+    assert(hsb.getAddr() != sb.getAddr() &&
+           "Host and associated symbol boxes are the same");
+
+    // 3) Perform the assignment.
+    fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+    mlir::Location loc = converter.genLocation(sym->name());
+    mlir::OpBuilder::InsertPoint insPt = builder.saveInsertionPoint();
+    if (lastPrivIP && lastPrivIP->isSet())
+      builder.restoreInsertionPoint(*lastPrivIP);
+    else
+      builder.setInsertionPointAfter(sb.getAddr().getDefiningOp());
+
+    hlfir::Entity dst{hsb.getAddr()};
+    hlfir::Entity src{sb.getAddr()};
+    builder.create<hlfir::AssignOp>(
+        loc, src, dst, /*isWholeAllocatableAssignment=*/allocatable,
+        /*keepLhsLengthInAllocatableAssignment=*/false,
+        /*temporary_lhs=*/false);
+
+    if (lastPrivIP && lastPrivIP->isSet() &&
+        sym->test(Fortran::semantics::Symbol::Flag::OmpLastPrivate)) {
+      builder.restoreInsertionPoint(insPt);
+    }
+  }
 }
 
 void DataSharingProcessor::collectOmpObjectListSymbol(
diff --git a/flang/test/Lower/OpenMP/lastprivate-allocatable.f90 b/flang/test/Lower/OpenMP/lastprivate-allocatable.f90
new file mode 100644
index 0000000000000..41bbb182aade2
--- /dev/null
+++ b/flang/test/Lower/OpenMP/lastprivate-allocatable.f90
@@ -0,0 +1,38 @@
+! RUN: %flang_fc1 -emit-hlfir -o - -fopenmp %s | FileCheck %s
+! RUN: bbc -emit-hlfir -o - -fopenmp %s | FileCheck %s
+
+program lastprivate_allocatable
+  integer, allocatable :: a
+  integer :: i
+  ! a is unallocated here
+  !$omp parallel do lastprivate(a)
+  do i=1,1
+    a = 42
+  enddo
+  !$omp end parallel do
+  ! a should be allocated here
+end program
+
+! CHECK-LABEL:   func.func @_QQmain()
+! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "a", uniq_name = "_QFEa"}
+! CHECK:           %[[VAL_1:.*]] = fir.zero_bits !fir.heap<i32>
+! CHECK:           %[[VAL_2:.*]] = fir.embox %[[VAL_1]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+! CHECK:           fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+! CHECK:           omp.parallel {
+!                    create original copy of private variable
+! CHECK:             %[[VAL_16:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+! CHECK:             %[[VAL_17:.*]] = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFEi"}
+! CHECK:             %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:             omp.wsloop {
+! CHECK:               omp.loop_nest
+!                        [...]
+!                        if this is the last iteration
+! CHECK:                 fir.if %{{.*}} {
+!                          store loop IV
+! CHECK:                   fir.store %{{.*}} to %[[VAL_18]]#1 : !fir.ref<i32>
+!                          assign private variable to original copy: realloc
+! CHECK:                   hlfir.assign %[[VAL_16]]#0 to %[[VAL_3]]#0 realloc : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT:            }
+! CHECK-NEXT:            omp.yield
+! CHECK-NEXT:          }

From 7ddcf7acf22ac2b490f17d0aea4c2b7f99911d63 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Mon, 22 Jul 2024 17:06:03 +0200
Subject: [PATCH 427/486] [libc] Change fsfilcnt_t and fsblkcnt_t to be 64-bits
 long (#99876)

In 32-bit systems with 64-bit offsets, both fsfilcnt_t and fsblkcnt_t are 64-bit long, just like 64-bit systems. This patch changes both types to be 64-bit long for all platforms and follows the reasoning used to change off_t: the standard only requires it to be an unsigned int, so making it 64-bit long doesn't violate this property.

It should be NFC for 64-bit systems.
---
 libc/include/llvm-libc-types/fsblkcnt_t.h | 2 +-
 libc/include/llvm-libc-types/fsfilcnt_t.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libc/include/llvm-libc-types/fsblkcnt_t.h b/libc/include/llvm-libc-types/fsblkcnt_t.h
index 88a53d38cb1b3..8c7d3307278cd 100644
--- a/libc/include/llvm-libc-types/fsblkcnt_t.h
+++ b/libc/include/llvm-libc-types/fsblkcnt_t.h
@@ -9,6 +9,6 @@
 #ifndef LLVM_LIBC_TYPES_FSBLKCNT_T_H
 #define LLVM_LIBC_TYPES_FSBLKCNT_T_H
 
-typedef __SIZE_TYPE__ fsblkcnt_t;
+typedef __UINT64_TYPE__ fsblkcnt_t;
 
 #endif // LLVM_LIBC_TYPES_FSBLKCNT_T_H
diff --git a/libc/include/llvm-libc-types/fsfilcnt_t.h b/libc/include/llvm-libc-types/fsfilcnt_t.h
index c5693591907ab..12697830e92eb 100644
--- a/libc/include/llvm-libc-types/fsfilcnt_t.h
+++ b/libc/include/llvm-libc-types/fsfilcnt_t.h
@@ -9,6 +9,6 @@
 #ifndef LLVM_LIBC_TYPES_FSFILCNT_T_H
 #define LLVM_LIBC_TYPES_FSFILCNT_T_H
 
-typedef __SIZE_TYPE__ fsfilcnt_t;
+typedef __UINT64_TYPE__ fsfilcnt_t;
 
 #endif // LLVM_LIBC_TYPES_FSFILCNT_T_H

From 28e6095082d97c753eadbbf080fe8c36892b56f7 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Mon, 22 Jul 2024 17:06:28 +0200
Subject: [PATCH 428/486] [libc] Add working entrypoints to riscv (#99885)

Added new fsqrt entrypoints and updated headers.txt, which I missed in PR #99771
---
 libc/config/linux/riscv/entrypoints.txt | 3 +++
 libc/config/linux/riscv/headers.txt     | 7 ++++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 6c557b7a69cb6..ea08957f4ee89 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -449,6 +449,8 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fromfpx
     libc.src.math.fromfpxf
     libc.src.math.fromfpxl
+    libc.src.math.fsqrt
+    libc.src.math.fsqrtl
     libc.src.math.hypot
     libc.src.math.hypotf
     libc.src.math.ilogb
@@ -575,6 +577,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.frexpf128
     libc.src.math.fromfpf128
     libc.src.math.fromfpxf128
+    libc.src.math.fsqrtf128
     libc.src.math.ilogbf128
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
diff --git a/libc/config/linux/riscv/headers.txt b/libc/config/linux/riscv/headers.txt
index 4bb8d23ab961a..0294f62bc2f7a 100644
--- a/libc/config/linux/riscv/headers.txt
+++ b/libc/config/linux/riscv/headers.txt
@@ -2,6 +2,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.assert
     libc.include.ctype
     libc.include.dirent
+    libc.include.dlfcn
     libc.include.errno
     libc.include.fcntl
     libc.include.features
@@ -18,6 +19,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.setjmp
     libc.include.stdckdint
     libc.include.stdbit
+    libc.include.stdfix
     libc.include.stdio
     libc.include.stdlib
     libc.include.string
@@ -33,13 +35,12 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.arpa_inet
 
     libc.include.sys_auxv
-    # Disabled due to epoll_wait syscalls not being available on this platform.
-    # libc.include.sys_epoll
+    libc.include.sys_epoll
     libc.include.sys_ioctl
     libc.include.sys_mman
     libc.include.sys_prctl
-    libc.include.sys_random
     libc.include.sys_queue
+    libc.include.sys_random
     libc.include.sys_resource
     libc.include.sys_select
     libc.include.sys_socket

From 280b04f65a169f09249031e5e12a30584ce2e8e0 Mon Sep 17 00:00:00 2001
From: Utkarsh Saxena <usx@google.com>
Date: Mon, 22 Jul 2024 17:10:41 +0200
Subject: [PATCH 429/486] Record mainfile name in the Frontend time trace
 (#99866)

---
 clang/lib/Parse/ParseAST.cpp                 | 10 +++++++++-
 clang/unittests/Support/TimeProfilerTest.cpp | 14 +++++++-------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/clang/lib/Parse/ParseAST.cpp b/clang/lib/Parse/ParseAST.cpp
index 77ab3b556da58..e008cc0e38ced 100644
--- a/clang/lib/Parse/ParseAST.cpp
+++ b/clang/lib/Parse/ParseAST.cpp
@@ -152,7 +152,15 @@ void clang::ParseAST(Sema &S, bool PrintStats, bool SkipFunctionBodies) {
   bool HaveLexer = S.getPreprocessor().getCurrentLexer();
 
   if (HaveLexer) {
-    llvm::TimeTraceScope TimeScope("Frontend");
+    llvm::TimeTraceScope TimeScope("Frontend", [&]() {
+      llvm::TimeTraceMetadata M;
+      if (llvm::isTimeTraceVerbose()) {
+        const SourceManager &SM = S.getSourceManager();
+        if (const auto *FE = SM.getFileEntryForID(SM.getMainFileID()))
+          M.File = FE->tryGetRealPathName();
+      }
+      return M;
+    });
     P.Initialize();
     Parser::DeclGroupPtrTy ADecl;
     Sema::ModuleImportState ImportState;
diff --git a/clang/unittests/Support/TimeProfilerTest.cpp b/clang/unittests/Support/TimeProfilerTest.cpp
index 56d880cffde61..f53fe71d630bf 100644
--- a/clang/unittests/Support/TimeProfilerTest.cpp
+++ b/clang/unittests/Support/TimeProfilerTest.cpp
@@ -76,18 +76,18 @@ bool compileFromString(StringRef Code, StringRef Standard, StringRef File,
 }
 
 std::string GetMetadata(json::Object *Event) {
-  std::string Metadata;
-  llvm::raw_string_ostream OS(Metadata);
+  std::string M;
+  llvm::raw_string_ostream OS(M);
   if (json::Object *Args = Event->getObject("args")) {
     if (auto Detail = Args->getString("detail"))
       OS << Detail;
     // Use only filename to not include os-specific path separators.
     if (auto File = Args->getString("file"))
-      OS << ", " << llvm::sys::path::filename(*File);
+      OS << (M.empty() ? "" : ", ") << llvm::sys::path::filename(*File);
     if (auto Line = Args->getInteger("line"))
       OS << ":" << *Line;
   }
-  return Metadata;
+  return M;
 }
 
 // Returns pretty-printed trace graph.
@@ -209,7 +209,7 @@ constexpr int slow_init_list[] = {1, 1, 2, 3, 5, 8, 13, 21}; // 25th line
   ASSERT_TRUE(compileFromString(Code, "-std=c++20", "test.cc"));
   std::string Json = teardownProfiler();
   ASSERT_EQ(R"(
-Frontend
+Frontend (test.cc)
 | ParseDeclarationOrFunctionDefinition (test.cc:2:1)
 | ParseDeclarationOrFunctionDefinition (test.cc:6:1)
 | | ParseFunctionDefinition (slow_func)
@@ -266,7 +266,7 @@ TEST(TimeProfilerTest, TemplateInstantiations) {
                                 /*Headers=*/{{"a.h", A_H}, {"b.h", B_H}}));
   std::string Json = teardownProfiler();
   ASSERT_EQ(R"(
-Frontend
+Frontend (test.cc)
 | ParseFunctionDefinition (fooB)
 | ParseFunctionDefinition (fooMTA)
 | ParseFunctionDefinition (fooA)
@@ -291,7 +291,7 @@ struct {
   ASSERT_TRUE(compileFromString(Code, "-std=c99", "test.c"));
   std::string Json = teardownProfiler();
   ASSERT_EQ(R"(
-Frontend
+Frontend (test.c)
 | ParseDeclarationOrFunctionDefinition (test.c:2:1)
 | | isIntegerConstantExpr (<test.c:3:18>)
 | | EvaluateKnownConstIntCheckOverflow (<test.c:3:18>)

From 65825cd5431c64b3b2c0537e4ff01517d0b32061 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 22 Jul 2024 10:12:43 -0500
Subject: [PATCH 430/486] [libc] Use `<assert.h>` in overlay mode for
 LIBC_ASSERT (#99875)

Summary:
This uses `internal::exit` which is not built in overlay mode, leading
to linker errors. Fix this to just use `assert.h`.
---
 libc/src/__support/libc_assert.h             | 2 +-
 libc/src/__support/threads/linux/raw_mutex.h | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/libc/src/__support/libc_assert.h b/libc/src/__support/libc_assert.h
index e3235199780c2..e21a58a0c8aad 100644
--- a/libc/src/__support/libc_assert.h
+++ b/libc/src/__support/libc_assert.h
@@ -10,7 +10,7 @@
 #define LLVM_LIBC_SRC___SUPPORT_LIBC_ASSERT_H
 
 #include "src/__support/macros/config.h"
-#ifdef LIBC_COPT_USE_C_ASSERT
+#if defined(LIBC_COPT_USE_C_ASSERT) || !defined(LIBC_FULL_BUILD)
 
 // The build is configured to just use the public <assert.h> API
 // for libc's internal assertions.
diff --git a/libc/src/__support/threads/linux/raw_mutex.h b/libc/src/__support/threads/linux/raw_mutex.h
index dbf8b53b42f7e..47f0aa70f1c46 100644
--- a/libc/src/__support/threads/linux/raw_mutex.h
+++ b/libc/src/__support/threads/linux/raw_mutex.h
@@ -13,6 +13,7 @@
 #include "src/__support/libc_assert.h"
 #include "src/__support/macros/attributes.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
 #include "src/__support/threads/linux/futex_utils.h"
 #include "src/__support/threads/linux/futex_word.h"
 #include "src/__support/threads/sleep.h"

From cda5b2b4b843635f0824c228f14f747361727a5e Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Mon, 22 Jul 2024 17:18:48 +0200
Subject: [PATCH 431/486] [libc] Change fcntl cmd when only fcntl64 is
 available (#99675)

In some systems like rv32, only fcntl64 is available and it employs a different structure for file locking and the correspoding F_GETLK64, F_SETLK64, and F_SETLKW64 commands.

So if we use fcntl64, the F_GETLK, F_SETLK, and F_SETLKW commands need to be changed to their 64 versions. This patch adds new cases to the swich(cmd) in our implementation of fcntl to do that.

The default case was moved to outside the switch, so we don't need to change anything, the F_GETLK, F_SETLK, and F_SETLKW commands will just go through the old implementation.
---
 libc/src/__support/OSUtil/linux/fcntl.cpp | 32 ++++++++++++++++-------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/libc/src/__support/OSUtil/linux/fcntl.cpp b/libc/src/__support/OSUtil/linux/fcntl.cpp
index e11013121b04b..7731aa04a178d 100644
--- a/libc/src/__support/OSUtil/linux/fcntl.cpp
+++ b/libc/src/__support/OSUtil/linux/fcntl.cpp
@@ -86,17 +86,31 @@ int fcntl(int fd, int cmd, void *arg) {
     libc_errno = -ret;
     return -1;
   }
-  // The general case
-  default: {
-    int retVal = LIBC_NAMESPACE::syscall_impl<int>(
-        FCNTL_SYSCALL_ID, fd, cmd, reinterpret_cast<void *>(arg));
-    if (retVal >= 0) {
-      return retVal;
-    }
-    libc_errno = -retVal;
-    return -1;
+#ifdef SYS_fcntl64
+  case F_GETLK: {
+    if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64)
+      return fcntl(fd, F_GETLK64, arg);
+    break;
+  }
+  case F_SETLK: {
+    if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64)
+      return fcntl(fd, F_SETLK64, arg);
+    break;
+  }
+  case F_SETLKW: {
+    if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64)
+      return fcntl(fd, F_SETLKW64, arg);
+    break;
   }
+#endif
+  }
+  int retVal = LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, cmd,
+                                                 reinterpret_cast<void *>(arg));
+  if (retVal >= 0) {
+    return retVal;
   }
+  libc_errno = -retVal;
+  return -1;
 }
 
 } // namespace internal

From 2b78303e3f78c1eeb4b1362933290c490ce5fff6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Pettersson?= <bjorn.a.pettersson@ericsson.com>
Date: Mon, 22 Jul 2024 17:19:46 +0200
Subject: [PATCH 432/486] [DAGCombiner] Freeze maybe poison operands when
 folding select to logic (#84924)

Just like for regular IR we need to treat SELECT as conditionally
blocking poison in SelectionDAG. So (unless the condition itself is
poison) the result is only poison if the selected true/false value is
poison.
Thus, when doing DAG combines that turn SELECT into arithmetic/logical
operations (e.g. AND/OR) we need to make sure that the new operations
aren't more poisonous. One way to do that is to use FREEZE to make
sure the operands aren't posion.

This patch aims at fixing the kind of miscompiles reported in
  https://github.com/llvm/llvm-project/issues/84653
and
  https://github.com/llvm/llvm-project/issues/85190

Solution is to make sure that we insert FREEZE, if needed to make
the fold sound, when using the foldBoolSelectToLogic and
foldVSelectToSignBitSplatMask DAG combines.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  34 +-
 ...rleaving-reductions-predicated-scalable.ll |   3 +-
 llvm/test/CodeGen/AArch64/fast-isel-select.ll |  28 +-
 .../AArch64/intrinsic-cttz-elts-sve.ll        |   3 +-
 .../CodeGen/AArch64/sve-fp-int-min-max.ll     |   3 +-
 llvm/test/CodeGen/AMDGPU/div_i128.ll          |  64 +-
 .../AMDGPU/divergence-driven-trunc-to-i1.ll   |  41 +-
 llvm/test/CodeGen/AMDGPU/fptoi.i128.ll        |  60 +-
 llvm/test/CodeGen/AMDGPU/rem_i128.ll          |  64 +-
 llvm/test/CodeGen/NVPTX/i128.ll               | 572 +++++++++---------
 llvm/test/CodeGen/RISCV/pr84653_pr85190.ll    |   3 +-
 llvm/test/CodeGen/SystemZ/pr60413.ll          | 104 ++--
 llvm/test/CodeGen/VE/Scalar/max.ll            |   2 +
 llvm/test/CodeGen/VE/Scalar/min.ll            |   2 +
 llvm/test/CodeGen/X86/combine-pmadd.ll        |  39 +-
 .../X86/div-rem-pair-recomposition-signed.ll  | 294 +++++----
 llvm/test/CodeGen/X86/pr64589.ll              |   4 +-
 .../test/CodeGen/X86/vector-compare-all_of.ll |  34 +-
 .../test/CodeGen/X86/vector-compare-any_of.ll |  20 +-
 19 files changed, 700 insertions(+), 674 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index dff9049dacf3c..aa9032ea2574c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -11501,28 +11501,28 @@ static SDValue foldBoolSelectToLogic(SDNode *N, const SDLoc &DL,
   if (VT != Cond.getValueType() || VT.getScalarSizeInBits() != 1)
     return SDValue();
 
-  // select Cond, Cond, F --> or Cond, F
-  // select Cond, 1, F    --> or Cond, F
+  // select Cond, Cond, F --> or Cond, freeze(F)
+  // select Cond, 1, F    --> or Cond, freeze(F)
   if (Cond == T || isOneOrOneSplat(T, /* AllowUndefs */ true))
-    return matcher.getNode(ISD::OR, DL, VT, Cond, F);
+    return matcher.getNode(ISD::OR, DL, VT, Cond, DAG.getFreeze(F));
 
-  // select Cond, T, Cond --> and Cond, T
-  // select Cond, T, 0    --> and Cond, T
+  // select Cond, T, Cond --> and Cond, freeze(T)
+  // select Cond, T, 0    --> and Cond, freeze(T)
   if (Cond == F || isNullOrNullSplat(F, /* AllowUndefs */ true))
-    return matcher.getNode(ISD::AND, DL, VT, Cond, T);
+    return matcher.getNode(ISD::AND, DL, VT, Cond, DAG.getFreeze(T));
 
-  // select Cond, T, 1 --> or (not Cond), T
+  // select Cond, T, 1 --> or (not Cond), freeze(T)
   if (isOneOrOneSplat(F, /* AllowUndefs */ true)) {
     SDValue NotCond =
         matcher.getNode(ISD::XOR, DL, VT, Cond, DAG.getAllOnesConstant(DL, VT));
-    return matcher.getNode(ISD::OR, DL, VT, NotCond, T);
+    return matcher.getNode(ISD::OR, DL, VT, NotCond, DAG.getFreeze(T));
   }
 
-  // select Cond, 0, F --> and (not Cond), F
+  // select Cond, 0, F --> and (not Cond), freeze(F)
   if (isNullOrNullSplat(T, /* AllowUndefs */ true)) {
     SDValue NotCond =
         matcher.getNode(ISD::XOR, DL, VT, Cond, DAG.getAllOnesConstant(DL, VT));
-    return matcher.getNode(ISD::AND, DL, VT, NotCond, F);
+    return matcher.getNode(ISD::AND, DL, VT, NotCond, DAG.getFreeze(F));
   }
 
   return SDValue();
@@ -11550,37 +11550,37 @@ static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG) {
   else
     return SDValue();
 
-  // (Cond0 s< 0) ? N1 : 0 --> (Cond0 s>> BW-1) & N1
+  // (Cond0 s< 0) ? N1 : 0 --> (Cond0 s>> BW-1) & freeze(N1)
   if (isNullOrNullSplat(N2)) {
     SDLoc DL(N);
     SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT);
     SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt);
-    return DAG.getNode(ISD::AND, DL, VT, Sra, N1);
+    return DAG.getNode(ISD::AND, DL, VT, Sra, DAG.getFreeze(N1));
   }
 
-  // (Cond0 s< 0) ? -1 : N2 --> (Cond0 s>> BW-1) | N2
+  // (Cond0 s< 0) ? -1 : N2 --> (Cond0 s>> BW-1) | freeze(N2)
   if (isAllOnesOrAllOnesSplat(N1)) {
     SDLoc DL(N);
     SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT);
     SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt);
-    return DAG.getNode(ISD::OR, DL, VT, Sra, N2);
+    return DAG.getNode(ISD::OR, DL, VT, Sra, DAG.getFreeze(N2));
   }
 
   // If we have to invert the sign bit mask, only do that transform if the
   // target has a bitwise 'and not' instruction (the invert is free).
-  // (Cond0 s< -0) ? 0 : N2 --> ~(Cond0 s>> BW-1) & N2
+  // (Cond0 s< -0) ? 0 : N2 --> ~(Cond0 s>> BW-1) & freeze(N2)
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (isNullOrNullSplat(N1) && TLI.hasAndNot(N1)) {
     SDLoc DL(N);
     SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT);
     SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt);
     SDValue Not = DAG.getNOT(DL, Sra, VT);
-    return DAG.getNode(ISD::AND, DL, VT, Not, N2);
+    return DAG.getNode(ISD::AND, DL, VT, Not, DAG.getFreeze(N2));
   }
 
   // TODO: There's another pattern in this family, but it may require
   //       implementing hasOrNot() to check for profitability:
-  //       (Cond0 s> -1) ? -1 : N2 --> ~(Cond0 s>> BW-1) | N2
+  //       (Cond0 s> -1) ? -1 : N2 --> ~(Cond0 s>> BW-1) | freeze(N2)
 
   return SDValue();
 }
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll
index 2ef35283568c3..1c8a8d635274e 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll
@@ -229,7 +229,8 @@ define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, pt
 ; CHECK-NEXT:    mov z6.d, z1.d
 ; CHECK-NEXT:    mov z7.d, z0.d
 ; CHECK-NEXT:    add x2, x2, x11
-; CHECK-NEXT:    cmpne p1.d, p1/z, z2.d, #0
+; CHECK-NEXT:    cmpne p2.d, p0/z, z2.d, #0
+; CHECK-NEXT:    and p1.b, p1/z, p1.b, p2.b
 ; CHECK-NEXT:    zip2 p2.d, p1.d, p1.d
 ; CHECK-NEXT:    zip1 p1.d, p1.d, p1.d
 ; CHECK-NEXT:    ld1d { z2.d }, p2/z, [x0, #1, mul vl]
diff --git a/llvm/test/CodeGen/AArch64/fast-isel-select.ll b/llvm/test/CodeGen/AArch64/fast-isel-select.ll
index 315ca4f285fd7..6e55bf4968e78 100644
--- a/llvm/test/CodeGen/AArch64/fast-isel-select.ll
+++ b/llvm/test/CodeGen/AArch64/fast-isel-select.ll
@@ -500,22 +500,11 @@ define float @select_icmp_sle(i32 %x, i32 %y, float %a, float %b) {
 
 ; Test peephole optimizations for select.
 define zeroext i1 @select_opt1(i1 zeroext %c, i1 zeroext %a) {
-; CHECK-SDAGISEL-LABEL: select_opt1:
-; CHECK-SDAGISEL:       ; %bb.0:
-; CHECK-SDAGISEL-NEXT:    orr w0, w0, w1
-; CHECK-SDAGISEL-NEXT:    ret
-;
-; CHECK-FASTISEL-LABEL: select_opt1:
-; CHECK-FASTISEL:       ; %bb.0:
-; CHECK-FASTISEL-NEXT:    orr w8, w0, w1
-; CHECK-FASTISEL-NEXT:    and w0, w8, #0x1
-; CHECK-FASTISEL-NEXT:    ret
-;
-; CHECK-GISEL-LABEL: select_opt1:
-; CHECK-GISEL:       ; %bb.0:
-; CHECK-GISEL-NEXT:    orr w8, w0, w1
-; CHECK-GISEL-NEXT:    and w0, w8, #0x1
-; CHECK-GISEL-NEXT:    ret
+; CHECK-LABEL: select_opt1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    orr w8, w0, w1
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
   %1 = select i1 %c, i1 true, i1 %a
   ret i1 %1
 }
@@ -523,8 +512,8 @@ define zeroext i1 @select_opt1(i1 zeroext %c, i1 zeroext %a) {
 define zeroext i1 @select_opt2(i1 zeroext %c, i1 zeroext %a) {
 ; CHECK-SDAGISEL-LABEL: select_opt2:
 ; CHECK-SDAGISEL:       ; %bb.0:
-; CHECK-SDAGISEL-NEXT:    eor w8, w0, #0x1
-; CHECK-SDAGISEL-NEXT:    orr w0, w8, w1
+; CHECK-SDAGISEL-NEXT:    orn w8, w1, w0
+; CHECK-SDAGISEL-NEXT:    and w0, w8, #0x1
 ; CHECK-SDAGISEL-NEXT:    ret
 ;
 ; CHECK-FASTISEL-LABEL: select_opt2:
@@ -547,7 +536,8 @@ define zeroext i1 @select_opt2(i1 zeroext %c, i1 zeroext %a) {
 define zeroext i1 @select_opt3(i1 zeroext %c, i1 zeroext %a) {
 ; CHECK-SDAGISEL-LABEL: select_opt3:
 ; CHECK-SDAGISEL:       ; %bb.0:
-; CHECK-SDAGISEL-NEXT:    bic w0, w1, w0
+; CHECK-SDAGISEL-NEXT:    eor w8, w0, #0x1
+; CHECK-SDAGISEL-NEXT:    and w0, w8, w1
 ; CHECK-SDAGISEL-NEXT:    ret
 ;
 ; CHECK-FASTISEL-LABEL: select_opt3:
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
index cdf2a962f9322..66d670d0b796b 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
@@ -319,8 +319,9 @@ define i32 @ctz_nxv16i1_poison(<vscale x 16 x i1> %a) {
 define i32 @ctz_and_nxv16i1(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: ctz_and_nxv16i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, z1.b
 ; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    cmpne p2.b, p1/z, z0.b, z1.b
+; CHECK-NEXT:    and p0.b, p0/z, p0.b, p2.b
 ; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
 ; CHECK-NEXT:    cntp x0, p0, p0.b
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
diff --git a/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll b/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll
index afe13851f0b95..0d7f230062650 100644
--- a/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll
@@ -24,7 +24,8 @@ define i64 @scalable_int_min_max(ptr %arg, ptr %arg1, <vscale x 2 x ptr> %i37, <
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, z4.s
 ; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z3.s
 ; CHECK-NEXT:    add z0.d, z2.d, z1.d
-; CHECK-NEXT:    bic p2.b, p1/z, p1.b, p2.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    and p2.b, p1/z, p1.b, p2.b
 ; CHECK-NEXT:    mov z0.d, p2/m, z2.d
 ; CHECK-NEXT:    sel z0.d, p1, z0.d, z2.d
 ; CHECK-NEXT:    uaddv d0, p0, z0.d
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index f0ab3a5342e01..fea1303d0a2b7 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -482,28 +482,21 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v9
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v6
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[8:9], s[8:9]
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[8:9], s[6:7]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[5:6], s[14:15]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[5:6], s[12:13]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[8:9], s[14:15]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[8:9], s[6:7]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, v7, v10, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v7, 1, v7
@@ -514,7 +507,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v7, v7, s14
-; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v5, v5, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
@@ -1048,10 +1040,10 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
@@ -2695,28 +2687,21 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v9
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v6
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[8:9], s[8:9]
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[8:9], s[6:7]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[5:6], s[14:15]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[5:6], s[12:13]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[8:9], s[14:15]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[8:9], s[6:7]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, v7, v10, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v7, 1, v7
@@ -2727,7 +2712,6 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v7, v7, s14
-; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v5, v5, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
@@ -3261,10 +3245,10 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
index ae4d302e04a7c..5b39cc2e185b7 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
@@ -15,13 +15,16 @@ define amdgpu_kernel void @uniform_trunc_i16_to_i1(ptr addrspace(1) %out, i16 %x
   ; GCN-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
   ; GCN-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, killed [[COPY1]], %subreg.sub1, killed [[S_MOV_B32_1]], %subreg.sub2, killed [[S_MOV_B32_]], %subreg.sub3
   ; GCN-NEXT:   [[S_SEXT_I32_I16_:%[0-9]+]]:sreg_32 = S_SEXT_I32_I16 [[S_LOAD_DWORD_IMM]]
-  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 65536, [[S_LOAD_DWORD_IMM]], implicit-def dead $scc
-  ; GCN-NEXT:   S_CMP_LG_U32 killed [[S_AND_B32_]], 0, implicit-def $scc
-  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_64 = COPY $scc
-  ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GCN-NEXT:   S_CMP_LT_I32 killed [[S_SEXT_I32_I16_]], killed [[S_MOV_B32_2]], implicit-def $scc
+  ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; GCN-NEXT:   [[S_LSHR_B32_:%[0-9]+]]:sreg_32 = S_LSHR_B32 [[S_LOAD_DWORD_IMM]], killed [[S_MOV_B32_2]], implicit-def dead $scc
+  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY killed [[S_LSHR_B32_]]
+  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY3]], implicit-def dead $scc
+  ; GCN-NEXT:   S_CMP_EQ_U32 killed [[S_AND_B32_]], 1, implicit-def $scc
   ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $scc
-  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY4]], killed [[COPY3]], implicit-def dead $scc
+  ; GCN-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GCN-NEXT:   S_CMP_LT_I32 killed [[S_SEXT_I32_I16_]], killed [[S_MOV_B32_3]], implicit-def $scc
+  ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_64 = COPY $scc
+  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY5]], killed [[COPY4]], implicit-def dead $scc
   ; GCN-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[S_OR_B64_]], implicit $exec
   ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
@@ -65,15 +68,16 @@ define amdgpu_kernel void @uniform_trunc_i32_to_i1(ptr addrspace(1) %out, i32 %x
   ; GCN-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
   ; GCN-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
   ; GCN-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, killed [[COPY1]], %subreg.sub1, killed [[S_MOV_B32_1]], %subreg.sub2, killed [[S_MOV_B32_]], %subreg.sub3
-  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub0
-  ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub1
-  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY4]], implicit-def dead $scc
+  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_64 = COPY killed [[S_LOAD_DWORDX2_IMM1]]
+  ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[COPY3]].sub0
+  ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY3]].sub1
+  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY5]], implicit-def dead $scc
   ; GCN-NEXT:   S_CMP_EQ_U32 killed [[S_AND_B32_]], 1, implicit-def $scc
-  ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_64 = COPY $scc
-  ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GCN-NEXT:   S_CMP_LT_I32 killed [[COPY3]], killed [[S_MOV_B32_2]], implicit-def $scc
   ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_64 = COPY $scc
-  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY6]], killed [[COPY5]], implicit-def dead $scc
+  ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GCN-NEXT:   S_CMP_LT_I32 killed [[COPY4]], killed [[S_MOV_B32_2]], implicit-def $scc
+  ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
+  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY7]], killed [[COPY6]], implicit-def dead $scc
   ; GCN-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[S_OR_B64_]], implicit $exec
   ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
@@ -122,13 +126,14 @@ define amdgpu_kernel void @uniform_trunc_i64_to_i1(ptr addrspace(1) %out, i64 %x
   ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
   ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
   ; GCN-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY6]], %subreg.sub0, killed [[COPY5]], %subreg.sub1
-  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[S_LOAD_DWORD_IMM]], implicit-def dead $scc
+  ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY killed [[S_LOAD_DWORD_IMM]]
+  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY7]], implicit-def dead $scc
   ; GCN-NEXT:   S_CMP_EQ_U32 killed [[S_AND_B32_]], 1, implicit-def $scc
-  ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
+  ; GCN-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY $scc
   ; GCN-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
-  ; GCN-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY killed [[S_MOV_B64_]]
-  ; GCN-NEXT:   [[V_CMP_LT_I64_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_I64_e64 killed [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
-  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[V_CMP_LT_I64_e64_]], killed [[COPY7]], implicit-def dead $scc
+  ; GCN-NEXT:   [[COPY9:%[0-9]+]]:vreg_64 = COPY killed [[S_MOV_B64_]]
+  ; GCN-NEXT:   [[V_CMP_LT_I64_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_I64_e64 killed [[REG_SEQUENCE2]], [[COPY9]], implicit $exec
+  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[V_CMP_LT_I64_e64_]], killed [[COPY8]], implicit-def dead $scc
   ; GCN-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[S_OR_B64_]], implicit $exec
   ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.2, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index 99818df6175bd..667a3f398c08a 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -22,13 +22,13 @@ define i128 @fptosi_f64_to_i128(double %x) {
 ; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffffb81, v6
 ; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v7, vcc
 ; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
-; SDAG-NEXT:    s_movk_i32 s4, 0xff7f
+; SDAG-NEXT:    s_movk_i32 s6, 0xff7f
 ; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
-; SDAG-NEXT:    s_mov_b32 s5, -1
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[0:1]
-; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], -1, v[2:3]
+; SDAG-NEXT:    s_mov_b32 s7, -1
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
 ; SDAG-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[4:5]
-; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
+; SDAG-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
@@ -394,13 +394,13 @@ define i128 @fptoui_f64_to_i128(double %x) {
 ; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffffb81, v6
 ; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v7, vcc
 ; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
-; SDAG-NEXT:    s_movk_i32 s4, 0xff7f
+; SDAG-NEXT:    s_movk_i32 s6, 0xff7f
 ; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
-; SDAG-NEXT:    s_mov_b32 s5, -1
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[0:1]
-; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], -1, v[2:3]
+; SDAG-NEXT:    s_mov_b32 s7, -1
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
 ; SDAG-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[4:5]
-; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
+; SDAG-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
@@ -765,13 +765,13 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
 ; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
 ; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
-; SDAG-NEXT:    s_movk_i32 s4, 0xff7f
+; SDAG-NEXT:    s_movk_i32 s6, 0xff7f
 ; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
-; SDAG-NEXT:    s_mov_b32 s5, -1
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[0:1]
-; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], -1, v[2:3]
+; SDAG-NEXT:    s_mov_b32 s7, -1
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
 ; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v4
-; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
+; SDAG-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
@@ -1123,13 +1123,13 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
 ; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
 ; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
-; SDAG-NEXT:    s_movk_i32 s4, 0xff7f
+; SDAG-NEXT:    s_movk_i32 s6, 0xff7f
 ; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
-; SDAG-NEXT:    s_mov_b32 s5, -1
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[0:1]
-; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], -1, v[2:3]
+; SDAG-NEXT:    s_mov_b32 s7, -1
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
 ; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v4
-; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
+; SDAG-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
@@ -1509,13 +1509,13 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
 ; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
 ; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
 ; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
-; SDAG-NEXT:    s_movk_i32 s4, 0xff7f
+; SDAG-NEXT:    s_movk_i32 s6, 0xff7f
 ; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
-; SDAG-NEXT:    s_mov_b32 s5, -1
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[0:1]
-; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], -1, v[2:3]
+; SDAG-NEXT:    s_mov_b32 s7, -1
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
 ; SDAG-NEXT:    v_cmp_lt_i16_e32 vcc, -1, v4
-; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
+; SDAG-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
@@ -1860,13 +1860,13 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) {
 ; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
 ; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
 ; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
-; SDAG-NEXT:    s_movk_i32 s4, 0xff7f
+; SDAG-NEXT:    s_movk_i32 s6, 0xff7f
 ; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
-; SDAG-NEXT:    s_mov_b32 s5, -1
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[0:1]
-; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], -1, v[2:3]
+; SDAG-NEXT:    s_mov_b32 s7, -1
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
 ; SDAG-NEXT:    v_cmp_lt_i16_e32 vcc, -1, v4
-; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
+; SDAG-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index 6b036f675929e..fe093d4ac8515 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -520,28 +520,21 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v9
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v6
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[8:9], s[8:9]
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[8:9], s[6:7]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[5:6], s[14:15]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[5:6], s[12:13]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[8:9], s[14:15]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[8:9], s[6:7]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, v7, v10, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v7, 1, v7
@@ -552,7 +545,6 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v7, v7, s14
-; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v5, v5, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
@@ -1086,10 +1078,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
@@ -1918,28 +1910,21 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v9
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v6
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[8:9], s[8:9]
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[8:9], s[6:7]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[5:6], s[14:15]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[5:6], s[12:13]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[8:9], s[14:15]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[8:9], s[6:7]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, v7, v10, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v7, 1, v7
@@ -1950,7 +1935,6 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v7, v7, s14
-; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v5, v5, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
@@ -2484,10 +2468,10 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/NVPTX/i128.ll b/llvm/test/CodeGen/NVPTX/i128.ll
index 4449e4f2ea4ed..396c29512933c 100644
--- a/llvm/test/CodeGen/NVPTX/i128.ll
+++ b/llvm/test/CodeGen/NVPTX/i128.ll
@@ -7,20 +7,20 @@ define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<19>;
 ; CHECK-NEXT:    .reg .b32 %r<20>;
-; CHECK-NEXT:    .reg .b64 %rd<127>;
+; CHECK-NEXT:    .reg .b64 %rd<129>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.u64 {%rd45, %rd46}, [srem_i128_param_0];
 ; CHECK-NEXT:    ld.param.v2.u64 {%rd49, %rd50}, [srem_i128_param_1];
 ; CHECK-NEXT:    shr.s64 %rd2, %rd46, 63;
-; CHECK-NEXT:    mov.u64 %rd117, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd52, %rd117, %rd45;
-; CHECK-NEXT:    subc.cc.s64 %rd53, %rd117, %rd46;
+; CHECK-NEXT:    mov.u64 %rd119, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd52, %rd119, %rd45;
+; CHECK-NEXT:    subc.cc.s64 %rd53, %rd119, %rd46;
 ; CHECK-NEXT:    setp.lt.s64 %p1, %rd46, 0;
 ; CHECK-NEXT:    selp.b64 %rd4, %rd53, %rd46, %p1;
 ; CHECK-NEXT:    selp.b64 %rd3, %rd52, %rd45, %p1;
-; CHECK-NEXT:    sub.cc.s64 %rd54, %rd117, %rd49;
-; CHECK-NEXT:    subc.cc.s64 %rd55, %rd117, %rd50;
+; CHECK-NEXT:    sub.cc.s64 %rd54, %rd119, %rd49;
+; CHECK-NEXT:    subc.cc.s64 %rd55, %rd119, %rd50;
 ; CHECK-NEXT:    setp.lt.s64 %p2, %rd50, 0;
 ; CHECK-NEXT:    selp.b64 %rd6, %rd55, %rd50, %p2;
 ; CHECK-NEXT:    selp.b64 %rd5, %rd54, %rd49, %p2;
@@ -43,109 +43,109 @@ define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd63, %r4;
 ; CHECK-NEXT:    add.s64 %rd64, %rd63, 64;
 ; CHECK-NEXT:    selp.b64 %rd65, %rd62, %rd64, %p7;
-; CHECK-NEXT:    sub.cc.s64 %rd7, %rd61, %rd65;
-; CHECK-NEXT:    subc.cc.s64 %rd8, %rd117, 0;
-; CHECK-NEXT:    setp.eq.s64 %p8, %rd8, 0;
-; CHECK-NEXT:    setp.ne.s64 %p9, %rd8, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd66, %rd61, %rd65;
+; CHECK-NEXT:    subc.cc.s64 %rd67, %rd119, 0;
+; CHECK-NEXT:    setp.eq.s64 %p8, %rd67, 0;
+; CHECK-NEXT:    setp.ne.s64 %p9, %rd67, 0;
 ; CHECK-NEXT:    selp.u32 %r5, -1, 0, %p9;
-; CHECK-NEXT:    setp.gt.u64 %p10, %rd7, 127;
+; CHECK-NEXT:    setp.gt.u64 %p10, %rd66, 127;
 ; CHECK-NEXT:    selp.u32 %r6, -1, 0, %p10;
 ; CHECK-NEXT:    selp.b32 %r7, %r6, %r5, %p8;
 ; CHECK-NEXT:    and.b32 %r8, %r7, 1;
 ; CHECK-NEXT:    setp.eq.b32 %p11, %r8, 1;
 ; CHECK-NEXT:    or.pred %p12, %p5, %p11;
-; CHECK-NEXT:    xor.b64 %rd66, %rd7, 127;
-; CHECK-NEXT:    or.b64 %rd67, %rd66, %rd8;
-; CHECK-NEXT:    setp.eq.s64 %p13, %rd67, 0;
-; CHECK-NEXT:    selp.b64 %rd126, 0, %rd4, %p12;
-; CHECK-NEXT:    selp.b64 %rd125, 0, %rd3, %p12;
+; CHECK-NEXT:    xor.b64 %rd68, %rd66, 127;
+; CHECK-NEXT:    or.b64 %rd69, %rd68, %rd67;
+; CHECK-NEXT:    setp.eq.s64 %p13, %rd69, 0;
+; CHECK-NEXT:    selp.b64 %rd128, 0, %rd4, %p12;
+; CHECK-NEXT:    selp.b64 %rd127, 0, %rd3, %p12;
 ; CHECK-NEXT:    or.pred %p14, %p12, %p13;
 ; CHECK-NEXT:    @%p14 bra $L__BB0_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd119, %rd7, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd120, %rd8, 0;
-; CHECK-NEXT:    or.b64 %rd70, %rd119, %rd120;
-; CHECK-NEXT:    setp.eq.s64 %p15, %rd70, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd7;
+; CHECK-NEXT:    add.cc.s64 %rd121, %rd66, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd122, %rd67, 0;
+; CHECK-NEXT:    or.b64 %rd72, %rd121, %rd122;
+; CHECK-NEXT:    setp.eq.s64 %p15, %rd72, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd66;
 ; CHECK-NEXT:    mov.b32 %r10, 127;
 ; CHECK-NEXT:    sub.s32 %r11, %r10, %r9;
-; CHECK-NEXT:    shl.b64 %rd71, %rd4, %r11;
+; CHECK-NEXT:    shl.b64 %rd73, %rd4, %r11;
 ; CHECK-NEXT:    mov.b32 %r12, 64;
 ; CHECK-NEXT:    sub.s32 %r13, %r12, %r11;
-; CHECK-NEXT:    shr.u64 %rd72, %rd3, %r13;
-; CHECK-NEXT:    or.b64 %rd73, %rd71, %rd72;
+; CHECK-NEXT:    shr.u64 %rd74, %rd3, %r13;
+; CHECK-NEXT:    or.b64 %rd75, %rd73, %rd74;
 ; CHECK-NEXT:    mov.b32 %r14, 63;
 ; CHECK-NEXT:    sub.s32 %r15, %r14, %r9;
-; CHECK-NEXT:    shl.b64 %rd74, %rd3, %r15;
+; CHECK-NEXT:    shl.b64 %rd76, %rd3, %r15;
 ; CHECK-NEXT:    setp.gt.s32 %p16, %r11, 63;
-; CHECK-NEXT:    selp.b64 %rd124, %rd74, %rd73, %p16;
-; CHECK-NEXT:    shl.b64 %rd123, %rd3, %r11;
-; CHECK-NEXT:    mov.u64 %rd114, %rd117;
+; CHECK-NEXT:    selp.b64 %rd126, %rd76, %rd75, %p16;
+; CHECK-NEXT:    shl.b64 %rd125, %rd3, %r11;
+; CHECK-NEXT:    mov.u64 %rd116, %rd119;
 ; CHECK-NEXT:    @%p15 bra $L__BB0_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r16, %rd119;
-; CHECK-NEXT:    shr.u64 %rd77, %rd3, %r16;
+; CHECK-NEXT:    cvt.u32.u64 %r16, %rd121;
+; CHECK-NEXT:    shr.u64 %rd79, %rd3, %r16;
 ; CHECK-NEXT:    sub.s32 %r18, %r12, %r16;
-; CHECK-NEXT:    shl.b64 %rd78, %rd4, %r18;
-; CHECK-NEXT:    or.b64 %rd79, %rd77, %rd78;
+; CHECK-NEXT:    shl.b64 %rd80, %rd4, %r18;
+; CHECK-NEXT:    or.b64 %rd81, %rd79, %rd80;
 ; CHECK-NEXT:    add.s32 %r19, %r16, -64;
-; CHECK-NEXT:    shr.u64 %rd80, %rd4, %r19;
+; CHECK-NEXT:    shr.u64 %rd82, %rd4, %r19;
 ; CHECK-NEXT:    setp.gt.s32 %p17, %r16, 63;
-; CHECK-NEXT:    selp.b64 %rd121, %rd80, %rd79, %p17;
-; CHECK-NEXT:    shr.u64 %rd122, %rd4, %r16;
+; CHECK-NEXT:    selp.b64 %rd123, %rd82, %rd81, %p17;
+; CHECK-NEXT:    shr.u64 %rd124, %rd4, %r16;
 ; CHECK-NEXT:    add.cc.s64 %rd35, %rd5, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd36, %rd6, -1;
-; CHECK-NEXT:    mov.u64 %rd114, 0;
-; CHECK-NEXT:    mov.u64 %rd117, %rd114;
+; CHECK-NEXT:    mov.u64 %rd116, 0;
+; CHECK-NEXT:    mov.u64 %rd119, %rd116;
 ; CHECK-NEXT:  $L__BB0_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd81, %rd121, 63;
-; CHECK-NEXT:    shl.b64 %rd82, %rd122, 1;
-; CHECK-NEXT:    or.b64 %rd83, %rd82, %rd81;
-; CHECK-NEXT:    shl.b64 %rd84, %rd121, 1;
-; CHECK-NEXT:    shr.u64 %rd85, %rd124, 63;
-; CHECK-NEXT:    or.b64 %rd86, %rd84, %rd85;
-; CHECK-NEXT:    shr.u64 %rd87, %rd123, 63;
-; CHECK-NEXT:    shl.b64 %rd88, %rd124, 1;
-; CHECK-NEXT:    or.b64 %rd89, %rd88, %rd87;
-; CHECK-NEXT:    shl.b64 %rd90, %rd123, 1;
-; CHECK-NEXT:    or.b64 %rd123, %rd117, %rd90;
-; CHECK-NEXT:    or.b64 %rd124, %rd114, %rd89;
-; CHECK-NEXT:    sub.cc.s64 %rd91, %rd35, %rd86;
-; CHECK-NEXT:    subc.cc.s64 %rd92, %rd36, %rd83;
-; CHECK-NEXT:    shr.s64 %rd93, %rd92, 63;
-; CHECK-NEXT:    and.b64 %rd117, %rd93, 1;
-; CHECK-NEXT:    and.b64 %rd94, %rd93, %rd5;
-; CHECK-NEXT:    and.b64 %rd95, %rd93, %rd6;
-; CHECK-NEXT:    sub.cc.s64 %rd121, %rd86, %rd94;
-; CHECK-NEXT:    subc.cc.s64 %rd122, %rd83, %rd95;
-; CHECK-NEXT:    add.cc.s64 %rd119, %rd119, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd120, %rd120, -1;
-; CHECK-NEXT:    or.b64 %rd96, %rd119, %rd120;
-; CHECK-NEXT:    setp.eq.s64 %p18, %rd96, 0;
+; CHECK-NEXT:    shr.u64 %rd83, %rd123, 63;
+; CHECK-NEXT:    shl.b64 %rd84, %rd124, 1;
+; CHECK-NEXT:    or.b64 %rd85, %rd84, %rd83;
+; CHECK-NEXT:    shl.b64 %rd86, %rd123, 1;
+; CHECK-NEXT:    shr.u64 %rd87, %rd126, 63;
+; CHECK-NEXT:    or.b64 %rd88, %rd86, %rd87;
+; CHECK-NEXT:    shr.u64 %rd89, %rd125, 63;
+; CHECK-NEXT:    shl.b64 %rd90, %rd126, 1;
+; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
+; CHECK-NEXT:    shl.b64 %rd92, %rd125, 1;
+; CHECK-NEXT:    or.b64 %rd125, %rd119, %rd92;
+; CHECK-NEXT:    or.b64 %rd126, %rd116, %rd91;
+; CHECK-NEXT:    sub.cc.s64 %rd93, %rd35, %rd88;
+; CHECK-NEXT:    subc.cc.s64 %rd94, %rd36, %rd85;
+; CHECK-NEXT:    shr.s64 %rd95, %rd94, 63;
+; CHECK-NEXT:    and.b64 %rd119, %rd95, 1;
+; CHECK-NEXT:    and.b64 %rd96, %rd95, %rd5;
+; CHECK-NEXT:    and.b64 %rd97, %rd95, %rd6;
+; CHECK-NEXT:    sub.cc.s64 %rd123, %rd88, %rd96;
+; CHECK-NEXT:    subc.cc.s64 %rd124, %rd85, %rd97;
+; CHECK-NEXT:    add.cc.s64 %rd121, %rd121, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd122, %rd122, -1;
+; CHECK-NEXT:    or.b64 %rd98, %rd121, %rd122;
+; CHECK-NEXT:    setp.eq.s64 %p18, %rd98, 0;
 ; CHECK-NEXT:    @%p18 bra $L__BB0_4;
 ; CHECK-NEXT:    bra.uni $L__BB0_2;
 ; CHECK-NEXT:  $L__BB0_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd97, %rd123, 63;
-; CHECK-NEXT:    shl.b64 %rd98, %rd124, 1;
-; CHECK-NEXT:    or.b64 %rd99, %rd98, %rd97;
-; CHECK-NEXT:    shl.b64 %rd100, %rd123, 1;
-; CHECK-NEXT:    or.b64 %rd125, %rd117, %rd100;
-; CHECK-NEXT:    or.b64 %rd126, %rd114, %rd99;
+; CHECK-NEXT:    shr.u64 %rd99, %rd125, 63;
+; CHECK-NEXT:    shl.b64 %rd100, %rd126, 1;
+; CHECK-NEXT:    or.b64 %rd101, %rd100, %rd99;
+; CHECK-NEXT:    shl.b64 %rd102, %rd125, 1;
+; CHECK-NEXT:    or.b64 %rd127, %rd119, %rd102;
+; CHECK-NEXT:    or.b64 %rd128, %rd116, %rd101;
 ; CHECK-NEXT:  $L__BB0_5: // %udiv-end
-; CHECK-NEXT:    mul.hi.u64 %rd101, %rd5, %rd125;
-; CHECK-NEXT:    mul.lo.s64 %rd102, %rd5, %rd126;
-; CHECK-NEXT:    add.s64 %rd103, %rd101, %rd102;
-; CHECK-NEXT:    mul.lo.s64 %rd104, %rd6, %rd125;
+; CHECK-NEXT:    mul.hi.u64 %rd103, %rd5, %rd127;
+; CHECK-NEXT:    mul.lo.s64 %rd104, %rd5, %rd128;
 ; CHECK-NEXT:    add.s64 %rd105, %rd103, %rd104;
-; CHECK-NEXT:    mul.lo.s64 %rd106, %rd5, %rd125;
-; CHECK-NEXT:    sub.cc.s64 %rd107, %rd3, %rd106;
-; CHECK-NEXT:    subc.cc.s64 %rd108, %rd4, %rd105;
-; CHECK-NEXT:    xor.b64 %rd109, %rd107, %rd2;
-; CHECK-NEXT:    xor.b64 %rd110, %rd108, %rd2;
-; CHECK-NEXT:    sub.cc.s64 %rd111, %rd109, %rd2;
-; CHECK-NEXT:    subc.cc.s64 %rd112, %rd110, %rd2;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0+0], {%rd111, %rd112};
+; CHECK-NEXT:    mul.lo.s64 %rd106, %rd6, %rd127;
+; CHECK-NEXT:    add.s64 %rd107, %rd105, %rd106;
+; CHECK-NEXT:    mul.lo.s64 %rd108, %rd5, %rd127;
+; CHECK-NEXT:    sub.cc.s64 %rd109, %rd3, %rd108;
+; CHECK-NEXT:    subc.cc.s64 %rd110, %rd4, %rd107;
+; CHECK-NEXT:    xor.b64 %rd111, %rd109, %rd2;
+; CHECK-NEXT:    xor.b64 %rd112, %rd110, %rd2;
+; CHECK-NEXT:    sub.cc.s64 %rd113, %rd111, %rd2;
+; CHECK-NEXT:    subc.cc.s64 %rd114, %rd112, %rd2;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0+0], {%rd113, %rd114};
 ; CHECK-NEXT:    ret;
   %div = srem i128 %lhs, %rhs
   ret i128 %div
@@ -156,7 +156,7 @@ define i128 @urem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<17>;
 ; CHECK-NEXT:    .reg .b32 %r<20>;
-; CHECK-NEXT:    .reg .b64 %rd<113>;
+; CHECK-NEXT:    .reg .b64 %rd<115>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.u64 {%rd41, %rd42}, [urem_i128_param_0];
@@ -180,106 +180,106 @@ define i128 @urem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd52, %r4;
 ; CHECK-NEXT:    add.s64 %rd53, %rd52, 64;
 ; CHECK-NEXT:    selp.b64 %rd54, %rd51, %rd53, %p5;
-; CHECK-NEXT:    mov.u64 %rd103, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd5, %rd50, %rd54;
-; CHECK-NEXT:    subc.cc.s64 %rd6, %rd103, 0;
-; CHECK-NEXT:    setp.eq.s64 %p6, %rd6, 0;
-; CHECK-NEXT:    setp.ne.s64 %p7, %rd6, 0;
+; CHECK-NEXT:    mov.u64 %rd105, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd56, %rd50, %rd54;
+; CHECK-NEXT:    subc.cc.s64 %rd57, %rd105, 0;
+; CHECK-NEXT:    setp.eq.s64 %p6, %rd57, 0;
+; CHECK-NEXT:    setp.ne.s64 %p7, %rd57, 0;
 ; CHECK-NEXT:    selp.u32 %r5, -1, 0, %p7;
-; CHECK-NEXT:    setp.gt.u64 %p8, %rd5, 127;
+; CHECK-NEXT:    setp.gt.u64 %p8, %rd56, 127;
 ; CHECK-NEXT:    selp.u32 %r6, -1, 0, %p8;
 ; CHECK-NEXT:    selp.b32 %r7, %r6, %r5, %p6;
 ; CHECK-NEXT:    and.b32 %r8, %r7, 1;
 ; CHECK-NEXT:    setp.eq.b32 %p9, %r8, 1;
 ; CHECK-NEXT:    or.pred %p10, %p3, %p9;
-; CHECK-NEXT:    xor.b64 %rd56, %rd5, 127;
-; CHECK-NEXT:    or.b64 %rd57, %rd56, %rd6;
-; CHECK-NEXT:    setp.eq.s64 %p11, %rd57, 0;
-; CHECK-NEXT:    selp.b64 %rd112, 0, %rd42, %p10;
-; CHECK-NEXT:    selp.b64 %rd111, 0, %rd41, %p10;
+; CHECK-NEXT:    xor.b64 %rd58, %rd56, 127;
+; CHECK-NEXT:    or.b64 %rd59, %rd58, %rd57;
+; CHECK-NEXT:    setp.eq.s64 %p11, %rd59, 0;
+; CHECK-NEXT:    selp.b64 %rd114, 0, %rd42, %p10;
+; CHECK-NEXT:    selp.b64 %rd113, 0, %rd41, %p10;
 ; CHECK-NEXT:    or.pred %p12, %p10, %p11;
 ; CHECK-NEXT:    @%p12 bra $L__BB1_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd105, %rd5, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd106, %rd6, 0;
-; CHECK-NEXT:    or.b64 %rd60, %rd105, %rd106;
-; CHECK-NEXT:    setp.eq.s64 %p13, %rd60, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd5;
+; CHECK-NEXT:    add.cc.s64 %rd107, %rd56, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd108, %rd57, 0;
+; CHECK-NEXT:    or.b64 %rd62, %rd107, %rd108;
+; CHECK-NEXT:    setp.eq.s64 %p13, %rd62, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd56;
 ; CHECK-NEXT:    mov.b32 %r10, 127;
 ; CHECK-NEXT:    sub.s32 %r11, %r10, %r9;
-; CHECK-NEXT:    shl.b64 %rd61, %rd42, %r11;
+; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r11;
 ; CHECK-NEXT:    mov.b32 %r12, 64;
 ; CHECK-NEXT:    sub.s32 %r13, %r12, %r11;
-; CHECK-NEXT:    shr.u64 %rd62, %rd41, %r13;
-; CHECK-NEXT:    or.b64 %rd63, %rd61, %rd62;
+; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r13;
+; CHECK-NEXT:    or.b64 %rd65, %rd63, %rd64;
 ; CHECK-NEXT:    mov.b32 %r14, 63;
 ; CHECK-NEXT:    sub.s32 %r15, %r14, %r9;
-; CHECK-NEXT:    shl.b64 %rd64, %rd41, %r15;
+; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r15;
 ; CHECK-NEXT:    setp.gt.s32 %p14, %r11, 63;
-; CHECK-NEXT:    selp.b64 %rd110, %rd64, %rd63, %p14;
-; CHECK-NEXT:    shl.b64 %rd109, %rd41, %r11;
-; CHECK-NEXT:    mov.u64 %rd100, %rd103;
+; CHECK-NEXT:    selp.b64 %rd112, %rd66, %rd65, %p14;
+; CHECK-NEXT:    shl.b64 %rd111, %rd41, %r11;
+; CHECK-NEXT:    mov.u64 %rd102, %rd105;
 ; CHECK-NEXT:    @%p13 bra $L__BB1_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r16, %rd105;
-; CHECK-NEXT:    shr.u64 %rd67, %rd41, %r16;
+; CHECK-NEXT:    cvt.u32.u64 %r16, %rd107;
+; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r16;
 ; CHECK-NEXT:    sub.s32 %r18, %r12, %r16;
-; CHECK-NEXT:    shl.b64 %rd68, %rd42, %r18;
-; CHECK-NEXT:    or.b64 %rd69, %rd67, %rd68;
+; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r18;
+; CHECK-NEXT:    or.b64 %rd71, %rd69, %rd70;
 ; CHECK-NEXT:    add.s32 %r19, %r16, -64;
-; CHECK-NEXT:    shr.u64 %rd70, %rd42, %r19;
+; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r19;
 ; CHECK-NEXT:    setp.gt.s32 %p15, %r16, 63;
-; CHECK-NEXT:    selp.b64 %rd107, %rd70, %rd69, %p15;
-; CHECK-NEXT:    shr.u64 %rd108, %rd42, %r16;
+; CHECK-NEXT:    selp.b64 %rd109, %rd72, %rd71, %p15;
+; CHECK-NEXT:    shr.u64 %rd110, %rd42, %r16;
 ; CHECK-NEXT:    add.cc.s64 %rd33, %rd3, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd34, %rd4, -1;
-; CHECK-NEXT:    mov.u64 %rd100, 0;
-; CHECK-NEXT:    mov.u64 %rd103, %rd100;
+; CHECK-NEXT:    mov.u64 %rd102, 0;
+; CHECK-NEXT:    mov.u64 %rd105, %rd102;
 ; CHECK-NEXT:  $L__BB1_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd71, %rd107, 63;
-; CHECK-NEXT:    shl.b64 %rd72, %rd108, 1;
-; CHECK-NEXT:    or.b64 %rd73, %rd72, %rd71;
-; CHECK-NEXT:    shl.b64 %rd74, %rd107, 1;
-; CHECK-NEXT:    shr.u64 %rd75, %rd110, 63;
-; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
-; CHECK-NEXT:    shr.u64 %rd77, %rd109, 63;
-; CHECK-NEXT:    shl.b64 %rd78, %rd110, 1;
-; CHECK-NEXT:    or.b64 %rd79, %rd78, %rd77;
-; CHECK-NEXT:    shl.b64 %rd80, %rd109, 1;
-; CHECK-NEXT:    or.b64 %rd109, %rd103, %rd80;
-; CHECK-NEXT:    or.b64 %rd110, %rd100, %rd79;
-; CHECK-NEXT:    sub.cc.s64 %rd81, %rd33, %rd76;
-; CHECK-NEXT:    subc.cc.s64 %rd82, %rd34, %rd73;
-; CHECK-NEXT:    shr.s64 %rd83, %rd82, 63;
-; CHECK-NEXT:    and.b64 %rd103, %rd83, 1;
-; CHECK-NEXT:    and.b64 %rd84, %rd83, %rd3;
-; CHECK-NEXT:    and.b64 %rd85, %rd83, %rd4;
-; CHECK-NEXT:    sub.cc.s64 %rd107, %rd76, %rd84;
-; CHECK-NEXT:    subc.cc.s64 %rd108, %rd73, %rd85;
-; CHECK-NEXT:    add.cc.s64 %rd105, %rd105, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd106, %rd106, -1;
-; CHECK-NEXT:    or.b64 %rd86, %rd105, %rd106;
-; CHECK-NEXT:    setp.eq.s64 %p16, %rd86, 0;
+; CHECK-NEXT:    shr.u64 %rd73, %rd109, 63;
+; CHECK-NEXT:    shl.b64 %rd74, %rd110, 1;
+; CHECK-NEXT:    or.b64 %rd75, %rd74, %rd73;
+; CHECK-NEXT:    shl.b64 %rd76, %rd109, 1;
+; CHECK-NEXT:    shr.u64 %rd77, %rd112, 63;
+; CHECK-NEXT:    or.b64 %rd78, %rd76, %rd77;
+; CHECK-NEXT:    shr.u64 %rd79, %rd111, 63;
+; CHECK-NEXT:    shl.b64 %rd80, %rd112, 1;
+; CHECK-NEXT:    or.b64 %rd81, %rd80, %rd79;
+; CHECK-NEXT:    shl.b64 %rd82, %rd111, 1;
+; CHECK-NEXT:    or.b64 %rd111, %rd105, %rd82;
+; CHECK-NEXT:    or.b64 %rd112, %rd102, %rd81;
+; CHECK-NEXT:    sub.cc.s64 %rd83, %rd33, %rd78;
+; CHECK-NEXT:    subc.cc.s64 %rd84, %rd34, %rd75;
+; CHECK-NEXT:    shr.s64 %rd85, %rd84, 63;
+; CHECK-NEXT:    and.b64 %rd105, %rd85, 1;
+; CHECK-NEXT:    and.b64 %rd86, %rd85, %rd3;
+; CHECK-NEXT:    and.b64 %rd87, %rd85, %rd4;
+; CHECK-NEXT:    sub.cc.s64 %rd109, %rd78, %rd86;
+; CHECK-NEXT:    subc.cc.s64 %rd110, %rd75, %rd87;
+; CHECK-NEXT:    add.cc.s64 %rd107, %rd107, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd108, %rd108, -1;
+; CHECK-NEXT:    or.b64 %rd88, %rd107, %rd108;
+; CHECK-NEXT:    setp.eq.s64 %p16, %rd88, 0;
 ; CHECK-NEXT:    @%p16 bra $L__BB1_4;
 ; CHECK-NEXT:    bra.uni $L__BB1_2;
 ; CHECK-NEXT:  $L__BB1_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd87, %rd109, 63;
-; CHECK-NEXT:    shl.b64 %rd88, %rd110, 1;
-; CHECK-NEXT:    or.b64 %rd89, %rd88, %rd87;
-; CHECK-NEXT:    shl.b64 %rd90, %rd109, 1;
-; CHECK-NEXT:    or.b64 %rd111, %rd103, %rd90;
-; CHECK-NEXT:    or.b64 %rd112, %rd100, %rd89;
+; CHECK-NEXT:    shr.u64 %rd89, %rd111, 63;
+; CHECK-NEXT:    shl.b64 %rd90, %rd112, 1;
+; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
+; CHECK-NEXT:    shl.b64 %rd92, %rd111, 1;
+; CHECK-NEXT:    or.b64 %rd113, %rd105, %rd92;
+; CHECK-NEXT:    or.b64 %rd114, %rd102, %rd91;
 ; CHECK-NEXT:  $L__BB1_5: // %udiv-end
-; CHECK-NEXT:    mul.hi.u64 %rd91, %rd3, %rd111;
-; CHECK-NEXT:    mul.lo.s64 %rd92, %rd3, %rd112;
-; CHECK-NEXT:    add.s64 %rd93, %rd91, %rd92;
-; CHECK-NEXT:    mul.lo.s64 %rd94, %rd4, %rd111;
+; CHECK-NEXT:    mul.hi.u64 %rd93, %rd3, %rd113;
+; CHECK-NEXT:    mul.lo.s64 %rd94, %rd3, %rd114;
 ; CHECK-NEXT:    add.s64 %rd95, %rd93, %rd94;
-; CHECK-NEXT:    mul.lo.s64 %rd96, %rd3, %rd111;
-; CHECK-NEXT:    sub.cc.s64 %rd97, %rd41, %rd96;
-; CHECK-NEXT:    subc.cc.s64 %rd98, %rd42, %rd95;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0+0], {%rd97, %rd98};
+; CHECK-NEXT:    mul.lo.s64 %rd96, %rd4, %rd113;
+; CHECK-NEXT:    add.s64 %rd97, %rd95, %rd96;
+; CHECK-NEXT:    mul.lo.s64 %rd98, %rd3, %rd113;
+; CHECK-NEXT:    sub.cc.s64 %rd99, %rd41, %rd98;
+; CHECK-NEXT:    subc.cc.s64 %rd100, %rd42, %rd97;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0+0], {%rd99, %rd100};
 ; CHECK-NEXT:    ret;
   %div = urem i128 %lhs, %rhs
   ret i128 %div
@@ -325,19 +325,19 @@ define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<19>;
 ; CHECK-NEXT:    .reg .b32 %r<20>;
-; CHECK-NEXT:    .reg .b64 %rd<120>;
+; CHECK-NEXT:    .reg .b64 %rd<122>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.u64 {%rd45, %rd46}, [sdiv_i128_param_0];
 ; CHECK-NEXT:    ld.param.v2.u64 {%rd49, %rd50}, [sdiv_i128_param_1];
-; CHECK-NEXT:    mov.u64 %rd110, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd52, %rd110, %rd45;
-; CHECK-NEXT:    subc.cc.s64 %rd53, %rd110, %rd46;
+; CHECK-NEXT:    mov.u64 %rd112, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd52, %rd112, %rd45;
+; CHECK-NEXT:    subc.cc.s64 %rd53, %rd112, %rd46;
 ; CHECK-NEXT:    setp.lt.s64 %p1, %rd46, 0;
 ; CHECK-NEXT:    selp.b64 %rd2, %rd53, %rd46, %p1;
 ; CHECK-NEXT:    selp.b64 %rd1, %rd52, %rd45, %p1;
-; CHECK-NEXT:    sub.cc.s64 %rd54, %rd110, %rd49;
-; CHECK-NEXT:    subc.cc.s64 %rd55, %rd110, %rd50;
+; CHECK-NEXT:    sub.cc.s64 %rd54, %rd112, %rd49;
+; CHECK-NEXT:    subc.cc.s64 %rd55, %rd112, %rd50;
 ; CHECK-NEXT:    setp.lt.s64 %p2, %rd50, 0;
 ; CHECK-NEXT:    selp.b64 %rd4, %rd55, %rd50, %p2;
 ; CHECK-NEXT:    selp.b64 %rd3, %rd54, %rd49, %p2;
@@ -362,101 +362,101 @@ define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd64, %r4;
 ; CHECK-NEXT:    add.s64 %rd65, %rd64, 64;
 ; CHECK-NEXT:    selp.b64 %rd66, %rd63, %rd65, %p7;
-; CHECK-NEXT:    sub.cc.s64 %rd7, %rd62, %rd66;
-; CHECK-NEXT:    subc.cc.s64 %rd8, %rd110, 0;
-; CHECK-NEXT:    setp.eq.s64 %p8, %rd8, 0;
-; CHECK-NEXT:    setp.ne.s64 %p9, %rd8, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd67, %rd62, %rd66;
+; CHECK-NEXT:    subc.cc.s64 %rd68, %rd112, 0;
+; CHECK-NEXT:    setp.eq.s64 %p8, %rd68, 0;
+; CHECK-NEXT:    setp.ne.s64 %p9, %rd68, 0;
 ; CHECK-NEXT:    selp.u32 %r5, -1, 0, %p9;
-; CHECK-NEXT:    setp.gt.u64 %p10, %rd7, 127;
+; CHECK-NEXT:    setp.gt.u64 %p10, %rd67, 127;
 ; CHECK-NEXT:    selp.u32 %r6, -1, 0, %p10;
 ; CHECK-NEXT:    selp.b32 %r7, %r6, %r5, %p8;
 ; CHECK-NEXT:    and.b32 %r8, %r7, 1;
 ; CHECK-NEXT:    setp.eq.b32 %p11, %r8, 1;
 ; CHECK-NEXT:    or.pred %p12, %p5, %p11;
-; CHECK-NEXT:    xor.b64 %rd67, %rd7, 127;
-; CHECK-NEXT:    or.b64 %rd68, %rd67, %rd8;
-; CHECK-NEXT:    setp.eq.s64 %p13, %rd68, 0;
-; CHECK-NEXT:    selp.b64 %rd119, 0, %rd2, %p12;
-; CHECK-NEXT:    selp.b64 %rd118, 0, %rd1, %p12;
+; CHECK-NEXT:    xor.b64 %rd69, %rd67, 127;
+; CHECK-NEXT:    or.b64 %rd70, %rd69, %rd68;
+; CHECK-NEXT:    setp.eq.s64 %p13, %rd70, 0;
+; CHECK-NEXT:    selp.b64 %rd121, 0, %rd2, %p12;
+; CHECK-NEXT:    selp.b64 %rd120, 0, %rd1, %p12;
 ; CHECK-NEXT:    or.pred %p14, %p12, %p13;
 ; CHECK-NEXT:    @%p14 bra $L__BB4_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd112, %rd7, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd113, %rd8, 0;
-; CHECK-NEXT:    or.b64 %rd71, %rd112, %rd113;
-; CHECK-NEXT:    setp.eq.s64 %p15, %rd71, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd7;
+; CHECK-NEXT:    add.cc.s64 %rd114, %rd67, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd115, %rd68, 0;
+; CHECK-NEXT:    or.b64 %rd73, %rd114, %rd115;
+; CHECK-NEXT:    setp.eq.s64 %p15, %rd73, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd67;
 ; CHECK-NEXT:    mov.b32 %r10, 127;
 ; CHECK-NEXT:    sub.s32 %r11, %r10, %r9;
-; CHECK-NEXT:    shl.b64 %rd72, %rd2, %r11;
+; CHECK-NEXT:    shl.b64 %rd74, %rd2, %r11;
 ; CHECK-NEXT:    mov.b32 %r12, 64;
 ; CHECK-NEXT:    sub.s32 %r13, %r12, %r11;
-; CHECK-NEXT:    shr.u64 %rd73, %rd1, %r13;
-; CHECK-NEXT:    or.b64 %rd74, %rd72, %rd73;
+; CHECK-NEXT:    shr.u64 %rd75, %rd1, %r13;
+; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
 ; CHECK-NEXT:    mov.b32 %r14, 63;
 ; CHECK-NEXT:    sub.s32 %r15, %r14, %r9;
-; CHECK-NEXT:    shl.b64 %rd75, %rd1, %r15;
+; CHECK-NEXT:    shl.b64 %rd77, %rd1, %r15;
 ; CHECK-NEXT:    setp.gt.s32 %p16, %r11, 63;
-; CHECK-NEXT:    selp.b64 %rd117, %rd75, %rd74, %p16;
-; CHECK-NEXT:    shl.b64 %rd116, %rd1, %r11;
-; CHECK-NEXT:    mov.u64 %rd107, %rd110;
+; CHECK-NEXT:    selp.b64 %rd119, %rd77, %rd76, %p16;
+; CHECK-NEXT:    shl.b64 %rd118, %rd1, %r11;
+; CHECK-NEXT:    mov.u64 %rd109, %rd112;
 ; CHECK-NEXT:    @%p15 bra $L__BB4_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r16, %rd112;
-; CHECK-NEXT:    shr.u64 %rd78, %rd1, %r16;
+; CHECK-NEXT:    cvt.u32.u64 %r16, %rd114;
+; CHECK-NEXT:    shr.u64 %rd80, %rd1, %r16;
 ; CHECK-NEXT:    sub.s32 %r18, %r12, %r16;
-; CHECK-NEXT:    shl.b64 %rd79, %rd2, %r18;
-; CHECK-NEXT:    or.b64 %rd80, %rd78, %rd79;
+; CHECK-NEXT:    shl.b64 %rd81, %rd2, %r18;
+; CHECK-NEXT:    or.b64 %rd82, %rd80, %rd81;
 ; CHECK-NEXT:    add.s32 %r19, %r16, -64;
-; CHECK-NEXT:    shr.u64 %rd81, %rd2, %r19;
+; CHECK-NEXT:    shr.u64 %rd83, %rd2, %r19;
 ; CHECK-NEXT:    setp.gt.s32 %p17, %r16, 63;
-; CHECK-NEXT:    selp.b64 %rd114, %rd81, %rd80, %p17;
-; CHECK-NEXT:    shr.u64 %rd115, %rd2, %r16;
+; CHECK-NEXT:    selp.b64 %rd116, %rd83, %rd82, %p17;
+; CHECK-NEXT:    shr.u64 %rd117, %rd2, %r16;
 ; CHECK-NEXT:    add.cc.s64 %rd35, %rd3, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd36, %rd4, -1;
-; CHECK-NEXT:    mov.u64 %rd107, 0;
-; CHECK-NEXT:    mov.u64 %rd110, %rd107;
+; CHECK-NEXT:    mov.u64 %rd109, 0;
+; CHECK-NEXT:    mov.u64 %rd112, %rd109;
 ; CHECK-NEXT:  $L__BB4_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd82, %rd114, 63;
-; CHECK-NEXT:    shl.b64 %rd83, %rd115, 1;
-; CHECK-NEXT:    or.b64 %rd84, %rd83, %rd82;
-; CHECK-NEXT:    shl.b64 %rd85, %rd114, 1;
-; CHECK-NEXT:    shr.u64 %rd86, %rd117, 63;
-; CHECK-NEXT:    or.b64 %rd87, %rd85, %rd86;
-; CHECK-NEXT:    shr.u64 %rd88, %rd116, 63;
-; CHECK-NEXT:    shl.b64 %rd89, %rd117, 1;
-; CHECK-NEXT:    or.b64 %rd90, %rd89, %rd88;
-; CHECK-NEXT:    shl.b64 %rd91, %rd116, 1;
-; CHECK-NEXT:    or.b64 %rd116, %rd110, %rd91;
-; CHECK-NEXT:    or.b64 %rd117, %rd107, %rd90;
-; CHECK-NEXT:    sub.cc.s64 %rd92, %rd35, %rd87;
-; CHECK-NEXT:    subc.cc.s64 %rd93, %rd36, %rd84;
-; CHECK-NEXT:    shr.s64 %rd94, %rd93, 63;
-; CHECK-NEXT:    and.b64 %rd110, %rd94, 1;
-; CHECK-NEXT:    and.b64 %rd95, %rd94, %rd3;
-; CHECK-NEXT:    and.b64 %rd96, %rd94, %rd4;
-; CHECK-NEXT:    sub.cc.s64 %rd114, %rd87, %rd95;
-; CHECK-NEXT:    subc.cc.s64 %rd115, %rd84, %rd96;
-; CHECK-NEXT:    add.cc.s64 %rd112, %rd112, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd113, %rd113, -1;
-; CHECK-NEXT:    or.b64 %rd97, %rd112, %rd113;
-; CHECK-NEXT:    setp.eq.s64 %p18, %rd97, 0;
+; CHECK-NEXT:    shr.u64 %rd84, %rd116, 63;
+; CHECK-NEXT:    shl.b64 %rd85, %rd117, 1;
+; CHECK-NEXT:    or.b64 %rd86, %rd85, %rd84;
+; CHECK-NEXT:    shl.b64 %rd87, %rd116, 1;
+; CHECK-NEXT:    shr.u64 %rd88, %rd119, 63;
+; CHECK-NEXT:    or.b64 %rd89, %rd87, %rd88;
+; CHECK-NEXT:    shr.u64 %rd90, %rd118, 63;
+; CHECK-NEXT:    shl.b64 %rd91, %rd119, 1;
+; CHECK-NEXT:    or.b64 %rd92, %rd91, %rd90;
+; CHECK-NEXT:    shl.b64 %rd93, %rd118, 1;
+; CHECK-NEXT:    or.b64 %rd118, %rd112, %rd93;
+; CHECK-NEXT:    or.b64 %rd119, %rd109, %rd92;
+; CHECK-NEXT:    sub.cc.s64 %rd94, %rd35, %rd89;
+; CHECK-NEXT:    subc.cc.s64 %rd95, %rd36, %rd86;
+; CHECK-NEXT:    shr.s64 %rd96, %rd95, 63;
+; CHECK-NEXT:    and.b64 %rd112, %rd96, 1;
+; CHECK-NEXT:    and.b64 %rd97, %rd96, %rd3;
+; CHECK-NEXT:    and.b64 %rd98, %rd96, %rd4;
+; CHECK-NEXT:    sub.cc.s64 %rd116, %rd89, %rd97;
+; CHECK-NEXT:    subc.cc.s64 %rd117, %rd86, %rd98;
+; CHECK-NEXT:    add.cc.s64 %rd114, %rd114, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd115, %rd115, -1;
+; CHECK-NEXT:    or.b64 %rd99, %rd114, %rd115;
+; CHECK-NEXT:    setp.eq.s64 %p18, %rd99, 0;
 ; CHECK-NEXT:    @%p18 bra $L__BB4_4;
 ; CHECK-NEXT:    bra.uni $L__BB4_2;
 ; CHECK-NEXT:  $L__BB4_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd98, %rd116, 63;
-; CHECK-NEXT:    shl.b64 %rd99, %rd117, 1;
-; CHECK-NEXT:    or.b64 %rd100, %rd99, %rd98;
-; CHECK-NEXT:    shl.b64 %rd101, %rd116, 1;
-; CHECK-NEXT:    or.b64 %rd118, %rd110, %rd101;
-; CHECK-NEXT:    or.b64 %rd119, %rd107, %rd100;
+; CHECK-NEXT:    shr.u64 %rd100, %rd118, 63;
+; CHECK-NEXT:    shl.b64 %rd101, %rd119, 1;
+; CHECK-NEXT:    or.b64 %rd102, %rd101, %rd100;
+; CHECK-NEXT:    shl.b64 %rd103, %rd118, 1;
+; CHECK-NEXT:    or.b64 %rd120, %rd112, %rd103;
+; CHECK-NEXT:    or.b64 %rd121, %rd109, %rd102;
 ; CHECK-NEXT:  $L__BB4_5: // %udiv-end
-; CHECK-NEXT:    xor.b64 %rd102, %rd118, %rd5;
-; CHECK-NEXT:    xor.b64 %rd103, %rd119, %rd5;
-; CHECK-NEXT:    sub.cc.s64 %rd104, %rd102, %rd5;
-; CHECK-NEXT:    subc.cc.s64 %rd105, %rd103, %rd5;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0+0], {%rd104, %rd105};
+; CHECK-NEXT:    xor.b64 %rd104, %rd120, %rd5;
+; CHECK-NEXT:    xor.b64 %rd105, %rd121, %rd5;
+; CHECK-NEXT:    sub.cc.s64 %rd106, %rd104, %rd5;
+; CHECK-NEXT:    subc.cc.s64 %rd107, %rd105, %rd5;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0+0], {%rd106, %rd107};
 ; CHECK-NEXT:    ret;
   %div = sdiv i128 %lhs, %rhs
   ret i128 %div
@@ -467,7 +467,7 @@ define i128 @udiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<17>;
 ; CHECK-NEXT:    .reg .b32 %r<20>;
-; CHECK-NEXT:    .reg .b64 %rd<105>;
+; CHECK-NEXT:    .reg .b64 %rd<107>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.u64 {%rd41, %rd42}, [udiv_i128_param_0];
@@ -491,98 +491,98 @@ define i128 @udiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd52, %r4;
 ; CHECK-NEXT:    add.s64 %rd53, %rd52, 64;
 ; CHECK-NEXT:    selp.b64 %rd54, %rd51, %rd53, %p5;
-; CHECK-NEXT:    mov.u64 %rd95, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd5, %rd50, %rd54;
-; CHECK-NEXT:    subc.cc.s64 %rd6, %rd95, 0;
-; CHECK-NEXT:    setp.eq.s64 %p6, %rd6, 0;
-; CHECK-NEXT:    setp.ne.s64 %p7, %rd6, 0;
+; CHECK-NEXT:    mov.u64 %rd97, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd56, %rd50, %rd54;
+; CHECK-NEXT:    subc.cc.s64 %rd57, %rd97, 0;
+; CHECK-NEXT:    setp.eq.s64 %p6, %rd57, 0;
+; CHECK-NEXT:    setp.ne.s64 %p7, %rd57, 0;
 ; CHECK-NEXT:    selp.u32 %r5, -1, 0, %p7;
-; CHECK-NEXT:    setp.gt.u64 %p8, %rd5, 127;
+; CHECK-NEXT:    setp.gt.u64 %p8, %rd56, 127;
 ; CHECK-NEXT:    selp.u32 %r6, -1, 0, %p8;
 ; CHECK-NEXT:    selp.b32 %r7, %r6, %r5, %p6;
 ; CHECK-NEXT:    and.b32 %r8, %r7, 1;
 ; CHECK-NEXT:    setp.eq.b32 %p9, %r8, 1;
 ; CHECK-NEXT:    or.pred %p10, %p3, %p9;
-; CHECK-NEXT:    xor.b64 %rd56, %rd5, 127;
-; CHECK-NEXT:    or.b64 %rd57, %rd56, %rd6;
-; CHECK-NEXT:    setp.eq.s64 %p11, %rd57, 0;
-; CHECK-NEXT:    selp.b64 %rd104, 0, %rd42, %p10;
-; CHECK-NEXT:    selp.b64 %rd103, 0, %rd41, %p10;
+; CHECK-NEXT:    xor.b64 %rd58, %rd56, 127;
+; CHECK-NEXT:    or.b64 %rd59, %rd58, %rd57;
+; CHECK-NEXT:    setp.eq.s64 %p11, %rd59, 0;
+; CHECK-NEXT:    selp.b64 %rd106, 0, %rd42, %p10;
+; CHECK-NEXT:    selp.b64 %rd105, 0, %rd41, %p10;
 ; CHECK-NEXT:    or.pred %p12, %p10, %p11;
 ; CHECK-NEXT:    @%p12 bra $L__BB5_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd97, %rd5, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd98, %rd6, 0;
-; CHECK-NEXT:    or.b64 %rd60, %rd97, %rd98;
-; CHECK-NEXT:    setp.eq.s64 %p13, %rd60, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd5;
+; CHECK-NEXT:    add.cc.s64 %rd99, %rd56, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd100, %rd57, 0;
+; CHECK-NEXT:    or.b64 %rd62, %rd99, %rd100;
+; CHECK-NEXT:    setp.eq.s64 %p13, %rd62, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd56;
 ; CHECK-NEXT:    mov.b32 %r10, 127;
 ; CHECK-NEXT:    sub.s32 %r11, %r10, %r9;
-; CHECK-NEXT:    shl.b64 %rd61, %rd42, %r11;
+; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r11;
 ; CHECK-NEXT:    mov.b32 %r12, 64;
 ; CHECK-NEXT:    sub.s32 %r13, %r12, %r11;
-; CHECK-NEXT:    shr.u64 %rd62, %rd41, %r13;
-; CHECK-NEXT:    or.b64 %rd63, %rd61, %rd62;
+; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r13;
+; CHECK-NEXT:    or.b64 %rd65, %rd63, %rd64;
 ; CHECK-NEXT:    mov.b32 %r14, 63;
 ; CHECK-NEXT:    sub.s32 %r15, %r14, %r9;
-; CHECK-NEXT:    shl.b64 %rd64, %rd41, %r15;
+; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r15;
 ; CHECK-NEXT:    setp.gt.s32 %p14, %r11, 63;
-; CHECK-NEXT:    selp.b64 %rd102, %rd64, %rd63, %p14;
-; CHECK-NEXT:    shl.b64 %rd101, %rd41, %r11;
-; CHECK-NEXT:    mov.u64 %rd92, %rd95;
+; CHECK-NEXT:    selp.b64 %rd104, %rd66, %rd65, %p14;
+; CHECK-NEXT:    shl.b64 %rd103, %rd41, %r11;
+; CHECK-NEXT:    mov.u64 %rd94, %rd97;
 ; CHECK-NEXT:    @%p13 bra $L__BB5_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r16, %rd97;
-; CHECK-NEXT:    shr.u64 %rd67, %rd41, %r16;
+; CHECK-NEXT:    cvt.u32.u64 %r16, %rd99;
+; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r16;
 ; CHECK-NEXT:    sub.s32 %r18, %r12, %r16;
-; CHECK-NEXT:    shl.b64 %rd68, %rd42, %r18;
-; CHECK-NEXT:    or.b64 %rd69, %rd67, %rd68;
+; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r18;
+; CHECK-NEXT:    or.b64 %rd71, %rd69, %rd70;
 ; CHECK-NEXT:    add.s32 %r19, %r16, -64;
-; CHECK-NEXT:    shr.u64 %rd70, %rd42, %r19;
+; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r19;
 ; CHECK-NEXT:    setp.gt.s32 %p15, %r16, 63;
-; CHECK-NEXT:    selp.b64 %rd99, %rd70, %rd69, %p15;
-; CHECK-NEXT:    shr.u64 %rd100, %rd42, %r16;
+; CHECK-NEXT:    selp.b64 %rd101, %rd72, %rd71, %p15;
+; CHECK-NEXT:    shr.u64 %rd102, %rd42, %r16;
 ; CHECK-NEXT:    add.cc.s64 %rd33, %rd43, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd34, %rd44, -1;
-; CHECK-NEXT:    mov.u64 %rd92, 0;
-; CHECK-NEXT:    mov.u64 %rd95, %rd92;
+; CHECK-NEXT:    mov.u64 %rd94, 0;
+; CHECK-NEXT:    mov.u64 %rd97, %rd94;
 ; CHECK-NEXT:  $L__BB5_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd71, %rd99, 63;
-; CHECK-NEXT:    shl.b64 %rd72, %rd100, 1;
-; CHECK-NEXT:    or.b64 %rd73, %rd72, %rd71;
-; CHECK-NEXT:    shl.b64 %rd74, %rd99, 1;
-; CHECK-NEXT:    shr.u64 %rd75, %rd102, 63;
-; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
-; CHECK-NEXT:    shr.u64 %rd77, %rd101, 63;
-; CHECK-NEXT:    shl.b64 %rd78, %rd102, 1;
-; CHECK-NEXT:    or.b64 %rd79, %rd78, %rd77;
-; CHECK-NEXT:    shl.b64 %rd80, %rd101, 1;
-; CHECK-NEXT:    or.b64 %rd101, %rd95, %rd80;
-; CHECK-NEXT:    or.b64 %rd102, %rd92, %rd79;
-; CHECK-NEXT:    sub.cc.s64 %rd81, %rd33, %rd76;
-; CHECK-NEXT:    subc.cc.s64 %rd82, %rd34, %rd73;
-; CHECK-NEXT:    shr.s64 %rd83, %rd82, 63;
-; CHECK-NEXT:    and.b64 %rd95, %rd83, 1;
-; CHECK-NEXT:    and.b64 %rd84, %rd83, %rd43;
-; CHECK-NEXT:    and.b64 %rd85, %rd83, %rd44;
-; CHECK-NEXT:    sub.cc.s64 %rd99, %rd76, %rd84;
-; CHECK-NEXT:    subc.cc.s64 %rd100, %rd73, %rd85;
-; CHECK-NEXT:    add.cc.s64 %rd97, %rd97, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd98, %rd98, -1;
-; CHECK-NEXT:    or.b64 %rd86, %rd97, %rd98;
-; CHECK-NEXT:    setp.eq.s64 %p16, %rd86, 0;
+; CHECK-NEXT:    shr.u64 %rd73, %rd101, 63;
+; CHECK-NEXT:    shl.b64 %rd74, %rd102, 1;
+; CHECK-NEXT:    or.b64 %rd75, %rd74, %rd73;
+; CHECK-NEXT:    shl.b64 %rd76, %rd101, 1;
+; CHECK-NEXT:    shr.u64 %rd77, %rd104, 63;
+; CHECK-NEXT:    or.b64 %rd78, %rd76, %rd77;
+; CHECK-NEXT:    shr.u64 %rd79, %rd103, 63;
+; CHECK-NEXT:    shl.b64 %rd80, %rd104, 1;
+; CHECK-NEXT:    or.b64 %rd81, %rd80, %rd79;
+; CHECK-NEXT:    shl.b64 %rd82, %rd103, 1;
+; CHECK-NEXT:    or.b64 %rd103, %rd97, %rd82;
+; CHECK-NEXT:    or.b64 %rd104, %rd94, %rd81;
+; CHECK-NEXT:    sub.cc.s64 %rd83, %rd33, %rd78;
+; CHECK-NEXT:    subc.cc.s64 %rd84, %rd34, %rd75;
+; CHECK-NEXT:    shr.s64 %rd85, %rd84, 63;
+; CHECK-NEXT:    and.b64 %rd97, %rd85, 1;
+; CHECK-NEXT:    and.b64 %rd86, %rd85, %rd43;
+; CHECK-NEXT:    and.b64 %rd87, %rd85, %rd44;
+; CHECK-NEXT:    sub.cc.s64 %rd101, %rd78, %rd86;
+; CHECK-NEXT:    subc.cc.s64 %rd102, %rd75, %rd87;
+; CHECK-NEXT:    add.cc.s64 %rd99, %rd99, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd100, %rd100, -1;
+; CHECK-NEXT:    or.b64 %rd88, %rd99, %rd100;
+; CHECK-NEXT:    setp.eq.s64 %p16, %rd88, 0;
 ; CHECK-NEXT:    @%p16 bra $L__BB5_4;
 ; CHECK-NEXT:    bra.uni $L__BB5_2;
 ; CHECK-NEXT:  $L__BB5_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd87, %rd101, 63;
-; CHECK-NEXT:    shl.b64 %rd88, %rd102, 1;
-; CHECK-NEXT:    or.b64 %rd89, %rd88, %rd87;
-; CHECK-NEXT:    shl.b64 %rd90, %rd101, 1;
-; CHECK-NEXT:    or.b64 %rd103, %rd95, %rd90;
-; CHECK-NEXT:    or.b64 %rd104, %rd92, %rd89;
+; CHECK-NEXT:    shr.u64 %rd89, %rd103, 63;
+; CHECK-NEXT:    shl.b64 %rd90, %rd104, 1;
+; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
+; CHECK-NEXT:    shl.b64 %rd92, %rd103, 1;
+; CHECK-NEXT:    or.b64 %rd105, %rd97, %rd92;
+; CHECK-NEXT:    or.b64 %rd106, %rd94, %rd91;
 ; CHECK-NEXT:  $L__BB5_5: // %udiv-end
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0+0], {%rd103, %rd104};
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0+0], {%rd105, %rd106};
 ; CHECK-NEXT:    ret;
   %div = udiv i128 %lhs, %rhs
   ret i128 %div
diff --git a/llvm/test/CodeGen/RISCV/pr84653_pr85190.ll b/llvm/test/CodeGen/RISCV/pr84653_pr85190.ll
index 3fa494e1a57dd..f9b9c8a69d431 100644
--- a/llvm/test/CodeGen/RISCV/pr84653_pr85190.ll
+++ b/llvm/test/CodeGen/RISCV/pr84653_pr85190.ll
@@ -54,7 +54,8 @@ define i1 @pr85190(i64 %a) {
 ; CHECK-ZBB-NEXT:    li a2, -1
 ; CHECK-ZBB-NEXT:    slli a2, a2, 63
 ; CHECK-ZBB-NEXT:    sub a2, a2, a1
-; CHECK-ZBB-NEXT:    slt a0, a0, a2
+; CHECK-ZBB-NEXT:    min a1, a2, zero
+; CHECK-ZBB-NEXT:    slt a0, a0, a1
 ; CHECK-ZBB-NEXT:    ret
   %or = or i64 %a, 7
   %cmp1 = icmp slt i64 %a, 0
diff --git a/llvm/test/CodeGen/SystemZ/pr60413.ll b/llvm/test/CodeGen/SystemZ/pr60413.ll
index 8a6a30318ae58..62f5d49192ea9 100644
--- a/llvm/test/CodeGen/SystemZ/pr60413.ll
+++ b/llvm/test/CodeGen/SystemZ/pr60413.ll
@@ -13,7 +13,6 @@ declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0
 define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-LABEL: m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    stmg %r13, %r15, 104(%r15)
 ; CHECK-NEXT:    aghi %r15, -168
 ; CHECK-NEXT:    lhrl %r1, f+4
 ; CHECK-NEXT:    sll %r1, 8
@@ -21,59 +20,66 @@ define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-NEXT:    ic %r1, 6(%r2)
 ; CHECK-NEXT:    larl %r2, e
 ; CHECK-NEXT:    lb %r0, 3(%r2)
-; CHECK-NEXT:    vlvgp %v0, %r0, %r1
-; CHECK-NEXT:    vlvgp %v1, %r1, %r0
 ; CHECK-NEXT:    vlvgf %v1, %r1, 0
-; CHECK-NEXT:    vlvgf %v1, %r1, 2
-; CHECK-NEXT:    vlvgp %v2, %r1, %r1
-; CHECK-NEXT:    # kill: def $r1l killed $r1l killed $r1d
+; CHECK-NEXT:    vlvgf %v1, %r1, 1
+; CHECK-NEXT:    larl %r2, .LCPI0_0
+; CHECK-NEXT:    vl %v2, 0(%r2), 3
+; CHECK-NEXT:    vlvgf %v1, %r1, 3
+; CHECK-NEXT:    vlvgf %v3, %r1, 3
+; CHECK-NEXT:    vlvgf %v0, %r1, 1
+; CHECK-NEXT:    vperm %v4, %v1, %v0, %v2
+; CHECK-NEXT:    vlvgf %v0, %r1, 3
 ; CHECK-NEXT:    nilh %r1, 255
 ; CHECK-NEXT:    chi %r1, 128
 ; CHECK-NEXT:    ipm %r1
 ; CHECK-NEXT:    risbg %r1, %r1, 63, 191, 36
-; CHECK-NEXT:    vlvgf %v0, %r0, 0
-; CHECK-NEXT:    vlvgf %v0, %r0, 2
-; CHECK-NEXT:    vgbm %v3, 30583
-; CHECK-NEXT:    vn %v0, %v0, %v3
-; CHECK-NEXT:    vn %v1, %v1, %v3
-; CHECK-NEXT:    vrepf %v2, %v2, 1
-; CHECK-NEXT:    vn %v2, %v2, %v3
-; CHECK-NEXT:    vrepif %v3, 127
-; CHECK-NEXT:    vchlf %v1, %v1, %v3
-; CHECK-NEXT:    vlgvf %r13, %v1, 0
-; CHECK-NEXT:    vchlf %v2, %v2, %v3
+; CHECK-NEXT:    vperm %v0, %v3, %v0, %v2
+; CHECK-NEXT:    larl %r2, .LCPI0_1
+; CHECK-NEXT:    vl %v5, 0(%r2), 3
+; CHECK-NEXT:    vgbm %v6, 30583
+; CHECK-NEXT:    vn %v0, %v0, %v6
+; CHECK-NEXT:    vn %v4, %v4, %v6
+; CHECK-NEXT:    vperm %v1, %v1, %v1, %v5
+; CHECK-NEXT:    vn %v5, %v1, %v6
+; CHECK-NEXT:    vperm %v1, %v0, %v3, %v2
+; CHECK-NEXT:    vn %v2, %v1, %v6
+; CHECK-NEXT:    vrepif %v1, 127
+; CHECK-NEXT:    vchlf %v3, %v5, %v1
+; CHECK-NEXT:    vlgvf %r3, %v3, 1
+; CHECK-NEXT:    vlgvf %r2, %v3, 0
+; CHECK-NEXT:    risbg %r2, %r2, 48, 176, 15
+; CHECK-NEXT:    rosbg %r2, %r3, 49, 49, 14
+; CHECK-NEXT:    vlgvf %r3, %v3, 2
+; CHECK-NEXT:    rosbg %r2, %r3, 50, 50, 13
+; CHECK-NEXT:    vlgvf %r3, %v3, 3
+; CHECK-NEXT:    rosbg %r2, %r3, 51, 51, 12
+; CHECK-NEXT:    vchlf %v3, %v4, %v1
+; CHECK-NEXT:    vlgvf %r3, %v3, 0
+; CHECK-NEXT:    rosbg %r2, %r3, 52, 52, 11
+; CHECK-NEXT:    vlgvf %r3, %v3, 1
+; CHECK-NEXT:    rosbg %r2, %r3, 53, 53, 10
+; CHECK-NEXT:    vlgvf %r3, %v3, 2
+; CHECK-NEXT:    rosbg %r2, %r3, 54, 54, 9
+; CHECK-NEXT:    vlgvf %r3, %v3, 3
+; CHECK-NEXT:    rosbg %r2, %r3, 55, 55, 8
+; CHECK-NEXT:    vchlf %v2, %v2, %v1
+; CHECK-NEXT:    vlgvf %r3, %v2, 0
+; CHECK-NEXT:    rosbg %r2, %r3, 56, 56, 7
 ; CHECK-NEXT:    vlgvf %r3, %v2, 1
-; CHECK-NEXT:    nilf %r3, 1
-; CHECK-NEXT:    vlgvf %r4, %v2, 0
-; CHECK-NEXT:    risbg %r2, %r4, 48, 176, 15
-; CHECK-NEXT:    rosbg %r2, %r3, 32, 49, 14
-; CHECK-NEXT:    vlgvf %r5, %v2, 2
-; CHECK-NEXT:    nilf %r5, 1
-; CHECK-NEXT:    rosbg %r2, %r5, 32, 50, 13
-; CHECK-NEXT:    vlgvf %r14, %v2, 3
-; CHECK-NEXT:    nilf %r14, 1
-; CHECK-NEXT:    rosbg %r2, %r14, 32, 51, 12
-; CHECK-NEXT:    rosbg %r2, %r13, 52, 52, 11
-; CHECK-NEXT:    vlgvf %r13, %v1, 1
-; CHECK-NEXT:    rosbg %r2, %r13, 53, 53, 10
-; CHECK-NEXT:    vlgvf %r13, %v1, 2
-; CHECK-NEXT:    rosbg %r2, %r13, 54, 54, 9
-; CHECK-NEXT:    vlgvf %r13, %v1, 3
-; CHECK-NEXT:    rosbg %r2, %r13, 55, 55, 8
-; CHECK-NEXT:    vchlf %v0, %v0, %v3
-; CHECK-NEXT:    vlgvf %r13, %v0, 0
-; CHECK-NEXT:    rosbg %r2, %r13, 56, 56, 7
-; CHECK-NEXT:    vlgvf %r13, %v0, 1
-; CHECK-NEXT:    rosbg %r2, %r13, 57, 57, 6
-; CHECK-NEXT:    vlgvf %r13, %v0, 2
-; CHECK-NEXT:    rosbg %r2, %r13, 58, 58, 5
-; CHECK-NEXT:    vlgvf %r13, %v0, 3
-; CHECK-NEXT:    rosbg %r2, %r13, 59, 59, 4
-; CHECK-NEXT:    nilf %r4, 1
-; CHECK-NEXT:    rosbg %r2, %r4, 32, 60, 3
-; CHECK-NEXT:    rosbg %r2, %r3, 32, 61, 2
-; CHECK-NEXT:    rosbg %r2, %r5, 32, 62, 1
-; CHECK-NEXT:    or %r2, %r14
+; CHECK-NEXT:    rosbg %r2, %r3, 57, 57, 6
+; CHECK-NEXT:    vlgvf %r3, %v2, 2
+; CHECK-NEXT:    rosbg %r2, %r3, 58, 58, 5
+; CHECK-NEXT:    vlgvf %r3, %v2, 3
+; CHECK-NEXT:    rosbg %r2, %r3, 59, 59, 4
+; CHECK-NEXT:    vchlf %v0, %v0, %v1
+; CHECK-NEXT:    vlgvf %r3, %v0, 0
+; CHECK-NEXT:    rosbg %r2, %r3, 60, 60, 3
+; CHECK-NEXT:    vlgvf %r3, %v0, 1
+; CHECK-NEXT:    rosbg %r2, %r3, 61, 61, 2
+; CHECK-NEXT:    vlgvf %r3, %v0, 2
+; CHECK-NEXT:    rosbg %r2, %r3, 62, 62, 1
+; CHECK-NEXT:    vlgvf %r3, %v0, 3
+; CHECK-NEXT:    rosbg %r2, %r3, 63, 63, 0
 ; CHECK-NEXT:    vlgvb %r4, %v0, 1
 ; CHECK-NEXT:    vlgvb %r3, %v0, 0
 ; CHECK-NEXT:    risbg %r3, %r3, 48, 176, 15
@@ -116,7 +122,7 @@ define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-NEXT:    nr %r2, %r0
 ; CHECK-NEXT:    larl %r1, g
 ; CHECK-NEXT:    stc %r2, 0(%r1)
-; CHECK-NEXT:    lmg %r13, %r15, 272(%r15)
+; CHECK-NEXT:    aghi %r15, 168
 ; CHECK-NEXT:    br %r14
 entry:
   %n = alloca i32, align 4
diff --git a/llvm/test/CodeGen/VE/Scalar/max.ll b/llvm/test/CodeGen/VE/Scalar/max.ll
index 12aa101cb48c4..51da557c6c49f 100644
--- a/llvm/test/CodeGen/VE/Scalar/max.ll
+++ b/llvm/test/CodeGen/VE/Scalar/max.ll
@@ -281,11 +281,13 @@ define zeroext i1 @maxi1(i1 zeroext, i1 zeroext) {
 ; CHECK-LABEL: maxi1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    or %s0, %s0, %s1
+; CHECK-NEXT:    and %s0, 1, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; OPT-LABEL: maxi1:
 ; OPT:       # %bb.0:
 ; OPT-NEXT:    or %s0, %s0, %s1
+; OPT-NEXT:    and %s0, 1, %s0
 ; OPT-NEXT:    b.l.t (, %s10)
   %3 = xor i1 %1, true
   %4 = and i1 %3, %0
diff --git a/llvm/test/CodeGen/VE/Scalar/min.ll b/llvm/test/CodeGen/VE/Scalar/min.ll
index da92ebafd0590..69d5ce48601f8 100644
--- a/llvm/test/CodeGen/VE/Scalar/min.ll
+++ b/llvm/test/CodeGen/VE/Scalar/min.ll
@@ -278,6 +278,7 @@ define i32 @min2u32(i32, i32) {
 define zeroext i1 @mini1(i1 zeroext, i1 zeroext) {
 ; CHECK-LABEL: mini1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    and %s2, %s1, %s0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
 ; CHECK-NEXT:    adds.w.zx %s0, %s2, (0)1
@@ -285,6 +286,7 @@ define zeroext i1 @mini1(i1 zeroext, i1 zeroext) {
 ;
 ; OPT-LABEL: mini1:
 ; OPT:       # %bb.0:
+; OPT-NEXT:    and %s0, %s0, (32)0
 ; OPT-NEXT:    and %s2, %s1, %s0
 ; OPT-NEXT:    cmov.w.ne %s2, %s1, %s0
 ; OPT-NEXT:    adds.w.zx %s0, %s2, (0)1
diff --git a/llvm/test/CodeGen/X86/combine-pmadd.ll b/llvm/test/CodeGen/X86/combine-pmadd.ll
index f18497065c943..d011efa5b6140 100644
--- a/llvm/test/CodeGen/X86/combine-pmadd.ll
+++ b/llvm/test/CodeGen/X86/combine-pmadd.ll
@@ -293,10 +293,41 @@ define i32 @combine_pmaddubsw_constant_sat() {
 
 ; Constant folding PMADDWD was causing an infinite loop in the PCMPGT commuting between 2 constant values.
 define i1 @pmaddwd_pcmpgt_infinite_loop() {
-; CHECK-LABEL: pmaddwd_pcmpgt_infinite_loop:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movb $1, %al
-; CHECK-NEXT:    retq
+; SSE-LABEL: pmaddwd_pcmpgt_infinite_loop:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrld $1, %xmm1
+; SSE-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE-NEXT:    pand %xmm0, %xmm1
+; SSE-NEXT:    movmskps %xmm1, %eax
+; SSE-NEXT:    testl %eax, %eax
+; SSE-NEXT:    sete %al
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: pmaddwd_pcmpgt_infinite_loop:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrld $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vtestps %xmm1, %xmm0
+; AVX1-NEXT:    sete %al
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: pmaddwd_pcmpgt_infinite_loop:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrld $1, %xmm0, %xmm1
+; AVX2-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vtestps %xmm1, %xmm0
+; AVX2-NEXT:    sete %al
+; AVX2-NEXT:    retq
   %1 = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> <i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768>, <8 x i16> <i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768>)
   %2 = icmp eq <4 x i32> %1, <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>
   %3 = select <4 x i1> %2, <4 x i32> <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>, <4 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index aa7b77f01d5ba..1c303de55c95d 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -177,7 +177,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $152, %esp
+; X86-NEXT:    subl $156, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -273,42 +273,44 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %ebp, %esi
 ; X86-NEXT:    orl %ebx, %esi
 ; X86-NEXT:    cmovnel %ecx, %edx
-; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    xorl %ebx, %ebx
 ; X86-NEXT:    subl %edx, %edi
-; X86-NEXT:    movl $0, %ebx
-; X86-NEXT:    sbbl %ebx, %ebx
 ; X86-NEXT:    movl $0, %edx
 ; X86-NEXT:    sbbl %edx, %edx
 ; X86-NEXT:    movl $0, %eax
 ; X86-NEXT:    sbbl %eax, %eax
+; X86-NEXT:    movl $0, %esi
+; X86-NEXT:    sbbl %esi, %esi
 ; X86-NEXT:    movl $127, %ecx
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    cmpl %edi, %ecx
-; X86-NEXT:    movl %ebx, %edi
-; X86-NEXT:    movl $0, %ecx
-; X86-NEXT:    sbbl %ebx, %ecx
 ; X86-NEXT:    movl $0, %ecx
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %edx, %ecx
 ; X86-NEXT:    movl $0, %ecx
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %eax, %ecx
+; X86-NEXT:    movl $0, %ecx
+; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    sbbl %esi, %ecx
 ; X86-NEXT:    setb %cl
 ; X86-NEXT:    orb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload
 ; X86-NEXT:    movl (%esp), %edx # 4-byte Reload
-; X86-NEXT:    cmovnel %esi, %edx
-; X86-NEXT:    cmovnel %esi, %ebp
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT:    cmovnel %esi, %ebx
-; X86-NEXT:    cmovel {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT:    cmovnel %ebx, %edx
+; X86-NEXT:    cmovnel %ebx, %ebp
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    cmovnel %ebx, %eax
+; X86-NEXT:    cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT:    movl %ebx, %esi
 ; X86-NEXT:    jne .LBB4_8
 ; X86-NEXT:  # %bb.1: # %_udiv-special-cases
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    xorl $127, %eax
-; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT:    movl %edi, %ecx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT:    xorl $127, %edi
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT:    movl %ebx, %ecx
 ; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT:    orl %eax, %ecx
+; X86-NEXT:    orl %edi, %ecx
 ; X86-NEXT:    je .LBB4_8
 ; X86-NEXT:  # %bb.2: # %udiv-bb1
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
@@ -331,34 +333,34 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    shrb $3, %al
 ; X86-NEXT:    andb $15, %al
 ; X86-NEXT:    negb %al
-; X86-NEXT:    movsbl %al, %eax
-; X86-NEXT:    movl 144(%esp,%eax), %edx
-; X86-NEXT:    movl 148(%esp,%eax), %esi
+; X86-NEXT:    movsbl %al, %edi
+; X86-NEXT:    movl 148(%esp,%edi), %edx
+; X86-NEXT:    movl 152(%esp,%edi), %esi
 ; X86-NEXT:    movb %ch, %cl
 ; X86-NEXT:    shldl %cl, %edx, %esi
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    notb %cl
-; X86-NEXT:    movl 140(%esp,%eax), %ebx
-; X86-NEXT:    movl %ebx, %ebp
+; X86-NEXT:    movl 144(%esp,%edi), %eax
+; X86-NEXT:    movl %eax, %ebp
 ; X86-NEXT:    shrl %ebp
 ; X86-NEXT:    shrl %cl, %ebp
 ; X86-NEXT:    orl %edx, %ebp
-; X86-NEXT:    movl 136(%esp,%eax), %eax
+; X86-NEXT:    movl 140(%esp,%edi), %edx
 ; X86-NEXT:    movb %ch, %cl
-; X86-NEXT:    shldl %cl, %eax, %ebx
-; X86-NEXT:    shll %cl, %eax
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    shldl %cl, %edx, %eax
+; X86-NEXT:    shll %cl, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    addl $1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X86-NEXT:    adcl $0, %edi
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    adcl $0, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    adcl $0, %edx
 ; X86-NEXT:    jae .LBB4_3
 ; X86-NEXT:  # %bb.6:
+; X86-NEXT:    xorl %edi, %edi
 ; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    jmp .LBB4_7
 ; X86-NEXT:  .LBB4_3: # %udiv-preheader
@@ -374,180 +376,176 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movb %al, %ch
+; X86-NEXT:    movb %dl, %ch
 ; X86-NEXT:    andb $7, %ch
-; X86-NEXT:    # kill: def $al killed $al killed $eax
-; X86-NEXT:    shrb $3, %al
-; X86-NEXT:    andb $15, %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movl 100(%esp,%eax), %esi
-; X86-NEXT:    movl %esi, (%esp) # 4-byte Spill
-; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl 96(%esp,%eax), %edx
-; X86-NEXT:    movl %ebp, %edi
-; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    movb %dl, %cl
+; X86-NEXT:    shrb $3, %cl
+; X86-NEXT:    andb $15, %cl
+; X86-NEXT:    movzbl %cl, %edx
+; X86-NEXT:    movl 104(%esp,%edx), %ebx
+; X86-NEXT:    movl 100(%esp,%edx), %edi
+; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edi, %ebp
 ; X86-NEXT:    movb %ch, %cl
-; X86-NEXT:    shrdl %cl, %esi, %ebp
-; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl 88(%esp,%eax), %ebx
-; X86-NEXT:    movl 92(%esp,%eax), %esi
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    shrl %cl, %eax
+; X86-NEXT:    shrdl %cl, %ebx, %ebp
+; X86-NEXT:    movl 92(%esp,%edx), %esi
+; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl 96(%esp,%edx), %esi
+; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    shrl %cl, %edx
 ; X86-NEXT:    notb %cl
-; X86-NEXT:    addl %edx, %edx
-; X86-NEXT:    shll %cl, %edx
-; X86-NEXT:    orl %eax, %edx
-; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    addl %edi, %edi
+; X86-NEXT:    shll %cl, %edi
+; X86-NEXT:    orl %edx, %edi
+; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movb %ch, %cl
-; X86-NEXT:    shrl %cl, (%esp) # 4-byte Folded Spill
-; X86-NEXT:    shrdl %cl, %esi, %ebx
+; X86-NEXT:    shrl %cl, %ebx
 ; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    addl $-1, %eax
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    adcl $-1, %eax
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    adcl $-1, %eax
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    adcl $-1, %eax
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    shrdl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    addl $-1, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    adcl $-1, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    adcl $-1, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    adcl $-1, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    .p2align 4, 0x90
 ; X86-NEXT:  .LBB4_4: # %udiv-do-while
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %ebp, %edx
-; X86-NEXT:    shldl $1, %ebp, (%esp) # 4-byte Folded Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NEXT:    shldl $1, %ebp, %edx
+; X86-NEXT:    movl %ebp, (%esp) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT:    shldl $1, %ebx, %ebp
-; X86-NEXT:    shldl $1, %ecx, %ebx
-; X86-NEXT:    shldl $1, %edi, %ecx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    orl %eax, %ecx
-; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    shldl $1, %ebp, %ebx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NEXT:    shldl $1, %ebp, (%esp) # 4-byte Folded Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT:    shldl $1, %edx, %ebp
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT:    shldl $1, %edi, %edx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    shldl $1, %ecx, %edi
-; X86-NEXT:    orl %eax, %edi
-; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %esi, %edi
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT:    shldl $1, %esi, %ecx
-; X86-NEXT:    orl %eax, %ecx
+; X86-NEXT:    orl %esi, %edi
+; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    shldl $1, %eax, %ecx
+; X86-NEXT:    orl %esi, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    addl %esi, %esi
-; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    sbbl %ebp, %ecx
+; X86-NEXT:    shldl $1, %ecx, %eax
+; X86-NEXT:    orl %esi, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    addl %ecx, %ecx
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    sbbl %edx, %ecx
+; X86-NEXT:    sbbl %ebp, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    sbbl (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    sbbl %ebx, %ecx
 ; X86-NEXT:    sarl $31, %ecx
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    andl $1, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    andl %edi, %esi
+; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X86-NEXT:    movl %ecx, %edi
 ; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT:    subl %ecx, %ebx
-; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    subl %ecx, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %eax, %ebp
 ; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    sbbl %edi, %edx
-; X86-NEXT:    movl %edx, %ebp
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT:    sbbl %esi, (%esp) # 4-byte Folded Spill
+; X86-NEXT:    movl (%esp), %ebp # 4-byte Reload
+; X86-NEXT:    sbbl %edi, %ebp
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    sbbl %esi, %ebx
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    addl $-1, %ecx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    adcl $-1, %eax
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT:    adcl $-1, %edi
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT:    adcl $-1, %ebx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %esi
-; X86-NEXT:    adcl $-1, %edx
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    orl %edx, %eax
-; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    orl %esi, %ecx
-; X86-NEXT:    orl %eax, %ecx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT:    orl %esi, %edi
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    orl %ebx, %ecx
+; X86-NEXT:    orl %edi, %ecx
 ; X86-NEXT:    jne .LBB4_4
 ; X86-NEXT:  # %bb.5:
-; X86-NEXT:    movl %edi, %ebp
-; X86-NEXT:    movl %ecx, %edx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X86-NEXT:  .LBB4_7: # %udiv-loop-exit
 ; X86-NEXT:    shldl $1, %ebp, %edx
-; X86-NEXT:    orl %eax, %edx
-; X86-NEXT:    shldl $1, %ebx, %ebp
-; X86-NEXT:    orl %eax, %ebp
+; X86-NEXT:    orl %ecx, %edx
+; X86-NEXT:    shldl $1, %eax, %ebp
+; X86-NEXT:    orl %ecx, %ebp
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT:    shldl $1, %esi, %ebx
-; X86-NEXT:    orl %eax, %ebx
+; X86-NEXT:    shldl $1, %esi, %eax
+; X86-NEXT:    orl %ecx, %eax
 ; X86-NEXT:    addl %esi, %esi
-; X86-NEXT:    orl %ecx, %esi
+; X86-NEXT:    orl %edi, %esi
 ; X86-NEXT:  .LBB4_8: # %udiv-end
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    xorl %eax, %edx
-; X86-NEXT:    xorl %eax, %ebp
-; X86-NEXT:    xorl %eax, %ebx
-; X86-NEXT:    xorl %eax, %esi
-; X86-NEXT:    subl %eax, %esi
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    xorl %ecx, %edx
+; X86-NEXT:    xorl %ecx, %ebp
+; X86-NEXT:    xorl %ecx, %eax
+; X86-NEXT:    xorl %ecx, %esi
+; X86-NEXT:    subl %ecx, %esi
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    sbbl %eax, %ebx
-; X86-NEXT:    sbbl %eax, %ebp
-; X86-NEXT:    sbbl %eax, %edx
+; X86-NEXT:    sbbl %ecx, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    sbbl %ecx, %ebp
+; X86-NEXT:    sbbl %ecx, %edx
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %esi, (%eax)
-; X86-NEXT:    movl %ebx, 4(%eax)
-; X86-NEXT:    movl %ebp, 8(%eax)
-; X86-NEXT:    movl %edx, 12(%eax)
-; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %esi, (%ecx)
+; X86-NEXT:    movl %eax, 4(%ecx)
+; X86-NEXT:    movl %ebp, 8(%ecx)
+; X86-NEXT:    movl %edx, 12(%ecx)
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl %ebp, %edi
 ; X86-NEXT:    mull %ecx
-; X86-NEXT:    movl %edx, %ebp
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    movl %eax, %ebp
 ; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    mull %ecx
 ; X86-NEXT:    movl %eax, (%esp) # 4-byte Spill
 ; X86-NEXT:    movl %edx, %ecx
-; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT:    adcl $0, %ebp
+; X86-NEXT:    addl %ebp, %ecx
+; X86-NEXT:    adcl $0, %ebx
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    mull %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    mull %ebp
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    adcl %ebp, %edx
-; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    adcl %ebx, %edx
+; X86-NEXT:    movl %edx, %ebx
 ; X86-NEXT:    setb %cl
-; X86-NEXT:    movl %ebx, %eax
-; X86-NEXT:    mull %esi
-; X86-NEXT:    addl %ebp, %eax
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    addl %ebx, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movzbl %cl, %eax
 ; X86-NEXT:    adcl %eax, %edx
@@ -557,12 +555,12 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    imull %eax, %ecx
 ; X86-NEXT:    mull %edi
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    imull %esi, %edi
+; X86-NEXT:    imull %ebp, %edi
 ; X86-NEXT:    addl %edx, %edi
 ; X86-NEXT:    addl %ecx, %edi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    imull %ebx, %ecx
+; X86-NEXT:    imull %esi, %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    imull %edx, %esi
@@ -586,7 +584,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %edx, 4(%eax)
 ; X86-NEXT:    movl %ebx, 8(%eax)
 ; X86-NEXT:    movl %edi, 12(%eax)
-; X86-NEXT:    addl $152, %esp
+; X86-NEXT:    addl $156, %esp
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
diff --git a/llvm/test/CodeGen/X86/pr64589.ll b/llvm/test/CodeGen/X86/pr64589.ll
index 130ef517ae28e..d93d54f4c31d0 100644
--- a/llvm/test/CodeGen/X86/pr64589.ll
+++ b/llvm/test/CodeGen/X86/pr64589.ll
@@ -7,8 +7,8 @@
 define i8 @test(ptr %p) {
 ; CHECK-LABEL: test:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movzbl (%rdi), %eax
-; CHECK-NEXT:    orb 1(%rdi), %al
+; CHECK-NEXT:    movzbl 1(%rdi), %eax
+; CHECK-NEXT:    orb (%rdi), %al
 ; CHECK-NEXT:    setne %al
 ; CHECK-NEXT:    addb %al, %al
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-compare-all_of.ll b/llvm/test/CodeGen/X86/vector-compare-all_of.ll
index ec7dca4285a35..30202701fdb8c 100644
--- a/llvm/test/CodeGen/X86/vector-compare-all_of.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-all_of.ll
@@ -1550,26 +1550,40 @@ define i1 @select_v2i8(ptr %s0, ptr %s1) {
 ;
 ; SSE42-LABEL: select_v2i8:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT:    pxor %xmm0, %xmm1
-; SSE42-NEXT:    ptest %xmm1, %xmm1
+; SSE42-NEXT:    movzwl (%rdi), %eax
+; SSE42-NEXT:    movd %eax, %xmm0
+; SSE42-NEXT:    movzwl (%rsi), %eax
+; SSE42-NEXT:    movd %eax, %xmm1
+; SSE42-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE42-NEXT:    pmovsxbq %xmm1, %xmm0
+; SSE42-NEXT:    movmskpd %xmm0, %eax
+; SSE42-NEXT:    cmpl $3, %eax
 ; SSE42-NEXT:    sete %al
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: select_v2i8:
 ; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1OR2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1OR2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1OR2-NEXT:    vptest %xmm0, %xmm0
-; AVX1OR2-NEXT:    sete %al
+; AVX1OR2-NEXT:    movzwl (%rdi), %eax
+; AVX1OR2-NEXT:    vmovd %eax, %xmm0
+; AVX1OR2-NEXT:    movzwl (%rsi), %eax
+; AVX1OR2-NEXT:    vmovd %eax, %xmm1
+; AVX1OR2-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpmovsxbq %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vtestpd %xmm1, %xmm0
+; AVX1OR2-NEXT:    setb %al
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512-LABEL: select_v2i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    movzwl (%rdi), %eax
-; AVX512-NEXT:    cmpw (%rsi), %ax
+; AVX512-NEXT:    vmovd %eax, %xmm0
+; AVX512-NEXT:    movzwl (%rsi), %eax
+; AVX512-NEXT:    vmovd %eax, %xmm1
+; AVX512-NEXT:    vpcmpeqb %xmm1, %xmm0, %k0
+; AVX512-NEXT:    knotw %k0, %k0
+; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    testb $3, %al
 ; AVX512-NEXT:    sete %al
 ; AVX512-NEXT:    retq
   %v0 = load <2 x i8>, ptr %s0, align 1
diff --git a/llvm/test/CodeGen/X86/vector-compare-any_of.ll b/llvm/test/CodeGen/X86/vector-compare-any_of.ll
index 951bcfa8fc1b7..2df39d69dbb75 100644
--- a/llvm/test/CodeGen/X86/vector-compare-any_of.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-any_of.ll
@@ -1433,19 +1433,25 @@ define i1 @select_v2i8(ptr %s0, ptr %s1) {
 ;
 ; SSE42-LABEL: select_v2i8:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm1
-; SSE42-NEXT:    movmskpd %xmm1, %eax
+; SSE42-NEXT:    movzwl (%rdi), %eax
+; SSE42-NEXT:    movd %eax, %xmm0
+; SSE42-NEXT:    movzwl (%rsi), %eax
+; SSE42-NEXT:    movd %eax, %xmm1
+; SSE42-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE42-NEXT:    pmovsxbq %xmm1, %xmm0
+; SSE42-NEXT:    movmskpd %xmm0, %eax
 ; SSE42-NEXT:    testl %eax, %eax
 ; SSE42-NEXT:    setne %al
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: select_v2i8:
 ; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1OR2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1OR2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    movzwl (%rdi), %eax
+; AVX1OR2-NEXT:    vmovd %eax, %xmm0
+; AVX1OR2-NEXT:    movzwl (%rsi), %eax
+; AVX1OR2-NEXT:    vmovd %eax, %xmm1
+; AVX1OR2-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpmovsxbq %xmm0, %xmm0
 ; AVX1OR2-NEXT:    vtestpd %xmm0, %xmm0
 ; AVX1OR2-NEXT:    setne %al
 ; AVX1OR2-NEXT:    retq

From 5ea38b86a3011b31f2b2a0de330e32971de4a067 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Mon, 22 Jul 2024 08:21:38 -0700
Subject: [PATCH 433/486] [lit][NFC] Avoid unintended -EMPTY suffix in check
 prefix (#99690)

FileCheck has special handline for the `-EMPTY` suffix, that should
match empty lines. Overloading the suffix can be a source of confusion
when reading tests. Additionally, the current implementation seems to
match the following expressions, which appears to be a bug in FileCheck.
---
 llvm/utils/lit/tests/Inputs/shtest-env/env-calls-env.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/utils/lit/tests/Inputs/shtest-env/env-calls-env.txt b/llvm/utils/lit/tests/Inputs/shtest-env/env-calls-env.txt
index e4997b1250ed3..26150c413dc03 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-env/env-calls-env.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-env/env-calls-env.txt
@@ -1,10 +1,10 @@
 # Check that internal env can call internal env.
 
 # RUN: env env %{python} print_environment.py \
-# RUN: | FileCheck -check-prefix=CHECK-2-EMPTY %s
+# RUN: | FileCheck -check-prefix=CHECK-2-EMPTY-ARGS %s
 #
-# CHECK-2-EMPTY: BAR = 2
-# CHECK-2-EMPTY: FOO = 1
+# CHECK-2-EMPTY-ARGS: BAR = 2
+# CHECK-2-EMPTY-ARGS: FOO = 1
 
 # RUN: env FOO=2 env BAR=1 %{python} print_environment.py \
 # RUN: | FileCheck -check-prefix=CHECK-2-VAL %s

From e9709899db7d4a8b1c36475e90e4a934335f3d95 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 17 Jul 2024 10:48:53 -0500
Subject: [PATCH 434/486] [clang][OpenMP] Avoid names that hide existing
 variables, NFC

---
 clang/lib/Sema/SemaOpenMP.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 3bd981cb442aa..67e3c1d9067f3 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6168,14 +6168,14 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
               DMC->getDefaultmapModifierLoc();
     }
     for (unsigned VC = 0; VC < DefaultmapKindNum; ++VC) {
-      auto Kind = static_cast<OpenMPDefaultmapClauseKind>(VC);
+      auto K = static_cast<OpenMPDefaultmapClauseKind>(VC);
       for (unsigned I = 0; I < OMPC_MAP_delete; ++I) {
-        ArrayRef<Expr *> ImplicitMap = DSAChecker.getImplicitMap(
-            Kind, static_cast<OpenMPMapClauseKind>(I));
+        ArrayRef<Expr *> ImplicitMap =
+            DSAChecker.getImplicitMap(K, static_cast<OpenMPMapClauseKind>(I));
         ImplicitMaps[VC][I].append(ImplicitMap.begin(), ImplicitMap.end());
       }
       ArrayRef<OpenMPMapModifierKind> ImplicitModifier =
-          DSAChecker.getImplicitMapModifier(Kind);
+          DSAChecker.getImplicitMapModifier(K);
       ImplicitMapModifiers[VC].append(ImplicitModifier.begin(),
                                       ImplicitModifier.end());
       std::fill_n(std::back_inserter(ImplicitMapModifiersLoc[VC]),
@@ -6249,10 +6249,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
           continue;
         CXXScopeSpec MapperIdScopeSpec;
         DeclarationNameInfo MapperId;
-        auto Kind = static_cast<OpenMPMapClauseKind>(ClauseKindCnt);
+        auto K = static_cast<OpenMPMapClauseKind>(ClauseKindCnt);
         if (OMPClause *Implicit = ActOnOpenMPMapClause(
                 nullptr, ImplicitMapModifiers[I], ImplicitMapModifiersLoc[I],
-                MapperIdScopeSpec, MapperId, Kind, /*IsMapTypeImplicit=*/true,
+                MapperIdScopeSpec, MapperId, K, /*IsMapTypeImplicit=*/true,
                 SourceLocation(), SourceLocation(), ImplicitMap,
                 OMPVarListLocTy())) {
           ClausesWithImplicit.emplace_back(Implicit);

From a5447613de6ce572e5b5e71821a1cb0d3ee890f3 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm@amd.com>
Date: Mon, 22 Jul 2024 17:30:45 +0200
Subject: [PATCH 435/486] [Flang][runtime] Add dependency to build
 FortranRuntime after flang-new (#99737)

Makefile-based builds did not have proper dependencies to built the
FortranRuntime target after Flang new is available. This PR introduces a
dependency to ensure that this is the case. Relates to PR #95388.

---------

Co-authored-by: Michael Kruse <github@meinersbur.de>
---
 flang/runtime/CMakeLists.txt | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index 8588af7e16eec..d587fd44b1678 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -295,3 +295,15 @@ else()
     FortranRuntime.static_dbg FortranRuntime.dynamic_dbg)
 endif()
 set_target_properties(FortranRuntime PROPERTIES FOLDER "Flang/Runtime Libraries")
+
+# If FortranRuntime is part of a Flang build (and not a separate build) then
+# add dependency to make sure that Fortran runtime library is being built after
+# we have the Flang compiler available.  This also includes the MODULE files
+# that compile when the 'flang-new' target is built.
+#
+# TODO: This is a workaround and should be updated when runtime build procedure
+# is changed to a regular runtime build.  See discussion in PR #95388.
+if (TARGET flang-new AND TARGET module_files)
+  add_dependencies(FortranRuntime flang-new module_files)
+endif()
+

From 9da221d15ff7baa786031a855f1aa40de63dd8cb Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Mon, 22 Jul 2024 08:22:15 -0700
Subject: [PATCH 436/486] [SLP][NFC]Remove incorrect attribure from the test,
 NFC.

---
 .../SLPVectorizer/X86/gather-extractelements-different-bbs.ll   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gather-extractelements-different-bbs.ll b/llvm/test/Transforms/SLPVectorizer/X86/gather-extractelements-different-bbs.ll
index 58c108b81a7ce..be790b772a2eb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gather-extractelements-different-bbs.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gather-extractelements-different-bbs.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux -mattr="-avx512pf,+avx512f,+avx512bw" -slp-threshold=-100 -slp-min-tree-size=0 < %s | FileCheck %s
+; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux -mattr="+avx512f,+avx512bw" -slp-threshold=-100 -slp-min-tree-size=0 < %s | FileCheck %s
 
 define i32 @foo(i32 %a) {
 ; CHECK-LABEL: @foo(

From d7e185cca98411e82655feecc297906cb7ed0850 Mon Sep 17 00:00:00 2001
From: Pranav Bhandarkar <pranav.bhandarkar@amd.com>
Date: Mon, 22 Jul 2024 10:56:45 -0500
Subject: [PATCH 437/486] [OMPIRBuilder] - Handle dependencies in
 `createTarget` (#93977)

This patch handles dependencies specified by the `depend` clause on an
OpenMP target construct. It does this much the same way clang does it by
materializing an OpenMP `task` that is tagged with the dependencies.

The following functions are relevant to this patch -
1) `createTarget` - This function itself is largely unchanged except
that it now accepts a vector of `DependData` objects that it simply
forwards to `emitTargetCall`
2) `emitTargetCall` - This function has changed now to check if an outer
target-task needs to be materialized (i.e if `target` construct has
`nowait` or has `depend` clause). If yes, it calls `emitTargetTask` to
do all the heavy lifting for creating and dispatching the task.
3) `emitTargetTask` - Bulk of the change is here. See the large comment
explaining what it does at the beginning of this function
---
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       |  25 +-
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 493 ++++++++++++++++--
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |  56 +-
 mlir/test/Target/LLVMIR/omptarget-depend.mlir | 140 +++++
 .../test/offloading/fortran/target-depend.f90 |  69 +++
 5 files changed, 730 insertions(+), 53 deletions(-)
 create mode 100644 mlir/test/Target/LLVMIR/omptarget-depend.mlir
 create mode 100644 offload/test/offloading/fortran/target-depend.f90

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 02c0265354ccd..a6995888de7d4 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -2322,6 +2322,26 @@ class OpenMPIRBuilder {
       EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
       Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP);
 
+  /// Generate a target-task for the target construct
+  ///
+  /// \param OutlinedFn The outlined device/target kernel function.
+  /// \param OutlinedFnID The ooulined function ID.
+  /// \param EmitTargetCallFallbackCB Call back function to generate host
+  ///        fallback code.
+  /// \param Args Data structure holding information about the kernel arguments.
+  /// \param DeviceID Identifier for the device via the 'device' clause.
+  /// \param RTLoc Source location identifier
+  /// \param AllocaIP The insertion point to be used for alloca instructions.
+  /// \param Dependencies Vector of DependData objects holding information of
+  ///        dependencies as specified by the 'depend' clause.
+  /// \param HasNoWait True if the target construct had 'nowait' on it, false
+  ///        otherwise
+  InsertPointTy emitTargetTask(
+      Function *OutlinedFn, Value *OutlinedFnID,
+      EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
+      Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP,
+      SmallVector<OpenMPIRBuilder::DependData> &Dependencies, bool HasNoWait);
+
   /// Emit the arguments to be passed to the runtime library based on the
   /// arrays of base pointers, pointers, sizes, map types, and mappers.  If
   /// ForEndCall, emit map types to be passed for the end of the region instead
@@ -2805,6 +2825,8 @@ class OpenMPIRBuilder {
   /// \param BodyGenCB Callback that will generate the region code.
   /// \param ArgAccessorFuncCB Callback that will generate accessors
   /// instructions for passed in target arguments where neccessary
+  /// \param Dependencies A vector of DependData objects that carry
+  // dependency information as passed in the depend clause
   InsertPointTy createTarget(const LocationDescription &Loc,
                              OpenMPIRBuilder::InsertPointTy AllocaIP,
                              OpenMPIRBuilder::InsertPointTy CodeGenIP,
@@ -2813,7 +2835,8 @@ class OpenMPIRBuilder {
                              SmallVectorImpl<Value *> &Inputs,
                              GenMapInfoCallbackTy GenMapInfoCB,
                              TargetBodyGenCallbackTy BodyGenCB,
-                             TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB);
+                             TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
+                             SmallVector<DependData> Dependencies = {});
 
   /// Returns __kmpc_for_static_init_* runtime function for the specified
   /// size \a IVSize and sign \a IVSigned. Will create a distribute call
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index a06e81dd280e0..ab2482b91213b 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -360,23 +360,23 @@ BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
 // This function creates a fake integer value and a fake use for the integer
 // value. It returns the fake value created. This is useful in modeling the
 // extra arguments to the outlined functions.
-Value *createFakeIntVal(IRBuilder<> &Builder,
+Value *createFakeIntVal(IRBuilderBase &Builder,
                         OpenMPIRBuilder::InsertPointTy OuterAllocaIP,
-                        std::stack<Instruction *> &ToBeDeleted,
+                        llvm::SmallVectorImpl<Instruction *> &ToBeDeleted,
                         OpenMPIRBuilder::InsertPointTy InnerAllocaIP,
                         const Twine &Name = "", bool AsPtr = true) {
   Builder.restoreIP(OuterAllocaIP);
   Instruction *FakeVal;
   AllocaInst *FakeValAddr =
       Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, Name + ".addr");
-  ToBeDeleted.push(FakeValAddr);
+  ToBeDeleted.push_back(FakeValAddr);
 
   if (AsPtr) {
     FakeVal = FakeValAddr;
   } else {
     FakeVal =
         Builder.CreateLoad(Builder.getInt32Ty(), FakeValAddr, Name + ".val");
-    ToBeDeleted.push(FakeVal);
+    ToBeDeleted.push_back(FakeVal);
   }
 
   // Generate a fake use of this value
@@ -389,7 +389,7 @@ Value *createFakeIntVal(IRBuilder<> &Builder,
     UseFakeVal =
         cast<BinaryOperator>(Builder.CreateAdd(FakeVal, Builder.getInt32(10)));
   }
-  ToBeDeleted.push(UseFakeVal);
+  ToBeDeleted.push_back(UseFakeVal);
   return FakeVal;
 }
 
@@ -1707,6 +1707,74 @@ void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
   emitTaskyieldImpl(Loc);
 }
 
+// Processes the dependencies in Dependencies and does the following
+// - Allocates space on the stack of an array of DependInfo objects
+// - Populates each DependInfo object with relevant information of
+//   the corresponding dependence.
+// - All code is inserted in the entry block of the current function.
+static Value *emitTaskDependencies(
+    OpenMPIRBuilder &OMPBuilder,
+    SmallVectorImpl<OpenMPIRBuilder::DependData> &Dependencies) {
+  // Early return if we have no dependencies to process
+  if (Dependencies.empty())
+    return nullptr;
+
+  // Given a vector of DependData objects, in this function we create an
+  // array on the stack that holds kmp_dep_info objects corresponding
+  // to each dependency. This is then passed to the OpenMP runtime.
+  // For example, if there are 'n' dependencies then the following psedo
+  // code is generated. Assume the first dependence is on a variable 'a'
+  //
+  // \code{c}
+  // DepArray = alloc(n x sizeof(kmp_depend_info);
+  // idx = 0;
+  // DepArray[idx].base_addr = ptrtoint(&a);
+  // DepArray[idx].len = 8;
+  // DepArray[idx].flags = Dep.DepKind; /*(See OMPContants.h for DepKind)*/
+  // ++idx;
+  // DepArray[idx].base_addr = ...;
+  // \endcode
+
+  IRBuilderBase &Builder = OMPBuilder.Builder;
+  Type *DependInfo = OMPBuilder.DependInfo;
+  Module &M = OMPBuilder.M;
+
+  Value *DepArray = nullptr;
+  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
+  Builder.SetInsertPoint(
+      OldIP.getBlock()->getParent()->getEntryBlock().getTerminator());
+
+  Type *DepArrayTy = ArrayType::get(DependInfo, Dependencies.size());
+  DepArray = Builder.CreateAlloca(DepArrayTy, nullptr, ".dep.arr.addr");
+
+  for (const auto &[DepIdx, Dep] : enumerate(Dependencies)) {
+    Value *Base =
+        Builder.CreateConstInBoundsGEP2_64(DepArrayTy, DepArray, 0, DepIdx);
+    // Store the pointer to the variable
+    Value *Addr = Builder.CreateStructGEP(
+        DependInfo, Base,
+        static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
+    Value *DepValPtr = Builder.CreatePtrToInt(Dep.DepVal, Builder.getInt64Ty());
+    Builder.CreateStore(DepValPtr, Addr);
+    // Store the size of the variable
+    Value *Size = Builder.CreateStructGEP(
+        DependInfo, Base, static_cast<unsigned int>(RTLDependInfoFields::Len));
+    Builder.CreateStore(
+        Builder.getInt64(M.getDataLayout().getTypeStoreSize(Dep.DepValueType)),
+        Size);
+    // Store the dependency kind
+    Value *Flags = Builder.CreateStructGEP(
+        DependInfo, Base,
+        static_cast<unsigned int>(RTLDependInfoFields::Flags));
+    Builder.CreateStore(
+        ConstantInt::get(Builder.getInt8Ty(),
+                         static_cast<unsigned int>(Dep.DepKind)),
+        Flags);
+  }
+  Builder.restoreIP(OldIP);
+  return DepArray;
+}
+
 OpenMPIRBuilder::InsertPointTy
 OpenMPIRBuilder::createTask(const LocationDescription &Loc,
                             InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB,
@@ -1751,7 +1819,7 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc,
   OI.ExitBB = TaskExitBB;
 
   // Add the thread ID argument.
-  std::stack<Instruction *> ToBeDeleted;
+  SmallVector<Instruction *, 4> ToBeDeleted;
   OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(
       Builder, AllocaIP, ToBeDeleted, TaskAllocaIP, "global.tid", false));
 
@@ -1948,10 +2016,8 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc,
           Shareds, [Shareds](Use &U) { return U.getUser() != Shareds; });
     }
 
-    while (!ToBeDeleted.empty()) {
-      ToBeDeleted.top()->eraseFromParent();
-      ToBeDeleted.pop();
-    }
+    llvm::for_each(llvm::reverse(ToBeDeleted),
+                   [](Instruction *I) { I->eraseFromParent(); });
   };
 
   addOutlineInfo(std::move(OI));
@@ -6617,6 +6683,91 @@ static Function *createOutlinedFunction(
   return Func;
 }
 
+/// Create an entry point for a target task with the following.
+/// It'll have the following signature
+/// void @.omp_target_task_proxy_func(i32 %thread.id, ptr %task)
+/// This function is called from emitTargetTask once the
+/// code to launch the target kernel has been outlined already.
+static Function *emitTargetTaskProxyFunction(OpenMPIRBuilder &OMPBuilder,
+                                             IRBuilderBase &Builder,
+                                             CallInst *StaleCI) {
+  Module &M = OMPBuilder.M;
+  // KernelLaunchFunction is the target launch function, i.e.
+  // the function that sets up kernel arguments and calls
+  // __tgt_target_kernel to launch the kernel on the device.
+  //
+  Function *KernelLaunchFunction = StaleCI->getCalledFunction();
+
+  // StaleCI is the CallInst which is the call to the outlined
+  // target kernel launch function. If there are values that the
+  // outlined function uses then these are aggregated into a structure
+  // which is passed as the second argument. If not, then there's
+  // only one argument, the threadID. So, StaleCI can be
+  //
+  // %structArg = alloca { ptr, ptr }, align 8
+  // %gep_ = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 0
+  // store ptr %20, ptr %gep_, align 8
+  // %gep_8 = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 1
+  // store ptr %21, ptr %gep_8, align 8
+  // call void @_QQmain..omp_par.1(i32 %global.tid.val6, ptr %structArg)
+  //
+  // OR
+  //
+  // call void @_QQmain..omp_par.1(i32 %global.tid.val6)
+  OpenMPIRBuilder::InsertPointTy IP(StaleCI->getParent(),
+                                    StaleCI->getIterator());
+  LLVMContext &Ctx = StaleCI->getParent()->getContext();
+  Type *ThreadIDTy = Type::getInt32Ty(Ctx);
+  Type *TaskPtrTy = OMPBuilder.TaskPtr;
+  Type *TaskTy = OMPBuilder.Task;
+  auto ProxyFnTy =
+      FunctionType::get(Builder.getVoidTy(), {ThreadIDTy, TaskPtrTy},
+                        /* isVarArg */ false);
+  auto ProxyFn = Function::Create(ProxyFnTy, GlobalValue::InternalLinkage,
+                                  ".omp_target_task_proxy_func",
+                                  Builder.GetInsertBlock()->getModule());
+  ProxyFn->getArg(0)->setName("thread.id");
+  ProxyFn->getArg(1)->setName("task");
+
+  BasicBlock *EntryBB =
+      BasicBlock::Create(Builder.getContext(), "entry", ProxyFn);
+  Builder.SetInsertPoint(EntryBB);
+
+  bool HasShareds = StaleCI->arg_size() > 1;
+  // TODO: This is a temporary assert to prove to ourselves that
+  // the outlined target launch function is always going to have
+  // atmost two arguments if there is any data shared between
+  // host and device.
+  assert((!HasShareds || (StaleCI->arg_size() == 2)) &&
+         "StaleCI with shareds should have exactly two arguments.");
+  if (HasShareds) {
+    auto *ArgStructAlloca = dyn_cast<AllocaInst>(StaleCI->getArgOperand(1));
+    assert(ArgStructAlloca &&
+           "Unable to find the alloca instruction corresponding to arguments "
+           "for extracted function");
+    auto *ArgStructType =
+        dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());
+
+    AllocaInst *NewArgStructAlloca =
+        Builder.CreateAlloca(ArgStructType, nullptr, "structArg");
+    Value *TaskT = ProxyFn->getArg(1);
+    Value *ThreadId = ProxyFn->getArg(0);
+    Value *SharedsSize =
+        Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
+
+    Value *Shareds = Builder.CreateStructGEP(TaskTy, TaskT, 0);
+    LoadInst *LoadShared =
+        Builder.CreateLoad(PointerType::getUnqual(Ctx), Shareds);
+
+    Builder.CreateMemCpy(
+        NewArgStructAlloca, NewArgStructAlloca->getAlign(), LoadShared,
+        LoadShared->getPointerAlignment(M.getDataLayout()), SharedsSize);
+
+    Builder.CreateCall(KernelLaunchFunction, {ThreadId, NewArgStructAlloca});
+  }
+  Builder.CreateRetVoid();
+  return ProxyFn;
+}
 static void emitTargetOutlinedFunction(
     OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
     TargetRegionEntryInfo &EntryInfo, Function *&OutlinedFn,
@@ -6634,13 +6785,281 @@ static void emitTargetOutlinedFunction(
   OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction, true,
                                       OutlinedFn, OutlinedFnID);
 }
+OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetTask(
+    Function *OutlinedFn, Value *OutlinedFnID,
+    EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
+    Value *DeviceID, Value *RTLoc, OpenMPIRBuilder::InsertPointTy AllocaIP,
+    SmallVector<llvm::OpenMPIRBuilder::DependData> &Dependencies,
+    bool HasNoWait) {
+
+  // When we arrive at this function, the target region itself has been
+  // outlined into the function OutlinedFn.
+  // So at ths point, for
+  // --------------------------------------------------
+  //   void user_code_that_offloads(...) {
+  //     omp target depend(..) map(from:a) map(to:b, c)
+  //        a = b + c
+  //   }
+  //
+  // --------------------------------------------------
+  //
+  // we have
+  //
+  // --------------------------------------------------
+  //
+  //   void user_code_that_offloads(...) {
+  //     %.offload_baseptrs = alloca [3 x ptr], align 8
+  //     %.offload_ptrs = alloca [3 x ptr], align 8
+  //     %.offload_mappers = alloca [3 x ptr], align 8
+  //     ;; target region has been outlined and now we need to
+  //     ;; offload to it via a target task.
+  //   }
+  //   void outlined_device_function(ptr a, ptr b, ptr c) {
+  //     *a = *b + *c
+  //   }
+  //
+  // We have to now do the following
+  // (i)   Make an offloading call to outlined_device_function using the OpenMP
+  //       RTL. See 'kernel_launch_function' in the pseudo code below. This is
+  //       emitted by emitKernelLaunch
+  // (ii)  Create a task entry point function that calls kernel_launch_function
+  //       and is the entry point for the target task. See
+  //       '@.omp_target_task_proxy_func in the pseudocode below.
+  // (iii) Create a task with the task entry point created in (ii)
+  //
+  // That is we create the following
+  //
+  //   void user_code_that_offloads(...) {
+  //     %.offload_baseptrs = alloca [3 x ptr], align 8
+  //     %.offload_ptrs = alloca [3 x ptr], align 8
+  //     %.offload_mappers = alloca [3 x ptr], align 8
+  //
+  //     %structArg = alloca { ptr, ptr, ptr }, align 8
+  //     %strucArg[0] = %.offload_baseptrs
+  //     %strucArg[1] = %.offload_ptrs
+  //     %strucArg[2] = %.offload_mappers
+  //     proxy_target_task = @__kmpc_omp_task_alloc(...,
+  //                                               @.omp_target_task_proxy_func)
+  //     memcpy(proxy_target_task->shareds, %structArg, sizeof(structArg))
+  //     dependencies_array = ...
+  //     ;; if nowait not present
+  //     call @__kmpc_omp_wait_deps(..., dependencies_array)
+  //     call @__kmpc_omp_task_begin_if0(...)
+  //     call @ @.omp_target_task_proxy_func(i32 thread_id, ptr
+  //     %proxy_target_task) call @__kmpc_omp_task_complete_if0(...)
+  //   }
+  //
+  //   define internal void @.omp_target_task_proxy_func(i32 %thread.id,
+  //                                                     ptr %task) {
+  //       %structArg = alloca {ptr, ptr, ptr}
+  //       %shared_data = load (getelementptr %task, 0, 0)
+  //       mempcy(%structArg, %shared_data, sizeof(structArg))
+  //       kernel_launch_function(%thread.id, %structArg)
+  //   }
+  //
+  //   We need the proxy function because the signature of the task entry point
+  //   expected by kmpc_omp_task is always the same and will be different from
+  //   that of the kernel_launch function.
+  //
+  //   kernel_launch_function is generated by emitKernelLaunch and has the
+  //   always_inline attribute.
+  //   void kernel_launch_function(thread_id,
+  //                               structArg) alwaysinline {
+  //       %kernel_args = alloca %struct.__tgt_kernel_arguments, align 8
+  //       offload_baseptrs = load(getelementptr structArg, 0, 0)
+  //       offload_ptrs = load(getelementptr structArg, 0, 1)
+  //       offload_mappers = load(getelementptr structArg, 0, 2)
+  //       ; setup kernel_args using offload_baseptrs, offload_ptrs and
+  //       ; offload_mappers
+  //       call i32 @__tgt_target_kernel(...,
+  //                                     outlined_device_function,
+  //                                     ptr %kernel_args)
+  //   }
+  //   void outlined_device_function(ptr a, ptr b, ptr c) {
+  //      *a = *b + *c
+  //   }
+  //
+  BasicBlock *TargetTaskBodyBB =
+      splitBB(Builder, /*CreateBranch=*/true, "target.task.body");
+  BasicBlock *TargetTaskAllocaBB =
+      splitBB(Builder, /*CreateBranch=*/true, "target.task.alloca");
+
+  InsertPointTy TargetTaskAllocaIP(TargetTaskAllocaBB,
+                                   TargetTaskAllocaBB->begin());
+  InsertPointTy TargetTaskBodyIP(TargetTaskBodyBB, TargetTaskBodyBB->begin());
+
+  OutlineInfo OI;
+  OI.EntryBB = TargetTaskAllocaBB;
+  OI.OuterAllocaBB = AllocaIP.getBlock();
+
+  // Add the thread ID argument.
+  SmallVector<Instruction *, 4> ToBeDeleted;
+  OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(
+      Builder, AllocaIP, ToBeDeleted, TargetTaskAllocaIP, "global.tid", false));
+
+  Builder.restoreIP(TargetTaskBodyIP);
+
+  // emitKernelLaunch makes the necessary runtime call to offload the kernel.
+  // We then outline all that code into a separate function
+  // ('kernel_launch_function' in the pseudo code above). This function is then
+  // called by the target task proxy function (see
+  // '@.omp_target_task_proxy_func' in the pseudo code above)
+  // "@.omp_target_task_proxy_func' is generated by emitTargetTaskProxyFunction
+  Builder.restoreIP(emitKernelLaunch(Builder, OutlinedFn, OutlinedFnID,
+                                     EmitTargetCallFallbackCB, Args, DeviceID,
+                                     RTLoc, TargetTaskAllocaIP));
+
+  OI.ExitBB = Builder.saveIP().getBlock();
+  OI.PostOutlineCB = [this, ToBeDeleted, Dependencies,
+                      HasNoWait](Function &OutlinedFn) mutable {
+    assert(OutlinedFn.getNumUses() == 1 &&
+           "there must be a single user for the outlined function");
+
+    CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
+    bool HasShareds = StaleCI->arg_size() > 1;
+
+    Function *ProxyFn = emitTargetTaskProxyFunction(*this, Builder, StaleCI);
+
+    LLVM_DEBUG(dbgs() << "Proxy task entry function created: " << *ProxyFn
+                      << "\n");
+
+    Builder.SetInsertPoint(StaleCI);
+
+    // Gather the arguments for emitting the runtime call.
+    uint32_t SrcLocStrSize;
+    Constant *SrcLocStr =
+        getOrCreateSrcLocStr(LocationDescription(Builder), SrcLocStrSize);
+    Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
+
+    // @__kmpc_omp_task_alloc
+    Function *TaskAllocFn =
+        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);
+
+    // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
+    // call.
+    Value *ThreadID = getOrCreateThreadID(Ident);
+
+    // Argument - `sizeof_kmp_task_t` (TaskSize)
+    // Tasksize refers to the size in bytes of kmp_task_t data structure
+    // including private vars accessed in task.
+    // TODO: add kmp_task_t_with_privates (privates)
+    Value *TaskSize =
+        Builder.getInt64(M.getDataLayout().getTypeStoreSize(Task));
+
+    // Argument - `sizeof_shareds` (SharedsSize)
+    // SharedsSize refers to the shareds array size in the kmp_task_t data
+    // structure.
+    Value *SharedsSize = Builder.getInt64(0);
+    if (HasShareds) {
+      auto *ArgStructAlloca = dyn_cast<AllocaInst>(StaleCI->getArgOperand(1));
+      assert(ArgStructAlloca &&
+             "Unable to find the alloca instruction corresponding to arguments "
+             "for extracted function");
+      auto *ArgStructType =
+          dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());
+      assert(ArgStructType && "Unable to find struct type corresponding to "
+                              "arguments for extracted function");
+      SharedsSize =
+          Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
+    }
+
+    // Argument - `flags`
+    // Task is tied iff (Flags & 1) == 1.
+    // Task is untied iff (Flags & 1) == 0.
+    // Task is final iff (Flags & 2) == 2.
+    // Task is not final iff (Flags & 2) == 0.
+    // A target task is not final and is untied.
+    Value *Flags = Builder.getInt32(0);
+
+    // Emit the @__kmpc_omp_task_alloc runtime call
+    // The runtime call returns a pointer to an area where the task captured
+    // variables must be copied before the task is run (TaskData)
+    CallInst *TaskData = Builder.CreateCall(
+        TaskAllocFn, {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
+                      /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
+                      /*task_func=*/ProxyFn});
+
+    if (HasShareds) {
+      Value *Shareds = StaleCI->getArgOperand(1);
+      Align Alignment = TaskData->getPointerAlignment(M.getDataLayout());
+      Value *TaskShareds = Builder.CreateLoad(VoidPtr, TaskData);
+      Builder.CreateMemCpy(TaskShareds, Alignment, Shareds, Alignment,
+                           SharedsSize);
+    }
+
+    Value *DepArray = emitTaskDependencies(*this, Dependencies);
+
+    // ---------------------------------------------------------------
+    // V5.2 13.8 target construct
+    // If the nowait clause is present, execution of the target task
+    // may be deferred. If the nowait clause is not present, the target task is
+    // an included task.
+    // ---------------------------------------------------------------
+    // The above means that the lack of a nowait on the target construct
+    // translates to '#pragma omp task if(0)'
+    if (!HasNoWait) {
+      if (DepArray) {
+        Function *TaskWaitFn =
+            getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_wait_deps);
+        Builder.CreateCall(
+            TaskWaitFn,
+            {/*loc_ref=*/Ident, /*gtid=*/ThreadID,
+             /*ndeps=*/Builder.getInt32(Dependencies.size()),
+             /*dep_list=*/DepArray,
+             /*ndeps_noalias=*/ConstantInt::get(Builder.getInt32Ty(), 0),
+             /*noalias_dep_list=*/
+             ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});
+      }
+      // Included task.
+      Function *TaskBeginFn =
+          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0);
+      Function *TaskCompleteFn =
+          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_complete_if0);
+      Builder.CreateCall(TaskBeginFn, {Ident, ThreadID, TaskData});
+      CallInst *CI = nullptr;
+      if (HasShareds)
+        CI = Builder.CreateCall(ProxyFn, {ThreadID, TaskData});
+      else
+        CI = Builder.CreateCall(ProxyFn, {ThreadID});
+      CI->setDebugLoc(StaleCI->getDebugLoc());
+      Builder.CreateCall(TaskCompleteFn, {Ident, ThreadID, TaskData});
+    } else if (DepArray) {
+      // HasNoWait - meaning the task may be deferred. Call
+      // __kmpc_omp_task_with_deps if there are dependencies,
+      // else call __kmpc_omp_task
+      Function *TaskFn =
+          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_with_deps);
+      Builder.CreateCall(
+          TaskFn,
+          {Ident, ThreadID, TaskData, Builder.getInt32(Dependencies.size()),
+           DepArray, ConstantInt::get(Builder.getInt32Ty(), 0),
+           ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});
+    } else {
+      // Emit the @__kmpc_omp_task runtime call to spawn the task
+      Function *TaskFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task);
+      Builder.CreateCall(TaskFn, {Ident, ThreadID, TaskData});
+    }
 
-static void emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
-                           OpenMPIRBuilder::InsertPointTy AllocaIP,
-                           Function *OutlinedFn, Constant *OutlinedFnID,
-                           int32_t NumTeams, int32_t NumThreads,
-                           SmallVectorImpl<Value *> &Args,
-                           OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB) {
+    StaleCI->eraseFromParent();
+    llvm::for_each(llvm::reverse(ToBeDeleted),
+                   [](Instruction *I) { I->eraseFromParent(); });
+  };
+  addOutlineInfo(std::move(OI));
+
+  LLVM_DEBUG(dbgs() << "Insert block after emitKernelLaunch = \n"
+                    << *(Builder.GetInsertBlock()) << "\n");
+  LLVM_DEBUG(dbgs() << "Module after emitKernelLaunch = \n"
+                    << *(Builder.GetInsertBlock()->getParent()->getParent())
+                    << "\n");
+  return Builder.saveIP();
+}
+static void emitTargetCall(
+    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
+    OpenMPIRBuilder::InsertPointTy AllocaIP, Function *OutlinedFn,
+    Constant *OutlinedFnID, int32_t NumTeams, int32_t NumThreads,
+    SmallVectorImpl<Value *> &Args,
+    OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
+    SmallVector<llvm::OpenMPIRBuilder::DependData> Dependencies = {}) {
 
   OpenMPIRBuilder::TargetDataInfo Info(
       /*RequiresDevicePointerInfo=*/false,
@@ -6677,23 +7096,34 @@ static void emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
   Value *DynCGGroupMem = Builder.getInt32(0);
 
   bool HasNoWait = false;
+  bool HasDependencies = Dependencies.size() > 0;
+  bool RequiresOuterTargetTask = HasNoWait || HasDependencies;
 
   OpenMPIRBuilder::TargetKernelArgs KArgs(NumTargetItems, RTArgs, NumIterations,
                                           NumTeamsVal, NumThreadsVal,
                                           DynCGGroupMem, HasNoWait);
 
-  Builder.restoreIP(OMPBuilder.emitKernelLaunch(
-      Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, KArgs,
-      DeviceID, RTLoc, AllocaIP));
+  // The presence of certain clauses on the target directive require the
+  // explicit generation of the target task.
+  if (RequiresOuterTargetTask) {
+    Builder.restoreIP(OMPBuilder.emitTargetTask(
+        OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, KArgs, DeviceID,
+        RTLoc, AllocaIP, Dependencies, HasNoWait));
+  } else {
+    Builder.restoreIP(OMPBuilder.emitKernelLaunch(
+        Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, KArgs,
+        DeviceID, RTLoc, AllocaIP));
+  }
 }
-
 OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
     const LocationDescription &Loc, InsertPointTy AllocaIP,
     InsertPointTy CodeGenIP, TargetRegionEntryInfo &EntryInfo, int32_t NumTeams,
     int32_t NumThreads, SmallVectorImpl<Value *> &Args,
     GenMapInfoCallbackTy GenMapInfoCB,
     OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
-    OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB) {
+    OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
+    SmallVector<DependData> Dependencies) {
+
   if (!updateToLocation(Loc))
     return InsertPointTy();
 
@@ -6701,12 +7131,18 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
 
   Function *OutlinedFn;
   Constant *OutlinedFnID;
+  // The target region is outlined into its own function. The LLVM IR for
+  // the target region itself is generated using the callbacks CBFunc
+  // and ArgAccessorFuncCB
   emitTargetOutlinedFunction(*this, Builder, EntryInfo, OutlinedFn,
                              OutlinedFnID, Args, CBFunc, ArgAccessorFuncCB);
+
+  // If we are not on the target device, then we need to generate code
+  // to make a remote call (offload) to the previously outlined function
+  // that represents the target region. Do that now.
   if (!Config.isTargetDevice())
     emitTargetCall(*this, Builder, AllocaIP, OutlinedFn, OutlinedFnID, NumTeams,
-                   NumThreads, Args, GenMapInfoCB);
-
+                   NumThreads, Args, GenMapInfoCB, Dependencies);
   return Builder.saveIP();
 }
 
@@ -7827,7 +8263,7 @@ OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
   OI.OuterAllocaBB = &OuterAllocaBB;
 
   // Insert fake values for global tid and bound tid.
-  std::stack<Instruction *> ToBeDeleted;
+  SmallVector<Instruction *, 8> ToBeDeleted;
   InsertPointTy OuterAllocaIP(&OuterAllocaBB, OuterAllocaBB.begin());
   OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(
       Builder, OuterAllocaIP, ToBeDeleted, AllocaIP, "gid", true));
@@ -7842,7 +8278,7 @@ OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
     assert(OutlinedFn.getNumUses() == 1 &&
            "there must be a single user for the outlined function");
     CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
-    ToBeDeleted.push(StaleCI);
+    ToBeDeleted.push_back(StaleCI);
 
     assert((OutlinedFn.arg_size() == 2 || OutlinedFn.arg_size() == 3) &&
            "Outlined function must have two or three arguments only");
@@ -7866,10 +8302,9 @@ OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
                            omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),
                        Args);
 
-    while (!ToBeDeleted.empty()) {
-      ToBeDeleted.top()->eraseFromParent();
-      ToBeDeleted.pop();
-    }
+    llvm::for_each(llvm::reverse(ToBeDeleted),
+                   [](Instruction *I) { I->eraseFromParent(); });
+
   };
 
   if (!Config.isTargetDevice())
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index f7cb52236e1a9..8b031deca8931 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -1009,6 +1009,32 @@ convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder,
   return bodyGenStatus;
 }
 
+static void
+buildDependData(std::optional<ArrayAttr> depends, OperandRange dependVars,
+                LLVM::ModuleTranslation &moduleTranslation,
+                SmallVectorImpl<llvm::OpenMPIRBuilder::DependData> &dds) {
+  if (dependVars.empty())
+    return;
+  for (auto dep : llvm::zip(dependVars, depends->getValue())) {
+    llvm::omp::RTLDependenceKindTy type;
+    switch (
+        cast<mlir::omp::ClauseTaskDependAttr>(std::get<1>(dep)).getValue()) {
+    case mlir::omp::ClauseTaskDepend::taskdependin:
+      type = llvm::omp::RTLDependenceKindTy::DepIn;
+      break;
+    // The OpenMP runtime requires that the codegen for 'depend' clause for
+    // 'out' dependency kind must be the same as codegen for 'depend' clause
+    // with 'inout' dependency.
+    case mlir::omp::ClauseTaskDepend::taskdependout:
+    case mlir::omp::ClauseTaskDepend::taskdependinout:
+      type = llvm::omp::RTLDependenceKindTy::DepInOut;
+      break;
+    };
+    llvm::Value *depVal = moduleTranslation.lookupValue(std::get<0>(dep));
+    llvm::OpenMPIRBuilder::DependData dd(type, depVal->getType(), depVal);
+    dds.emplace_back(dd);
+  }
+}
 /// Converts an OpenMP task construct into LLVM IR using OpenMPIRBuilder.
 static LogicalResult
 convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
@@ -1032,28 +1058,8 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
   };
 
   SmallVector<llvm::OpenMPIRBuilder::DependData> dds;
-  if (!taskOp.getDependVars().empty() && taskOp.getDepends()) {
-    for (auto dep :
-         llvm::zip(taskOp.getDependVars(), taskOp.getDepends()->getValue())) {
-      llvm::omp::RTLDependenceKindTy type;
-      switch (
-          cast<mlir::omp::ClauseTaskDependAttr>(std::get<1>(dep)).getValue()) {
-      case mlir::omp::ClauseTaskDepend::taskdependin:
-        type = llvm::omp::RTLDependenceKindTy::DepIn;
-        break;
-      // The OpenMP runtime requires that the codegen for 'depend' clause for
-      // 'out' dependency kind must be the same as codegen for 'depend' clause
-      // with 'inout' dependency.
-      case mlir::omp::ClauseTaskDepend::taskdependout:
-      case mlir::omp::ClauseTaskDepend::taskdependinout:
-        type = llvm::omp::RTLDependenceKindTy::DepInOut;
-        break;
-      };
-      llvm::Value *depVal = moduleTranslation.lookupValue(std::get<0>(dep));
-      llvm::OpenMPIRBuilder::DependData dd(type, depVal->getType(), depVal);
-      dds.emplace_back(dd);
-    }
-  }
+  buildDependData(taskOp.getDepends(), taskOp.getDependVars(),
+                  moduleTranslation, dds);
 
   llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
       findAllocaInsertPoint(builder, moduleTranslation);
@@ -3292,10 +3298,14 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
     if (!mapData.IsDeclareTarget[i] && !mapData.IsAMember[i])
       kernelInput.push_back(mapData.OriginalValue[i]);
   }
+  SmallVector<llvm::OpenMPIRBuilder::DependData> dds;
+  buildDependData(targetOp.getDepends(), targetOp.getDependVars(),
+                  moduleTranslation, dds);
 
   builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTarget(
       ompLoc, allocaIP, builder.saveIP(), entryInfo, defaultValTeams,
-      defaultValThreads, kernelInput, genMapInfoCB, bodyCB, argAccessorCB));
+      defaultValThreads, kernelInput, genMapInfoCB, bodyCB, argAccessorCB,
+      dds));
 
   // Remap access operations to declare target reference pointers for the
   // device, essentially generating extra loadop's as necessary
diff --git a/mlir/test/Target/LLVMIR/omptarget-depend.mlir b/mlir/test/Target/LLVMIR/omptarget-depend.mlir
new file mode 100644
index 0000000000000..c386342005e5e
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-depend.mlir
@@ -0,0 +1,140 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+  llvm.func @_QQmain() attributes {fir.bindc_name = "main"} {
+    %0 = llvm.mlir.constant(39 : index) : i64
+    %1 = llvm.mlir.constant(0 : index) : i64
+    %2 = llvm.mlir.constant(1 : index) : i64
+    %3 = llvm.mlir.constant(40 : index) : i64
+    %4 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+    %5 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+    %6 = llvm.mlir.constant(1 : i64) : i64
+    %7 = llvm.alloca %6 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+    %8 = llvm.mlir.addressof @_QFEn : !llvm.ptr
+    omp.task {
+      %14 = llvm.mlir.constant(1 : i64) : i64
+      %15 = llvm.alloca %14 x i32 {bindc_name = "i", pinned} : (i64) -> !llvm.ptr
+      %16 = llvm.load %8 : !llvm.ptr -> i32
+      %17 = llvm.sext %16 : i32 to i64
+      %18 = llvm.trunc %2 : i64 to i32
+      llvm.br ^bb1(%18, %17 : i32, i64)
+    ^bb1(%19: i32, %20: i64):  // 2 preds: ^bb0, ^bb2
+      %21 = llvm.icmp "sgt" %20, %1 : i64
+      llvm.cond_br %21, ^bb2, ^bb3
+    ^bb2:  // pred: ^bb1
+      llvm.store %19, %15 : i32, !llvm.ptr
+      %22 = llvm.load %15 : !llvm.ptr -> i32
+      %23 = llvm.sext %22 : i32 to i64
+      %24 = llvm.mlir.constant(1 : i64) : i64
+      %25 = llvm.mlir.constant(0 : i64) : i64
+      %26 = llvm.sub %23, %24 overflow<nsw> : i64
+      %27 = llvm.mul %26, %24 overflow<nsw> : i64
+      %28 = llvm.mul %27, %24 overflow<nsw> : i64
+      %29 = llvm.add %28, %25 overflow<nsw> : i64
+      %30 = llvm.mul %24, %3 overflow<nsw> : i64
+      %31 = llvm.getelementptr %4[%29] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+      llvm.store %22, %31 : i32, !llvm.ptr
+      %32 = llvm.load %15 : !llvm.ptr -> i32
+      %33 = llvm.add %32, %18 : i32
+      %34 = llvm.sub %20, %2 : i64
+      llvm.br ^bb1(%33, %34 : i32, i64)
+    ^bb3:  // pred: ^bb1
+      llvm.store %19, %15 : i32, !llvm.ptr
+      omp.terminator
+    }
+    %9 = omp.map.bounds lower_bound(%1 : i64) upper_bound(%0 : i64) extent(%3 : i64) stride(%2 : i64) start_idx(%2 : i64)
+    %10 = omp.map.info var_ptr(%4 : !llvm.ptr, !llvm.array<40 x i32>) map_clauses(to) capture(ByRef) bounds(%9) -> !llvm.ptr {name = "a"}
+    %11 = omp.map.info var_ptr(%5 : !llvm.ptr, !llvm.array<40 x i32>) map_clauses(from) capture(ByRef) bounds(%9) -> !llvm.ptr {name = "b"}
+    %12 = omp.map.info var_ptr(%7 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "i"}
+    %13 = omp.map.info var_ptr(%8 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "n"}
+    omp.target map_entries(%10 -> %arg0, %11 -> %arg1, %12 -> %arg2, %13 -> %arg3 : !llvm.ptr, !llvm.ptr, !llvm.ptr, !llvm.ptr) depend(taskdependin -> %4 : !llvm.ptr) {
+    ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr, %arg3: !llvm.ptr):
+      %14 = llvm.mlir.constant(0 : index) : i64
+      %15 = llvm.mlir.constant(10 : i32) : i32
+      %16 = llvm.mlir.constant(1 : index) : i64
+      %17 = llvm.mlir.constant(40 : index) : i64
+      %18 = llvm.load %arg3 : !llvm.ptr -> i32
+      %19 = llvm.sext %18 : i32 to i64
+      %20 = llvm.trunc %16 : i64 to i32
+      llvm.br ^bb1(%20, %19 : i32, i64)
+    ^bb1(%21: i32, %22: i64):  // 2 preds: ^bb0, ^bb2
+      %23 = llvm.icmp "sgt" %22, %14 : i64
+      llvm.cond_br %23, ^bb2, ^bb3
+    ^bb2:  // pred: ^bb1
+      llvm.store %21, %arg2 : i32, !llvm.ptr
+      %24 = llvm.load %arg2 : !llvm.ptr -> i32
+      %25 = llvm.sext %24 : i32 to i64
+      %26 = llvm.mlir.constant(1 : i64) : i64
+      %27 = llvm.mlir.constant(0 : i64) : i64
+      %28 = llvm.sub %25, %26 overflow<nsw> : i64
+      %29 = llvm.mul %28, %26 overflow<nsw> : i64
+      %30 = llvm.mul %29, %26 overflow<nsw> : i64
+      %31 = llvm.add %30, %27 overflow<nsw> : i64
+      %32 = llvm.mul %26, %17 overflow<nsw> : i64
+      %33 = llvm.getelementptr %arg0[%31] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+      %34 = llvm.load %33 : !llvm.ptr -> i32
+      %35 = llvm.add %34, %15 : i32
+      %36 = llvm.mlir.constant(1 : i64) : i64
+      %37 = llvm.mlir.constant(0 : i64) : i64
+      %38 = llvm.sub %25, %36 overflow<nsw> : i64
+      %39 = llvm.mul %38, %36 overflow<nsw> : i64
+      %40 = llvm.mul %39, %36 overflow<nsw> : i64
+      %41 = llvm.add %40, %37 overflow<nsw> : i64
+      %42 = llvm.mul %36, %17 overflow<nsw> : i64
+      %43 = llvm.getelementptr %arg1[%41] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+      llvm.store %35, %43 : i32, !llvm.ptr
+      %44 = llvm.load %arg2 : !llvm.ptr -> i32
+      %45 = llvm.add %44, %20 : i32
+      %46 = llvm.sub %22, %16 : i64
+      llvm.br ^bb1(%45, %46 : i32, i64)
+    ^bb3:  // pred: ^bb1
+      llvm.store %21, %arg2 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
+  }
+  llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<40 x i32> {
+    %0 = llvm.mlir.zero : !llvm.array<40 x i32>
+    llvm.return %0 : !llvm.array<40 x i32>
+  }
+  llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<40 x i32> {
+    %0 = llvm.mlir.zero : !llvm.array<40 x i32>
+    llvm.return %0 : !llvm.array<40 x i32>
+  }
+  llvm.mlir.global internal @_QFEc() {addr_space = 0 : i32} : !llvm.array<40 x i32> {
+    %0 = llvm.mlir.zero : !llvm.array<40 x i32>
+    llvm.return %0 : !llvm.array<40 x i32>
+  }
+  llvm.mlir.global internal @_QFEn() {addr_space = 0 : i32} : i32 {
+    %0 = llvm.mlir.constant(40 : i32) : i32
+    llvm.return %0 : i32
+  }
+  llvm.func @_FortranAProgramStart(i32, !llvm.ptr, !llvm.ptr, !llvm.ptr) attributes {sym_visibility = "private"}
+  llvm.func @_FortranAProgramEndStatement() attributes {sym_visibility = "private"}
+  llvm.func @main(%arg0: i32, %arg1: !llvm.ptr, %arg2: !llvm.ptr) -> i32 {
+    %0 = llvm.mlir.constant(0 : i32) : i32
+    %1 = llvm.mlir.zero : !llvm.ptr
+    llvm.call @_FortranAProgramStart(%arg0, %arg1, %arg2, %1) {fastmathFlags = #llvm.fastmath<contract>} : (i32, !llvm.ptr, !llvm.ptr, !llvm.ptr) -> ()
+    llvm.call @_QQmain() {fastmathFlags = #llvm.fastmath<contract>} : () -> ()
+    llvm.call @_FortranAProgramEndStatement() {fastmathFlags = #llvm.fastmath<contract>} : () -> ()
+    llvm.return %0 : i32
+  }
+
+// %strucArg holds pointers to shared data.
+// CHECK: define void @_QQmain() {
+// CHECK-DAG: %[[STRUCTARG:.+]] = alloca { ptr, ptr, ptr }, align 8
+// CHECK-DAG:  %[[DEP_ARRAY:.+]] = alloca [1 x %struct.kmp_dep_info], align 8
+// CHECK: %[[DEP_INFO:.+]]  = getelementptr inbounds [1 x %struct.kmp_dep_info], ptr %[[DEP_ARRAY]], i64 0, i64 0
+// CHECK: %[[PTR0:.+]] = getelementptr inbounds %struct.kmp_dep_info, ptr %[[DEP_INFO]], i32 0, i32 0
+// CHECK: store i64 ptrtoint (ptr @_QFEa to i64), ptr %[[PTR0]], align 4
+// CHECK: %[[PTR1:.+]] = getelementptr inbounds %struct.kmp_dep_info, ptr %[[DEP_INFO]], i32 0, i32 1
+// CHECK: store i64 8, ptr %[[PTR1]], align 4
+// CHECK: %[[PTR2:.+]] = getelementptr inbounds %struct.kmp_dep_info, ptr %[[DEP_INFO]], i32 0, i32 2
+// CHECK: store i8 1, ptr %[[PTR2]], align 1
+
+// CHECK: %[[TASKDATA:.+]] = call ptr @__kmpc_omp_task_alloc({{.+}}, ptr @.omp_target_task_proxy_func)
+// CHECK: %[[SHARED_DATA:.+]] = load ptr, ptr %[[TASKDATA]], align 8
+// CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 1 %[[SHARED_DATA]], ptr align 1 %[[STRUCTARG]], i64 24, i1 false)
+// CHECK: call void @__kmpc_omp_wait_deps({{.+}}, i32 1, ptr %[[DEP_ARRAY]], i32 0, ptr null)
+// CHECK: call void @__kmpc_omp_task_begin_if0({{.+}}, ptr  %[[TASKDATA]])
+// CHECK: call void @.omp_target_task_proxy_func({{.+}}, ptr %[[TASKDATA]])
+// CHECK: call void @__kmpc_omp_task_complete_if0({{.+}}, ptr %[[TASKDATA]])
+	      
diff --git a/offload/test/offloading/fortran/target-depend.f90 b/offload/test/offloading/fortran/target-depend.f90
new file mode 100644
index 0000000000000..928eb671c9706
--- /dev/null
+++ b/offload/test/offloading/fortran/target-depend.f90
@@ -0,0 +1,69 @@
+! Offloading test checking the use of the depend clause on
+! the target construct
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+  implicit none
+  integer :: a = 0
+  INTERFACE
+     FUNCTION omp_get_device_num() BIND(C)
+       USE, INTRINSIC :: iso_c_binding, ONLY: C_INT
+       integer :: omp_get_device_num
+     END FUNCTION omp_get_device_num
+  END INTERFACE
+
+  call foo(5, a)
+  print*, "======= FORTRAN Test passed! ======="
+  print*, "foo(5) returned ", a, ", expected 6\n"
+
+  !       stop 0
+  contains
+    subroutine foo(N, r)
+      integer, intent(in) :: N
+      integer, intent(out) :: r
+      integer :: z, i, accumulator
+      z = 1
+      accumulator = 0
+      ! Spawn 3 threads
+      !$omp parallel num_threads(3)
+
+      ! A single thread will then create two tasks - one is the 'producer' and
+      ! potentially slower task that updates 'z' to 'N'. The second is an
+      ! offloaded target task that increments 'z'. If the depend clauses work
+      ! properly, the target task should wait for the 'producer' task to
+      ! complete before incrementing 'z'. We use 'omp single' here because the
+      ! depend clause establishes dependencies between sibling tasks only.
+      ! This is the easiest way of creating two sibling tasks.
+      !$omp single
+      !$omp task depend(out: z) shared(z)
+      do i=1, 32766
+         ! dumb loop nest to slow down the update of 'z'.
+         ! Adding a function call slows down the producer to the point
+         ! that removing the depend clause from the target construct below
+         ! frequently results in the wrong answer.
+         accumulator = accumulator + omp_get_device_num()
+      end do
+      z = N
+      !$omp end task
+
+      ! z is 5 now. Increment z to 6.
+      !$omp target map(tofrom: z) depend(in:z)
+      z = z + 1
+      !$omp end target
+      !$omp end single
+      !$omp end parallel
+      ! Use 'accumulator' so it is not optimized away by the compiler.
+      print *, accumulator
+      r = z
+    end subroutine foo
+
+!CHECK: ======= FORTRAN Test passed! =======
+!CHECK: foo(5) returned 6 , expected 6
+end program main

From 4010ddf780d90c42877c9ab8d28b7e084cfe197e Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 22 Jul 2024 09:03:05 -0700
Subject: [PATCH 438/486] [MC,AArch64] Create mapping symbols with non-unique
 names

Add `createLocalSymbol` to create a local, non-temporary symbol.
Different from `createRenamableSymbol`, the `Used` bit is ignored,
therefore multiple local symbols might share the same name.

Utilizing `createLocalSymbol` in AArch64 allows for efficient mapping
symbol creation with non-unique names, saving .strtab space.
The behavior matches GNU assembler.

Pull Request: https://github.com/llvm/llvm-project/pull/99836
---
 .../aarch64-cortex-a53-843419-abs-mapsyms.s   |  2 +-
 lld/test/ELF/aarch64-gnu-ifunc.s              |  2 +-
 lld/test/ELF/aarch64-reloc-pauth.s            | 12 ++---
 lld/test/ELF/aarch64-thunk-script.s           |  4 +-
 lld/test/ELF/basic-aarch64.s                  |  4 +-
 lld/test/ELF/pack-dyn-relocs.s                | 48 +++++++++----------
 llvm/include/llvm/MC/MCContext.h              |  4 ++
 llvm/lib/MC/MCContext.cpp                     |  5 ++
 .../MCTargetDesc/AArch64ELFStreamer.cpp       |  8 +---
 .../CodeGen/AArch64/callbr-asm-obj-file.ll    | 16 +++----
 .../CodeGen/AArch64/inlineasm-ldr-pseudo.ll   |  2 +-
 .../Symbolize/ELF/aarch64-mapping-symbol.s    |  6 +--
 llvm/test/MC/AArch64/CheckDataSymbol.s        |  2 +-
 .../test/MC/AArch64/mapping-across-sections.s | 20 ++++----
 llvm/test/MC/AArch64/mapping-within-section.s | 15 ++++--
 llvm/test/MC/AArch64/size-directive.s         |  2 +-
 llvm/test/MC/ELF/AArch64/cfi.s                |  4 +-
 .../AArch64/elf-aarch64-mapping-symbols.test  |  8 ++--
 18 files changed, 88 insertions(+), 76 deletions(-)

diff --git a/lld/test/ELF/aarch64-cortex-a53-843419-abs-mapsyms.s b/lld/test/ELF/aarch64-cortex-a53-843419-abs-mapsyms.s
index 608b1aceb0a00..a9b95b1a63ef6 100644
--- a/lld/test/ELF/aarch64-cortex-a53-843419-abs-mapsyms.s
+++ b/lld/test/ELF/aarch64-cortex-a53-843419-abs-mapsyms.s
@@ -12,7 +12,7 @@
         .type _start, %function
 _start: ret
 
-// CHECK:     Name: $x.0
+// CHECK:     Name: $x
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Binding: Local (0x0)
diff --git a/lld/test/ELF/aarch64-gnu-ifunc.s b/lld/test/ELF/aarch64-gnu-ifunc.s
index d76b54eabf8a4..f39ae0cd84934 100644
--- a/lld/test/ELF/aarch64-gnu-ifunc.s
+++ b/lld/test/ELF/aarch64-gnu-ifunc.s
@@ -37,7 +37,7 @@
 // CHECK-NEXT:    Section: Undefined
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
-// CHECK-NEXT:    Name: $x.0
+// CHECK-NEXT:    Name: $x
 // CHECK-NEXT:    Value: 0x210188
 // CHECK-NEXT:    Size: 0
 // CHECK-NEXT:    Binding: Local
diff --git a/lld/test/ELF/aarch64-reloc-pauth.s b/lld/test/ELF/aarch64-reloc-pauth.s
index 19493fcfa1365..6b4d23d37babc 100644
--- a/lld/test/ELF/aarch64-reloc-pauth.s
+++ b/lld/test/ELF/aarch64-reloc-pauth.s
@@ -68,11 +68,11 @@
 # RELR-EMPTY:
 # RELR-NEXT: Relocation section '.relr.auth.dyn' at offset 0x[[ADDR2]] contains 5 entries:
 # RELR-NEXT: Index: Entry Address Symbolic Address
-# RELR-NEXT: 0000: 0000000000030440 0000000000030440 $d.0
-# RELR-NEXT: 0001: 000000000000000f 0000000000030448 $d.0 + 0x8
-# RELR-NEXT:  0000000000030450 $d.0 + 0x10
-# RELR-NEXT:  0000000000030458 $d.0 + 0x18
-# RELR-NEXT: 0002: 0000000000030492 0000000000030492 $d.0 + 0x52
+# RELR-NEXT: 0000: 0000000000030440 0000000000030440 $d
+# RELR-NEXT: 0001: 000000000000000f 0000000000030448 $d + 0x8
+# RELR-NEXT:  0000000000030450 $d + 0x10
+# RELR-NEXT:  0000000000030458 $d + 0x18
+# RELR-NEXT: 0002: 0000000000030492 0000000000030492 $d + 0x52
 
 # HEX:      Hex dump of section '.test':
 # HEX-NEXT: 0x00030440 01000000 2a000020 42040300 2b000000
@@ -201,7 +201,7 @@
 # EMPTY-RELA-EMPTY:
 # EMPTY-RELA-NEXT: Relocation section '.relr.auth.dyn' at offset {{.+}} contains 1 entries:
 # EMPTY-RELA-NEXT: Index: Entry Address Symbolic Address
-# EMPTY-RELA-NEXT: 0000: 0000000000030310 0000000000030310 $d.0
+# EMPTY-RELA-NEXT: 0000: 0000000000030310 0000000000030310 $d
 
 # EMPTY-RELA-RO-NOT: .rela.dyn
 
diff --git a/lld/test/ELF/aarch64-thunk-script.s b/lld/test/ELF/aarch64-thunk-script.s
index 08ff4e9871180..a6c524f548288 100644
--- a/lld/test/ELF/aarch64-thunk-script.s
+++ b/lld/test/ELF/aarch64-thunk-script.s
@@ -43,8 +43,8 @@ high_target:
 // CHECK-NEXT:                 ret
 
 /// Local symbols copied from %t.o
-// NM:      t $x.0
-// NM-NEXT: t $x.1
+// NM:      t $x
+// NM-NEXT: t $x
 /// Local thunk symbols.
 // NM-NEXT: t __AArch64AbsLongThunk_high_target
 // NM-NEXT: t $x
diff --git a/lld/test/ELF/basic-aarch64.s b/lld/test/ELF/basic-aarch64.s
index 6b109e8da2c3e..1f59d33c4230b 100644
--- a/lld/test/ELF/basic-aarch64.s
+++ b/lld/test/ELF/basic-aarch64.s
@@ -119,7 +119,7 @@ _start:
 # CHECK-NEXT:     ]
 # CHECK-NEXT:     Address: 0x0
 # CHECK-NEXT:     Offset: 0x1AA
-# CHECK-NEXT:     Size: 13
+# CHECK-NEXT:     Size: 11
 # CHECK-NEXT:     Link: 0
 # CHECK-NEXT:     Info: 0
 # CHECK-NEXT:     AddressAlignment: 1
@@ -189,7 +189,7 @@ _start:
 # CHECK-NEXT:     Section: Undefined (0x0)
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Symbol {
-# CHECK-NEXT:     Name: $x.0
+# CHECK-NEXT:     Name: $x
 # CHECK-NEXT:     Value: 0x210120
 # CHECK-NEXT:     Size: 0
 # CHECK-NEXT:     Binding: Local (0x0)
diff --git a/lld/test/ELF/pack-dyn-relocs.s b/lld/test/ELF/pack-dyn-relocs.s
index 3cbba68716972..dd18c92c7352e 100644
--- a/lld/test/ELF/pack-dyn-relocs.s
+++ b/lld/test/ELF/pack-dyn-relocs.s
@@ -264,30 +264,30 @@
 // RELR64-EMPTY: 
 // RELR64-NEXT:  Relocation section '.relr.dyn' at offset {{.*}} contains 24 entries:
 // RELR64-NEXT:  Symbolic Address
-// RELR64-NEXT:  $d.0{{$}}
-// RELR64-NEXT:  $d.0 + 0x8
-// RELR64-NEXT:  $d.0 + 0x10
-// RELR64-NEXT:  $d.0 + 0x18
-// RELR64-NEXT:  $d.0 + 0x20
-// RELR64-NEXT:  $d.0 + 0x28
-// RELR64-NEXT:  $d.0 + 0x30
-// RELR64-NEXT:  $d.0 + 0x38
-// RELR64-NEXT:  $d.0 + 0x48
-// RELR64-NEXT:  $d.0 + 0x50
-// RELR64-NEXT:  $d.0 + 0x58
-// RELR64-NEXT:  $d.0 + 0x60
-// RELR64-NEXT:  $d.0 + 0x68
-// RELR64-NEXT:  $d.0 + 0x70
-// RELR64-NEXT:  $d.0 + 0x78
-// RELR64-NEXT:  $d.0 + 0x90
-// RELR64-NEXT:  $d.0 + 0x98
-// RELR64-NEXT:  $d.0 + 0xa0
-// RELR64-NEXT:  $d.0 + 0xa8
-// RELR64-NEXT:  $d.0 + 0xb0
-// RELR64-NEXT:  $d.0 + 0xb8
-// RELR64-NEXT:  $d.0 + 0xc0
-// RELR64-NEXT:  $d.0 + 0xc8
-// RELR64-NEXT:  $d.0 + 0xd0
+// RELR64-NEXT:  $d{{$}}
+// RELR64-NEXT:  $d + 0x8
+// RELR64-NEXT:  $d + 0x10
+// RELR64-NEXT:  $d + 0x18
+// RELR64-NEXT:  $d + 0x20
+// RELR64-NEXT:  $d + 0x28
+// RELR64-NEXT:  $d + 0x30
+// RELR64-NEXT:  $d + 0x38
+// RELR64-NEXT:  $d + 0x48
+// RELR64-NEXT:  $d + 0x50
+// RELR64-NEXT:  $d + 0x58
+// RELR64-NEXT:  $d + 0x60
+// RELR64-NEXT:  $d + 0x68
+// RELR64-NEXT:  $d + 0x70
+// RELR64-NEXT:  $d + 0x78
+// RELR64-NEXT:  $d + 0x90
+// RELR64-NEXT:  $d + 0x98
+// RELR64-NEXT:  $d + 0xa0
+// RELR64-NEXT:  $d + 0xa8
+// RELR64-NEXT:  $d + 0xb0
+// RELR64-NEXT:  $d + 0xb8
+// RELR64-NEXT:  $d + 0xc0
+// RELR64-NEXT:  $d + 0xc8
+// RELR64-NEXT:  $d + 0xd0
 // RELR64-EMPTY:
 // RELR64-NEXT: Hex dump of section '.data':
 // RELR64-NEXT: 0x00030490 90040300 00000000 91040300 00000000 .
diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index 4c2be7353cdd6..57ba40f7ac26f 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -468,6 +468,10 @@ class MCContext {
   /// true, behaves like getOrCreateSymbol, prefixed with PrivateLabelPrefix.
   MCSymbol *createBlockSymbol(const Twine &Name, bool AlwaysEmit = false);
 
+  /// Create a local, non-temporary symbol like an ELF mapping symbol. Calling
+  /// the function with the same name will generate new, unique instances.
+  MCSymbol *createLocalSymbol(StringRef Name);
+
   /// Create the definition of a directional local symbol for numbered label
   /// (used for "1:" definitions).
   MCSymbol *createDirectionalLocalSymbol(unsigned LocalLabelVal);
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index 44853f3bf43f3..228c4fb03a276 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -349,6 +349,11 @@ MCSymbol *MCContext::createNamedTempSymbol() {
   return createNamedTempSymbol("tmp");
 }
 
+MCSymbol *MCContext::createLocalSymbol(StringRef Name) {
+  MCSymbolTableEntry &NameEntry = getSymbolTableEntry(Name);
+  return createSymbolImpl(&NameEntry, /*IsTemporary=*/false);
+}
+
 unsigned MCContext::NextInstance(unsigned LocalLabelVal) {
   MCLabel *&Label = Instances[LocalLabelVal];
   if (!Label)
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index 27751633017ca..ed670bce594ec 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -181,7 +181,7 @@ class AArch64ELFStreamer : public MCELFStreamer {
                      std::unique_ptr<MCCodeEmitter> Emitter)
       : MCELFStreamer(Context, std::move(TAB), std::move(OW),
                       std::move(Emitter)),
-        MappingSymbolCounter(0), LastEMS(EMS_None) {}
+        LastEMS(EMS_None) {}
 
   void changeSection(MCSection *Section, uint32_t Subsection = 0) override {
     // We have to keep track of the mapping symbol state of any sections we
@@ -195,7 +195,6 @@ class AArch64ELFStreamer : public MCELFStreamer {
 
   // Reset state between object emissions
   void reset() override {
-    MappingSymbolCounter = 0;
     MCELFStreamer::reset();
     LastMappingSymbols.clear();
     LastEMS = EMS_None;
@@ -271,15 +270,12 @@ class AArch64ELFStreamer : public MCELFStreamer {
   }
 
   void emitMappingSymbol(StringRef Name) {
-    auto *Symbol = cast<MCSymbolELF>(getContext().getOrCreateSymbol(
-        Name + "." + Twine(MappingSymbolCounter++)));
+    auto *Symbol = cast<MCSymbolELF>(getContext().createLocalSymbol(Name));
     emitLabel(Symbol);
     Symbol->setType(ELF::STT_NOTYPE);
     Symbol->setBinding(ELF::STB_LOCAL);
   }
 
-  int64_t MappingSymbolCounter;
-
   DenseMap<const MCSection *, ElfMappingSymbol> LastMappingSymbols;
   ElfMappingSymbol LastEMS;
 };
diff --git a/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll b/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
index e601f03d524a4..8b6740667a229 100644
--- a/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
+++ b/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
@@ -7,10 +7,10 @@
 @l = common hidden local_unnamed_addr global i32 0, align 4
 
 ; CHECK-LABEL: <test1>:
-; CHECK-LABEL: <$d.1>:
-; CHECK-LABEL: <$x.2>:
+; CHECK-LABEL: <$d>:
+; CHECK-LABEL: <$x>:
 ; CHECK-NEXT:    b 0x2c <test1+0x2c>
-; CHECK-LABEL: <$x.4>:
+; CHECK-LABEL: <$x>:
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldr x30, [sp], #16
 ; CHECK-NEXT:    ret
@@ -41,8 +41,8 @@ declare dso_local i32 @i(...) local_unnamed_addr
 
 ; CHECK-LABEL: <test2>:
 ; CHECK:         b {{.*}} <test2+0x1c>
-; CHECK-LABEL: <$d.5>:
-; CHECK-LABEL: <$x.6>:
+; CHECK-LABEL: <$d>:
+; CHECK-LABEL: <$x>:
 ; CHECK-NEXT:    b {{.*}} <test2+0x18>
 define hidden i32 @test2() local_unnamed_addr {
   %1 = load i32, ptr @l, align 4
@@ -70,10 +70,10 @@ define hidden i32 @test2() local_unnamed_addr {
 }
 
 ; CHECK-LABEL: <test3>:
-; CHECK-LABEL: <$d.9>:
-; CHECK-LABEL: <$x.10>:
+; CHECK-LABEL: <$d>:
+; CHECK-LABEL: <$x>:
 ; CHECK-NEXT:    b {{.*}} <test3+0x34>
-; CHECK-LABEL: <$x.12>:
+; CHECK-LABEL: <$x>:
 ; CHECK-NEXT:    ldr x30, [sp], #16
 ; CHECK-NEXT:    ret
 define internal i1 @test3() {
diff --git a/llvm/test/CodeGen/AArch64/inlineasm-ldr-pseudo.ll b/llvm/test/CodeGen/AArch64/inlineasm-ldr-pseudo.ll
index f2b5e5f0064b7..f3623928d67bf 100644
--- a/llvm/test/CodeGen/AArch64/inlineasm-ldr-pseudo.ll
+++ b/llvm/test/CodeGen/AArch64/inlineasm-ldr-pseudo.ll
@@ -16,7 +16,7 @@ entry:
 ; CHECK:        58000040                                         ldr    x0, 0x10
 ; CHECK:        d65f03c0                                         ret
 ; Make sure the constant pool entry comes after the return
-; CHECK-LABEL:        <$d.1>:
+; CHECK-LABEL:        <$d>:
 define i32 @bar() nounwind {
 entry:
   %0 = tail call i32 asm sideeffect "ldr $0,=0x10001", "=r"() nounwind
diff --git a/llvm/test/DebugInfo/Symbolize/ELF/aarch64-mapping-symbol.s b/llvm/test/DebugInfo/Symbolize/ELF/aarch64-mapping-symbol.s
index 2f1140cf62910..6990a82c4acfe 100644
--- a/llvm/test/DebugInfo/Symbolize/ELF/aarch64-mapping-symbol.s
+++ b/llvm/test/DebugInfo/Symbolize/ELF/aarch64-mapping-symbol.s
@@ -7,9 +7,9 @@
 ## addresses.
 # RUN: llvm-nm --special-syms %t | FileCheck %s -check-prefix MAPPING_SYM
 
-# MAPPING_SYM:      0000000000000000 t $d.0
-# MAPPING_SYM-NEXT: 000000000000000c t $d.2
-# MAPPING_SYM-NEXT: 0000000000000004 t $x.1
+# MAPPING_SYM:      0000000000000000 t $d
+# MAPPING_SYM-NEXT: 000000000000000c t $d
+# MAPPING_SYM-NEXT: 0000000000000004 t $x
 # MAPPING_SYM-NEXT: 0000000000000000 T foo
 
 # RUN: llvm-symbolizer --obj=%t 0 4 0xc | FileCheck %s -check-prefix SYMBOL
diff --git a/llvm/test/MC/AArch64/CheckDataSymbol.s b/llvm/test/MC/AArch64/CheckDataSymbol.s
index baa1158984386..13d1db7da091b 100644
--- a/llvm/test/MC/AArch64/CheckDataSymbol.s
+++ b/llvm/test/MC/AArch64/CheckDataSymbol.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc -filetype=obj -assemble \
 # RUN: -triple=aarch64- %s -o - \
 # RUN: | llvm-readobj -S --symbols - | FileCheck %s
-# CHECK:     Name: $d.1 ({{[1-9][0-9]+}})
+# CHECK:     Name: $d ({{[1-9][0-9]+}})
 # CHECK-NEXT:     Value: 0x4
 # CHECK-NEXT:     Size: 0
 # CHECK-NEXT:     Binding: Local (0x0)
diff --git a/llvm/test/MC/AArch64/mapping-across-sections.s b/llvm/test/MC/AArch64/mapping-across-sections.s
index 6bb5a8811b57d..f453c86d45fb6 100644
--- a/llvm/test/MC/AArch64/mapping-across-sections.s
+++ b/llvm/test/MC/AArch64/mapping-across-sections.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple=aarch64 -filetype=obj %s | llvm-objdump -t - | FileCheck %s
+// RUN: llvm-mc -triple=aarch64 -filetype=obj %s | llvm-objdump -t - | FileCheck %s --match-full-lines
 
 .section .text1,"ax"
 add w0, w0, w0
@@ -28,13 +28,13 @@ add w0, w0, w0
 .section ".note.GNU-stack","",@progbits
 
 // CHECK:      SYMBOL TABLE:
-// CHECK-NEXT: 0000000000000000 l       .text1 0000000000000000 $x.0
-// CHECK-NEXT: 0000000000000000 l       .text  0000000000000000 $x.1
-// CHECK-NEXT: 0000000000000004 l       .text  0000000000000000 $d.2
-// CHECK-NEXT: 0000000000000000 l       .data  0000000000000000 $d.3
-// CHECK-NEXT: 0000000000000008 l       .text  0000000000000000 $x.4
-// CHECK-NEXT: 000000000000000c l       .text  0000000000000000 $d.5
-// CHECK-NEXT: 0000000000000000 l       .rodata        0000000000000000 $d.6
-// CHECK-NEXT: 0000000000000004 l       .rodata        0000000000000000 $x.7
-// CHECK-NEXT: 0000000000000000 l       .comment       0000000000000000 $d.8
+// CHECK-NEXT: 0000000000000000 l       .text1 0000000000000000 $x
+// CHECK-NEXT: 0000000000000000 l       .text  0000000000000000 $x
+// CHECK-NEXT: 0000000000000004 l       .text  0000000000000000 $d
+// CHECK-NEXT: 0000000000000000 l       .data  0000000000000000 $d
+// CHECK-NEXT: 0000000000000008 l       .text  0000000000000000 $x
+// CHECK-NEXT: 000000000000000c l       .text  0000000000000000 $d
+// CHECK-NEXT: 0000000000000000 l       .rodata        0000000000000000 $d
+// CHECK-NEXT: 0000000000000004 l       .rodata        0000000000000000 $x
+// CHECK-NEXT: 0000000000000000 l       .comment       0000000000000000 $d
 // CHECK-NOT:  {{.}}
diff --git a/llvm/test/MC/AArch64/mapping-within-section.s b/llvm/test/MC/AArch64/mapping-within-section.s
index 07a2d3cc0fed9..c84e3a4d2fe64 100644
--- a/llvm/test/MC/AArch64/mapping-within-section.s
+++ b/llvm/test/MC/AArch64/mapping-within-section.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s | llvm-nm --special-syms - | FileCheck %s
+// RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s | llvm-nm --no-sort --special-syms - | FileCheck %s --match-full-lines
 
     .text
 // $x at 0x0000
@@ -18,6 +18,13 @@
 // $x at 0x0018
     add x0, x0, x0
 
-// CHECK:      0000000000000004 t $d.1
-// CHECK-NEXT: 0000000000000000 t $x.0
-// CHECK-NEXT: 0000000000000064 t $x.2
+.globl $d
+$d:
+$x:
+
+// CHECK:      0000000000000000 t $x
+// CHECK-NEXT: 0000000000000004 t $d
+// CHECK-NEXT: 0000000000000064 t $x
+// CHECK-NEXT: 0000000000000068 t $x
+// CHECK-NEXT: 0000000000000068 T $d
+// CHECK-NOT:  {{.}}
diff --git a/llvm/test/MC/AArch64/size-directive.s b/llvm/test/MC/AArch64/size-directive.s
index 0b19cda4eaa1e..d9bdf970cf68b 100644
--- a/llvm/test/MC/AArch64/size-directive.s
+++ b/llvm/test/MC/AArch64/size-directive.s
@@ -32,7 +32,7 @@ aarch64_size:
 // CHECK-OBJ-NEXT: )
 
 // SYMS:      Type   Bind   Vis     Ndx Name
-// SYMS:      NOTYPE LOCAL  DEFAULT   3 $d.0
+// SYMS:      NOTYPE LOCAL  DEFAULT   3 $d{{$}}
 // SYMS-NEXT: FUNC   GLOBAL DEFAULT   3 aarch64_size
 // SYMS-NEXT: NOTYPE GLOBAL DEFAULT UND half_word
 // SYMS-NEXT: NOTYPE GLOBAL DEFAULT UND full_word
diff --git a/llvm/test/MC/ELF/AArch64/cfi.s b/llvm/test/MC/ELF/AArch64/cfi.s
index 033c8d9c04094..6bdf03cc7bb85 100644
--- a/llvm/test/MC/ELF/AArch64/cfi.s
+++ b/llvm/test/MC/ELF/AArch64/cfi.s
@@ -227,7 +227,7 @@ f37:
         .cfi_endproc
 
 // CHECK:       Section {
-// CHECK:         Name: .eh_frame (20)
+// CHECK:         Name: .eh_frame
 // CHECK-NEXT:    Type: SHT_PROGBITS (0x1)
 // CHECK-NEXT:    Flags [ (0x2)
 // CHECK-NEXT:      SHF_ALLOC (0x2)
@@ -355,7 +355,7 @@ f37:
 // CHECK-NEXT:    )
 // CHECK-NEXT:  }
 // CHECK:       Section {
-// CHECK:         Name: .rela.eh_frame (15)
+// CHECK:         Name: .rela.eh_frame
 // CHECK-NEXT:    Type: SHT_RELA (0x4)
 // CHECK-NEXT:    Flags [ (0x40)
 // CHECK-NEXT:      SHF_INFO_LINK (0x40)
diff --git a/llvm/test/tools/llvm-objdump/ELF/AArch64/elf-aarch64-mapping-symbols.test b/llvm/test/tools/llvm-objdump/ELF/AArch64/elf-aarch64-mapping-symbols.test
index cf24952a2d8d4..011e67cda8c32 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AArch64/elf-aarch64-mapping-symbols.test
+++ b/llvm/test/tools/llvm-objdump/ELF/AArch64/elf-aarch64-mapping-symbols.test
@@ -4,11 +4,11 @@
 
 # CHECK:       Disassembly of section .mysection:
 # CHECK-EMPTY:
-# ALL-NEXT:    <$x.0>:
+# ALL-NEXT:    <$x>:
 # CHECK-NEXT:  <_start>:
 # CHECK-NEXT:         0:       10000021        adr     x1, 0x4
 # CHECK-EMPTY:
-# ALL-NEXT:    <$d.1>:
+# ALL-NEXT:    <$d>:
 # CHECK-NEXT:  <msg>:
 # CHECK-NEXT:         4:       48 65 6c 6c     .word
 # CHECK-NEXT:         8:       6f 2c 20 77     .word
@@ -18,10 +18,10 @@
 # CHECK-NEXT:  Disassembly of section .myothersection:
 # CHECK-EMPTY:
 # NOALL-NEXT:  <.myothersection>:
-# ALL-NEXT:    <$x.2>:
+# ALL-NEXT:    <$x>:
 # CHECK-NEXT:         0:       90000001        adrp    x1, 0x0
 # CHECK-EMPTY:
-# ALL-NEXT:    <$d.3>:
+# ALL-NEXT:    <$d>:
 # CHECK-NEXT:  <mystr>:
 # CHECK-NEXT:         4:       62 6c 61 68     .word
 # CHECK-NEXT:         8:       9a              .byte   0x9a

From ec7f8e11133ff6cd0370398ee32d684a6f857fcc Mon Sep 17 00:00:00 2001
From: Jessica Del <50999226+OutOfCache@users.noreply.github.com>
Date: Mon, 22 Jul 2024 18:04:49 +0200
Subject: [PATCH 439/486] [AMDGPU] Add intrinsic for raw atomic buffer loads
 (#97707)

Upstream the intrinsics `llvm.amdgcn.raw.atomic.buffer.load`
and `llvm.amdgcn.raw.atomic.ptr.buffer.load`.

These additional intrinsics mark atomic buffer loads
as atomic to LLVM by removing the `IntrReadMem`
attribute. Otherwise, it could hoist these
intrinsics out of loops in cases where LLVM marks
them as invariant. That can cause issues such as
infinite loops.

Continuation of https://reviews.llvm.org/D138786
with the additional use in the fat buffer lowering,
more test cases and the additional ptr versions
of these intrinsics.

---------

Co-authored-by: rtayl <>
Co-authored-by: Jay Foad <jay.foad@amd.com>
Co-authored-by: Mariusz Sikora <mariusz.sikora@amd.com>
---
 llvm/include/llvm/IR/IntrinsicsAMDGPU.td      |  26 ++
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |   2 +
 .../AMDGPU/AMDGPULowerBufferFatPointers.cpp   |   5 +-
 .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp  |   2 +
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  10 +
 .../llvm.amdgcn.raw.atomic.buffer.load.ll     | 304 ++++++++++++++++++
 .../llvm.amdgcn.raw.ptr.atomic.buffer.load.ll | 304 ++++++++++++++++++
 .../lower-buffer-fat-pointers-memops.ll       |   6 +-
 8 files changed, 654 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll

diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index ca85ff30f683f..ab2620fdcf6b3 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1138,6 +1138,19 @@ class AMDGPURawBufferLoad<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntrinsi
 def int_amdgcn_raw_buffer_load_format : AMDGPURawBufferLoad<llvm_anyfloat_ty>;
 def int_amdgcn_raw_buffer_load : AMDGPURawBufferLoad;
 
+class AMDGPURawAtomicBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic <
+  [data_ty],
+  [llvm_v4i32_ty,     // rsrc(SGPR)
+   llvm_i32_ty,       // offset(VGPR/imm, included in bounds checking and swizzling)
+   llvm_i32_ty,       // soffset(SGPR/imm, excluded from bounds checking and swizzling)
+   llvm_i32_ty],      // auxiliary data (imm, cachepolicy     (bit 0 = glc,
+                      //                                       bit 1 = slc,
+                      //                                       bit 2 = dlc on gfx10+),
+                      //                      swizzled buffer (bit 3 = swz))
+  [ImmArg<ArgIndex<3>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
+  AMDGPURsrcIntrinsic<0>;
+def int_amdgcn_raw_atomic_buffer_load : AMDGPURawAtomicBufferLoad;
+
 class AMDGPURawPtrBufferLoad<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntrinsic <
   [data_ty],
   [AMDGPUBufferRsrcTy,    // rsrc(SGPR)
@@ -1156,6 +1169,19 @@ class AMDGPURawPtrBufferLoad<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntri
 def int_amdgcn_raw_ptr_buffer_load_format : AMDGPURawPtrBufferLoad<llvm_anyfloat_ty>;
 def int_amdgcn_raw_ptr_buffer_load : AMDGPURawPtrBufferLoad;
 
+class AMDGPURawPtrAtomicBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic <
+  [data_ty],
+  [AMDGPUBufferRsrcTy,// rsrc(SGPR)
+   llvm_i32_ty,       // offset(VGPR/imm, included in bounds checking and swizzling)
+   llvm_i32_ty,       // soffset(SGPR/imm, excluded from bounds checking and swizzling)
+   llvm_i32_ty],      // auxiliary data (imm, cachepolicy     (bit 0 = glc,
+                      //                                       bit 1 = slc,
+                      //                                       bit 2 = dlc on gfx10+),
+                      //                      swizzled buffer (bit 3 = swz))
+  [IntrArgMemOnly, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<3>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
+  AMDGPURsrcIntrinsic<0>;
+def int_amdgcn_raw_ptr_atomic_buffer_load : AMDGPURawPtrAtomicBufferLoad;
+
 class AMDGPUStructBufferLoad<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntrinsic <
   [data_ty],
   [llvm_v4i32_ty,    // rsrc(SGPR)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 89ef0f299feab..74e93b0620d26 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -7366,6 +7366,8 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
     return legalizeBufferStore(MI, MRI, B, true, true);
   case Intrinsic::amdgcn_raw_buffer_load:
   case Intrinsic::amdgcn_raw_ptr_buffer_load:
+  case Intrinsic::amdgcn_raw_atomic_buffer_load:
+  case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
   case Intrinsic::amdgcn_struct_buffer_load:
   case Intrinsic::amdgcn_struct_ptr_buffer_load:
     return legalizeBufferLoad(MI, MRI, B, false, false);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 6be9be21a8a86..77971323aa1ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1092,8 +1092,9 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
 
   Intrinsic::ID IID = Intrinsic::not_intrinsic;
   if (isa<LoadInst>(I))
-    // TODO: Do we need to do something about atomic loads?
-    IID = Intrinsic::amdgcn_raw_ptr_buffer_load;
+    IID = Order == AtomicOrdering::NotAtomic
+              ? Intrinsic::amdgcn_raw_ptr_buffer_load
+              : Intrinsic::amdgcn_raw_ptr_atomic_buffer_load;
   else if (isa<StoreInst>(I))
     IID = Intrinsic::amdgcn_raw_ptr_buffer_store;
   else if (auto *RMW = dyn_cast<AtomicRMWInst>(I)) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 68f4767458703..aa329a58547f3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -4985,6 +4985,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     }
     case Intrinsic::amdgcn_raw_buffer_load:
     case Intrinsic::amdgcn_raw_ptr_buffer_load:
+    case Intrinsic::amdgcn_raw_atomic_buffer_load:
+    case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
     case Intrinsic::amdgcn_raw_tbuffer_load:
     case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
       // FIXME: Should make intrinsic ID the last operand of the instruction,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a09e0ad2c0c29..7f95442401dbc 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1277,6 +1277,14 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
         Info.ptrVal = CI.getArgOperand(1);
         return true;
       }
+      case Intrinsic::amdgcn_raw_atomic_buffer_load:
+      case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load: {
+        Info.memVT =
+            memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
+                                    std::numeric_limits<unsigned>::max());
+        Info.flags &= ~MachineMemOperand::MOStore;
+        return true;
+      }
       }
     }
     return true;
@@ -8889,6 +8897,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
   }
   case Intrinsic::amdgcn_raw_buffer_load:
   case Intrinsic::amdgcn_raw_ptr_buffer_load:
+  case Intrinsic::amdgcn_raw_atomic_buffer_load:
+  case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
   case Intrinsic::amdgcn_raw_buffer_load_format:
   case Intrinsic::amdgcn_raw_ptr_buffer_load_format: {
     const bool IsFormat =
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
new file mode 100644
index 0000000000000..03f94d6e853f0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
@@ -0,0 +1,304 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=amdgcn -mcpu=gfx1100 -global-isel=0 | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -march=amdgcn -mcpu=gfx1100 -global-isel=1 | FileCheck %s -check-prefix=CHECK
+
+define amdgpu_kernel void @raw_atomic_buffer_load_i32(<4 x i32> %addr) {
+; CHECK-LABEL: raw_atomic_buffer_load_i32:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB0_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB0_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 0, i32 0, i32 1)
+  %cmp = icmp eq i32 %load, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_atomic_buffer_load_i32_off(<4 x i32> %addr) {
+; CHECK-LABEL: raw_atomic_buffer_load_i32_off:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB1_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB1_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 0, i32 0, i32 1)
+  %cmp = icmp eq i32 %load, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+define amdgpu_kernel void @raw_atomic_buffer_load_i32_soff(<4 x i32> %addr) {
+; CHECK-LABEL: raw_atomic_buffer_load_i32_soff:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB2_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB2_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 4, i32 4, i32 1)
+  %cmp = icmp eq i32 %load, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+define amdgpu_kernel void @raw_atomic_buffer_load_i32_dlc(<4 x i32> %addr) {
+; CHECK-LABEL: raw_atomic_buffer_load_i32_dlc:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB3_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB3_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 4, i32 0, i32 4)
+  %cmp = icmp eq i32 %load, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_nonatomic_buffer_load_i32(<4 x i32> %addr) {
+; CHECK-LABEL: raw_nonatomic_buffer_load_i32:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:  .LBB4_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; CHECK-NEXT:    s_or_b32 s0, s1, s0
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; CHECK-NEXT:    s_cbranch_execnz .LBB4_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %addr, i32 4, i32 0, i32 1)
+  %cmp = icmp eq i32 %load, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_atomic_buffer_load_i64(<4 x i32> %addr) {
+; CHECK-LABEL: raw_atomic_buffer_load_i64:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB5_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB5_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %id.zext = zext i32 %id to i64
+  br label %bb1
+bb1:
+  %load = call i64 @llvm.amdgcn.raw.atomic.buffer.load.i64(<4 x i32> %addr, i32 4, i32 0, i32 1)
+  %cmp = icmp eq i64 %load, %id.zext
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_atomic_buffer_load_v2i16(<4 x i32> %addr) {
+; CHECK-LABEL: raw_atomic_buffer_load_v2i16:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB6_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB6_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call <2 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v2i16(<4 x i32> %addr, i32 0, i32 0, i32 1)
+  %bitcast = bitcast <2 x i16> %load to i32
+  %cmp = icmp eq i32 %bitcast, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
+; CHECK-LABEL: raw_atomic_buffer_load_v4i16:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB7_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB7_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call <4 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v4i16(<4 x i32> %addr, i32 4, i32 0, i32 1)
+  %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
+  %bitcast = bitcast <2 x i16> %shortened to i32
+  %cmp = icmp eq i32 %bitcast, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_atomic_buffer_load_v4i32(<4 x i32> %addr) {
+; CHECK-LABEL: raw_atomic_buffer_load_v4i32:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB8_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB8_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call <4 x i32> @llvm.amdgcn.raw.atomic.buffer.load.v4i32(<4 x i32> %addr, i32 4, i32 0, i32 1)
+  %extracted = extractelement <4 x i32> %load, i32 3
+  %cmp = icmp eq i32 %extracted, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_atomic_buffer_load_ptr(<4 x i32> %addr) {
+; CHECK-LABEL: raw_atomic_buffer_load_ptr:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB9_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_load_b32 v1, v[1:2]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB9_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call ptr @llvm.amdgcn.raw.atomic.buffer.load.ptr(<4 x i32> %addr, i32 4, i32 0, i32 1)
+  %elem = load i32, ptr %load
+  %cmp = icmp eq i32 %elem, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+; Function Attrs: nounwind readonly
+declare i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32>, i32, i32, i32 immarg)
+declare i64 @llvm.amdgcn.raw.atomic.buffer.load.i64(<4 x i32>, i32, i32, i32 immarg)
+declare <2 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v2i16(<4 x i32>, i32, i32, i32 immarg)
+declare <4 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v4i16(<4 x i32>, i32, i32, i32 immarg)
+declare <4 x i32> @llvm.amdgcn.raw.atomic.buffer.load.v4i32(<4 x i32>, i32, i32, i32 immarg)
+declare ptr @llvm.amdgcn.raw.atomic.buffer.load.ptr(<4 x i32>, i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32>, i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
new file mode 100644
index 0000000000000..3228335073d07
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
@@ -0,0 +1,304 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=amdgcn -mcpu=gfx1100 -global-isel=0 | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -march=amdgcn -mcpu=gfx1100 -global-isel=1 | FileCheck %s -check-prefix=CHECK
+
+define amdgpu_kernel void @raw_ptr_atomic_buffer_ptr_load_i32(ptr addrspace(8) %ptr) {
+; CHECK-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB0_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB0_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 0, i32 0, i32 1)
+  %cmp = icmp eq i32 %load, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_off(ptr addrspace(8) %ptr) {
+; CHECK-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB1_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB1_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 0, i32 0, i32 1)
+  %cmp = icmp eq i32 %load, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_soff(ptr addrspace(8) %ptr) {
+; CHECK-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB2_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB2_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 4, i32 4, i32 1)
+  %cmp = icmp eq i32 %load, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8) %ptr) {
+; CHECK-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB3_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB3_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 4, i32 0, i32 4)
+  %cmp = icmp eq i32 %load, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_nonptr_atomic_buffer_load_i32(ptr addrspace(8) %ptr) {
+; CHECK-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:  .LBB4_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; CHECK-NEXT:    s_or_b32 s0, s1, s0
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; CHECK-NEXT:    s_cbranch_execnz .LBB4_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
+  %cmp = icmp eq i32 %load, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr) {
+; CHECK-LABEL: raw_ptr_atomic_buffer_load_i64:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB5_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB5_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %id.zext = zext i32 %id to i64
+  br label %bb1
+bb1:
+  %load = call i64 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i64(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
+  %cmp = icmp eq i64 %load, %id.zext
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v2i16(ptr addrspace(8) %ptr) {
+; CHECK-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB6_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB6_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call <2 x i16> @llvm.amdgcn.raw.ptr.atomic.buffer.load.v2i16(ptr addrspace(8) %ptr, i32 0, i32 0, i32 1)
+  %bitcast = bitcast <2 x i16> %load to i32
+  %cmp = icmp eq i32 %bitcast, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %ptr) {
+; CHECK-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB7_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB7_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call <4 x i16> @llvm.amdgcn.raw.ptr.atomic.buffer.load.v4i16(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
+  %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
+  %bitcast = bitcast <2 x i16> %shortened to i32
+  %cmp = icmp eq i32 %bitcast, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i32(ptr addrspace(8) %ptr) {
+; CHECK-LABEL: raw_ptr_atomic_buffer_load_v4i32:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB8_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB8_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call <4 x i32> @llvm.amdgcn.raw.ptr.atomic.buffer.load.v4i32(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
+  %extracted = extractelement <4 x i32> %load, i32 3
+  %cmp = icmp eq i32 %extracted, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %ptr) {
+; CHECK-LABEL: raw_ptr_atomic_buffer_load_ptr:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  .LBB9_1: ; %bb1
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_load_b32 v1, v[1:2]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execnz .LBB9_1
+; CHECK-NEXT:  ; %bb.2: ; %bb2
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb1
+bb1:
+  %load = call ptr @llvm.amdgcn.raw.ptr.atomic.buffer.load.ptr(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
+  %elem = load i32, ptr %load
+  %cmp = icmp eq i32 %elem, %id
+  br i1 %cmp, label %bb1, label %bb2
+bb2:
+  ret void
+}
+
+; Function Attrs: nounwind readonly
+declare i32 @llvm.amdgcn.raw.ptr.atom.buffer.load.i32(ptr addrspace(8), i32, i32, i32 immarg)
+declare i64 @llvm.amdgcn.raw.ptr.atom.buffer.load.i64(ptr addrspace(8), i32, i32, i32 immarg)
+declare <2 x i16> @llvm.amdgcn.raw.ptr.atom.buffer.load.v2i16(ptr addrspace(8), i32, i32, i32 immarg)
+declare <4 x i16> @llvm.amdgcn.raw.ptr.atom.buffer.load.v4i16(ptr addrspace(8), i32, i32, i32 immarg)
+declare <4 x i32> @llvm.amdgcn.raw.ptr.atom.buffer.load.v4i32(ptr addrspace(8), i32, i32, i32 immarg)
+declare ptr @llvm.amdgcn.raw.ptr.atom.buffer.load.ptr(ptr addrspace(8), i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8), i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
index 66c68f7cc731e..57028a0f9b14f 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
@@ -17,10 +17,10 @@ define void @loads(ptr addrspace(8) %buf) {
 ; CHECK-NEXT:    [[VOLATILE:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
 ; CHECK-NEXT:    [[VOLATILE_NONTEMPORAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483646), !nontemporal [[META0]]
 ; CHECK-NEXT:    fence syncscope("wavefront") release
-; CHECK-NEXT:    [[ATOMIC:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483647)
+; CHECK-NEXT:    [[ATOMIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483647)
 ; CHECK-NEXT:    fence syncscope("wavefront") acquire
-; CHECK-NEXT:    [[ATOMIC_MONOTONIC:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 1)
-; CHECK-NEXT:    [[ATOMIC_ACQUIRE:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 1)
+; CHECK-NEXT:    [[ATOMIC_MONOTONIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 1)
+; CHECK-NEXT:    [[ATOMIC_ACQUIRE:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 1)
 ; CHECK-NEXT:    fence acquire
 ; CHECK-NEXT:    ret void
 ;

From b1ca2a954643d2c07d5308297d1f2b911f794ba4 Mon Sep 17 00:00:00 2001
From: xur-llvm <59886942+xur-llvm@users.noreply.github.com>
Date: Mon, 22 Jul 2024 09:19:17 -0700
Subject: [PATCH 440/486] [PGO] Sampled instrumentation in PGO to speed up
 instrumentation binary (#69535)

In comparison to non-instrumented binaries, PGO instrumentation binaries
can be significantly slower. For highly threaded programs, this slowdown
can
reach 10x due to data races or false sharing within counters.

This patch incorporates sampling into the PGO instrumentation process to
enhance the speed of instrumentation binaries. The fundamental concept
is similar to the one proposed in https://reviews.llvm.org/D63949.

Three sampling modes are introduced:
1. Simple Sampling: When '-sampled-instr-bust-duration' is set to 1.
2. Fast Burst Sampling: When not using simple sampling, and
'-sampled-instr-period' is set to 65535. This is the default mode of
sampling.
3. Full Burst Sampling: When neither simple nor fast burst sampling is
used.

Utilizing this sampled instrumentation significantly improves the
binary's
execution speed. Measurements show up to 5x speedup with default
settings. Fast burst sampling now results in only around 20% to 30%
slowdown (compared to 8 to 10x slowdown without sampling).

Out tests show that profile quality remains good with sampling,
with edge counts typically showing more than 90% overlap.
For applications whose behavior changes due to binary speed,
sampling instrumentation can enhance performance.
Observations have shown some apps experiencing up to
a ~2% improvement in PGO.

A potential drawback of this patch is the increased binary size
and compilation time. The Sampling method in this patch does
not improve single threaded program instrumentation binary
speed.
---
 .../llvm/ProfileData/InstrProfData.inc        |   1 +
 .../include/llvm/Transforms/Instrumentation.h |   6 +
 .../Instrumentation/PGOInstrumentation.h      |   6 +-
 llvm/lib/Passes/PassBuilderPipelines.cpp      |  12 +-
 .../Instrumentation/InstrProfiling.cpp        | 245 ++++++++++++++++--
 .../Instrumentation/PGOInstrumentation.cpp    |   2 +
 .../PGOProfile/Inputs/cspgo_bar_sample.ll     |  82 ++++++
 .../PGOProfile/counter_promo_sampling.ll      |  78 ++++++
 .../Transforms/PGOProfile/cspgo_sample.ll     | 112 ++++++++
 .../instrprof_burst_sampling_fast.ll          |  47 ++++
 .../instrprof_burst_sampling_full.ll          |  45 ++++
 .../instrprof_burst_sampling_full_intsize.ll  |  45 ++++
 .../PGOProfile/instrprof_simple_sampling.ll   |  60 +++++
 13 files changed, 716 insertions(+), 25 deletions(-)
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/cspgo_bar_sample.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/counter_promo_sampling.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/cspgo_sample.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_fast.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_full.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_full_intsize.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/instrprof_simple_sampling.ll

diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index e9866d94b762c..847e53cfa7432 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -739,6 +739,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
 #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
 #define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
+#define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling
 
 /* The variable that holds the name of the profile data
  * specified via command line. */
diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h
index ea97ab2562a5b..969c2cd12f3f0 100644
--- a/llvm/include/llvm/Transforms/Instrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation.h
@@ -121,12 +121,18 @@ struct InstrProfOptions {
   // Use BFI to guide register promotion
   bool UseBFIInPromotion = false;
 
+  // Use sampling to reduce the profile instrumentation runtime overhead.
+  bool Sampling = false;
+
   // Name of the profile file to use as output
   std::string InstrProfileOutput;
 
   InstrProfOptions() = default;
 };
 
+// Create the variable for profile sampling.
+void createProfileSamplingVar(Module &M);
+
 // Options for sanitizer coverage instrumentation.
 struct SanitizerCoverageOptions {
   enum Type {
diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
index 5b1977b7de9a2..7199f27dbc991 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
@@ -43,12 +43,14 @@ class FileSystem;
 class PGOInstrumentationGenCreateVar
     : public PassInfoMixin<PGOInstrumentationGenCreateVar> {
 public:
-  PGOInstrumentationGenCreateVar(std::string CSInstrName = "")
-      : CSInstrName(CSInstrName) {}
+  PGOInstrumentationGenCreateVar(std::string CSInstrName = "",
+                                 bool Sampling = false)
+      : CSInstrName(CSInstrName), ProfileSampling(Sampling) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 
 private:
   std::string CSInstrName;
+  bool ProfileSampling;
 };
 
 /// The instrumentation (profile-instr-gen) pass for IR based PGO.
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 4fd5ee1946bb7..935504b070d2e 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -296,6 +296,9 @@ static cl::opt<AttributorRunOption> AttributorRun(
                clEnumValN(AttributorRunOption::NONE, "none",
                           "disable attributor runs")));
 
+static cl::opt<bool> EnableSampledInstr(
+    "enable-sampled-instrumentation", cl::init(false), cl::Hidden,
+    cl::desc("Enable profile instrumentation sampling (default = off)"));
 static cl::opt<bool> UseLoopVersioningLICM(
     "enable-loop-versioning-licm", cl::init(false), cl::Hidden,
     cl::desc("Enable the experimental Loop Versioning LICM pass"));
@@ -847,6 +850,12 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
   // Do counter promotion at Level greater than O0.
   Options.DoCounterPromotion = true;
   Options.UseBFIInPromotion = IsCS;
+  if (EnableSampledInstr) {
+    Options.Sampling = true;
+    // With sampling, there is little beneifit to enable counter promotion.
+    // But note that sampling does work with counter promotion.
+    Options.DoCounterPromotion = false;
+  }
   Options.Atomic = AtomicCounterUpdate;
   MPM.addPass(InstrProfilingLoweringPass(Options, IsCS));
 }
@@ -1185,7 +1194,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
     MPM.addPass(PGOIndirectCallPromotion(false, false));
 
   if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr)
-    MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
+    MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile,
+                                               EnableSampledInstr));
 
   if (IsMemprofUse)
     MPM.addPass(MemProfUsePass(PGOOpt->MemoryProfile, PGOOpt->FS));
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 09c56eb5fe6aa..d1396071c0bad 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -170,6 +170,30 @@ cl::opt<bool> SkipRetExitBlock(
     "skip-ret-exit-block", cl::init(true),
     cl::desc("Suppress counter promotion if exit blocks contain ret."));
 
+static cl::opt<bool> SampledInstr("sampled-instrumentation", cl::ZeroOrMore,
+                                  cl::init(false),
+                                  cl::desc("Do PGO instrumentation sampling"));
+
+static cl::opt<unsigned> SampledInstrPeriod(
+    "sampled-instr-period",
+    cl::desc("Set the profile instrumentation sample period. For each sample "
+             "period, a fixed number of consecutive samples will be recorded. "
+             "The number is controlled by 'sampled-instr-burst-duration' flag. "
+             "The default sample period of 65535 is optimized for generating "
+             "efficient code that leverages unsigned integer wrapping in "
+             "overflow."),
+    cl::init(65535));
+
+static cl::opt<unsigned> SampledInstrBurstDuration(
+    "sampled-instr-burst-duration",
+    cl::desc("Set the profile instrumentation burst duration, which can range "
+             "from 0 to one less than the value of 'sampled-instr-period'. "
+             "This number of samples will be recorded for each "
+             "'sampled-instr-period' count update. Setting to 1 enables "
+             "simple sampling, in which case it is recommended to set "
+             "'sampled-instr-period' to a prime number."),
+    cl::init(200));
+
 using LoadStorePair = std::pair<Instruction *, Instruction *>;
 
 static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
@@ -260,6 +284,9 @@ class InstrLowerer final {
   /// Returns true if profile counter update register promotion is enabled.
   bool isCounterPromotionEnabled() const;
 
+  /// Return true if profile sampling is enabled.
+  bool isSamplingEnabled() const;
+
   /// Count the number of instrumented value sites for the function.
   void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
 
@@ -291,6 +318,9 @@ class InstrLowerer final {
   /// acts on.
   Value *getCounterAddress(InstrProfCntrInstBase *I);
 
+  /// Lower the incremental instructions under profile sampling predicates.
+  void doSampling(Instruction *I);
+
   /// Get the region counters for an increment, creating them if necessary.
   ///
   /// If the counter array doesn't yet exist, the profile data variables
@@ -635,33 +665,169 @@ PreservedAnalyses InstrProfilingLoweringPass::run(Module &M,
   return PreservedAnalyses::none();
 }
 
+//
+// Perform instrumentation sampling.
+//
+// There are 3 favors of sampling:
+// (1) Full burst sampling: We transform:
+//   Increment_Instruction;
+// to:
+//   if (__llvm_profile_sampling__ < SampledInstrBurstDuration) {
+//     Increment_Instruction;
+//   }
+//   __llvm_profile_sampling__ += 1;
+//   if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
+//     __llvm_profile_sampling__ = 0;
+//   }
+//
+// "__llvm_profile_sampling__" is a thread-local global shared by all PGO
+// counters (value-instrumentation and edge instrumentation).
+//
+// (2) Fast burst sampling:
+// "__llvm_profile_sampling__" variable is an unsigned type, meaning it will
+// wrap around to zero when overflows. In this case, the second check is
+// unnecessary, so we won't generate check2 when the SampledInstrPeriod is
+// set to 65535 (64K - 1). The code after:
+//   if (__llvm_profile_sampling__ < SampledInstrBurstDuration) {
+//     Increment_Instruction;
+//   }
+//   __llvm_profile_sampling__ += 1;
+//
+// (3) Simple sampling:
+// When SampledInstrBurstDuration sets to 1, we do a simple sampling:
+//   __llvm_profile_sampling__ += 1;
+//   if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
+//     __llvm_profile_sampling__ = 0;
+//     Increment_Instruction;
+//   }
+//
+// Note that, the code snippet after the transformation can still be counter
+// promoted. However, with sampling enabled, counter updates are expected to
+// be infrequent, making the benefits of counter promotion negligible.
+// Moreover, counter promotion can potentially cause issues in server
+// applications, particularly when the counters are dumped without a clean
+// exit. To mitigate this risk, counter promotion is disabled by default when
+// sampling is enabled. This behavior can be overridden using the internal
+// option.
+void InstrLowerer::doSampling(Instruction *I) {
+  if (!isSamplingEnabled())
+    return;
+
+  unsigned SampledBurstDuration = SampledInstrBurstDuration.getValue();
+  unsigned SampledPeriod = SampledInstrPeriod.getValue();
+  if (SampledBurstDuration >= SampledPeriod) {
+    report_fatal_error(
+        "SampledPeriod needs to be greater than SampledBurstDuration");
+  }
+  bool UseShort = (SampledPeriod <= USHRT_MAX);
+  bool IsSimpleSampling = (SampledBurstDuration == 1);
+  // If (SampledBurstDuration == 1 && SampledPeriod == 65535), generate
+  // the simple sampling style code.
+  bool IsFastSampling = (!IsSimpleSampling && SampledPeriod == 65535);
+
+  auto GetConstant = [UseShort](IRBuilder<> &Builder, uint32_t C) {
+    if (UseShort)
+      return Builder.getInt16(C);
+    else
+      return Builder.getInt32(C);
+  };
+
+  IntegerType *SamplingVarTy;
+  if (UseShort)
+    SamplingVarTy = Type::getInt16Ty(M.getContext());
+  else
+    SamplingVarTy = Type::getInt32Ty(M.getContext());
+  auto *SamplingVar =
+      M.getGlobalVariable(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
+  assert(SamplingVar && "SamplingVar not set properly");
+
+  // Create the condition for checking the burst duration.
+  Instruction *SamplingVarIncr;
+  Value *NewSamplingVarVal;
+  MDBuilder MDB(I->getContext());
+  MDNode *BranchWeight;
+  IRBuilder<> CondBuilder(I);
+  auto *LoadSamplingVar = CondBuilder.CreateLoad(SamplingVarTy, SamplingVar);
+  if (IsSimpleSampling) {
+    // For the simple sampling, just create the load and increments.
+    IRBuilder<> IncBuilder(I);
+    NewSamplingVarVal =
+        IncBuilder.CreateAdd(LoadSamplingVar, GetConstant(IncBuilder, 1));
+    SamplingVarIncr = IncBuilder.CreateStore(NewSamplingVarVal, SamplingVar);
+  } else {
+    // For the bust-sampling, create the conditonal update.
+    auto *DurationCond = CondBuilder.CreateICmpULE(
+        LoadSamplingVar, GetConstant(CondBuilder, SampledBurstDuration));
+    BranchWeight = MDB.createBranchWeights(
+        SampledBurstDuration, SampledPeriod + 1 - SampledBurstDuration);
+    Instruction *ThenTerm = SplitBlockAndInsertIfThen(
+        DurationCond, I, /* Unreachable */ false, BranchWeight);
+    IRBuilder<> IncBuilder(I);
+    NewSamplingVarVal =
+        IncBuilder.CreateAdd(LoadSamplingVar, GetConstant(IncBuilder, 1));
+    SamplingVarIncr = IncBuilder.CreateStore(NewSamplingVarVal, SamplingVar);
+    I->moveBefore(ThenTerm);
+  }
+
+  if (IsFastSampling)
+    return;
+
+  // Create the condtion for checking the period.
+  Instruction *ThenTerm, *ElseTerm;
+  IRBuilder<> PeriodCondBuilder(SamplingVarIncr);
+  auto *PeriodCond = PeriodCondBuilder.CreateICmpUGE(
+      NewSamplingVarVal, GetConstant(PeriodCondBuilder, SampledPeriod));
+  BranchWeight = MDB.createBranchWeights(1, SampledPeriod);
+  SplitBlockAndInsertIfThenElse(PeriodCond, SamplingVarIncr, &ThenTerm,
+                                &ElseTerm, BranchWeight);
+
+  // For the simple sampling, the counter update happens in sampling var reset.
+  if (IsSimpleSampling)
+    I->moveBefore(ThenTerm);
+
+  IRBuilder<> ResetBuilder(ThenTerm);
+  ResetBuilder.CreateStore(GetConstant(ResetBuilder, 0), SamplingVar);
+  SamplingVarIncr->moveBefore(ElseTerm);
+}
+
 bool InstrLowerer::lowerIntrinsics(Function *F) {
   bool MadeChange = false;
   PromotionCandidates.clear();
+  SmallVector<InstrProfInstBase *, 8> InstrProfInsts;
+
+  // To ensure compatibility with sampling, we save the intrinsics into
+  // a buffer to prevent potential breakage of the iterator (as the
+  // intrinsics will be moved to a different BB).
   for (BasicBlock &BB : *F) {
     for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
-      if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
-        lowerIncrement(IPIS);
-        MadeChange = true;
-      } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
-        lowerIncrement(IPI);
-        MadeChange = true;
-      } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(&Instr)) {
-        lowerTimestamp(IPC);
-        MadeChange = true;
-      } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
-        lowerCover(IPC);
-        MadeChange = true;
-      } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
-        lowerValueProfileInst(IPVP);
-        MadeChange = true;
-      } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(&Instr)) {
-        IPMP->eraseFromParent();
-        MadeChange = true;
-      } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(&Instr)) {
-        lowerMCDCTestVectorBitmapUpdate(IPBU);
-        MadeChange = true;
-      }
+      if (auto *IP = dyn_cast<InstrProfInstBase>(&Instr))
+        InstrProfInsts.push_back(IP);
+    }
+  }
+
+  for (auto *Instr : InstrProfInsts) {
+    doSampling(Instr);
+    if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(Instr)) {
+      lowerIncrement(IPIS);
+      MadeChange = true;
+    } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(Instr)) {
+      lowerIncrement(IPI);
+      MadeChange = true;
+    } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(Instr)) {
+      lowerTimestamp(IPC);
+      MadeChange = true;
+    } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(Instr)) {
+      lowerCover(IPC);
+      MadeChange = true;
+    } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(Instr)) {
+      lowerValueProfileInst(IPVP);
+      MadeChange = true;
+    } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(Instr)) {
+      IPMP->eraseFromParent();
+      MadeChange = true;
+    } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(Instr)) {
+      lowerMCDCTestVectorBitmapUpdate(IPBU);
+      MadeChange = true;
     }
   }
 
@@ -684,6 +850,12 @@ bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {
   return TT.isOSFuchsia();
 }
 
+bool InstrLowerer::isSamplingEnabled() const {
+  if (SampledInstr.getNumOccurrences() > 0)
+    return SampledInstr;
+  return Options.Sampling;
+}
+
 bool InstrLowerer::isCounterPromotionEnabled() const {
   if (DoCounterPromotion.getNumOccurrences() > 0)
     return DoCounterPromotion;
@@ -754,6 +926,9 @@ bool InstrLowerer::lower() {
   if (NeedsRuntimeHook)
     MadeChange = emitRuntimeHook();
 
+  if (!IsCS && isSamplingEnabled())
+    createProfileSamplingVar(M);
+
   bool ContainsProfiling = containsProfilingIntrinsics(M);
   GlobalVariable *CoverageNamesVar =
       M.getNamedGlobal(getCoverageUnusedNamesVarName());
@@ -1955,3 +2130,29 @@ void InstrLowerer::emitInitialization() {
 
   appendToGlobalCtors(M, F, 0);
 }
+
+namespace llvm {
+// Create the variable for profile sampling.
+void createProfileSamplingVar(Module &M) {
+  const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
+  IntegerType *SamplingVarTy;
+  Constant *ValueZero;
+  if (SampledInstrPeriod.getValue() <= USHRT_MAX) {
+    SamplingVarTy = Type::getInt16Ty(M.getContext());
+    ValueZero = Constant::getIntegerValue(SamplingVarTy, APInt(16, 0));
+  } else {
+    SamplingVarTy = Type::getInt32Ty(M.getContext());
+    ValueZero = Constant::getIntegerValue(SamplingVarTy, APInt(32, 0));
+  }
+  auto SamplingVar = new GlobalVariable(
+      M, SamplingVarTy, false, GlobalValue::WeakAnyLinkage, ValueZero, VarName);
+  SamplingVar->setVisibility(GlobalValue::DefaultVisibility);
+  SamplingVar->setThreadLocal(true);
+  Triple TT(M.getTargetTriple());
+  if (TT.supportsCOMDAT()) {
+    SamplingVar->setLinkage(GlobalValue::ExternalLinkage);
+    SamplingVar->setComdat(M.getOrInsertComdat(VarName));
+  }
+  appendToCompilerUsed(M, SamplingVar);
+}
+} // namespace llvm
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 35b1bbf21be97..4f0ccc8f962db 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1875,6 +1875,8 @@ PGOInstrumentationGenCreateVar::run(Module &M, ModuleAnalysisManager &MAM) {
   // The variable in a comdat may be discarded by LTO. Ensure the declaration
   // will be retained.
   appendToCompilerUsed(M, createIRLevelProfileFlagVar(M, /*IsCS=*/true));
+  if (ProfileSampling)
+    createProfileSamplingVar(M);
   PreservedAnalyses PA;
   PA.preserve<FunctionAnalysisManagerModuleProxy>();
   PA.preserveSet<AllAnalysesOn<Function>>();
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/cspgo_bar_sample.ll b/llvm/test/Transforms/PGOProfile/Inputs/cspgo_bar_sample.ll
new file mode 100644
index 0000000000000..1c8be82715f25
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/cspgo_bar_sample.ll
@@ -0,0 +1,82 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$__llvm_profile_filename = comdat any
+$__llvm_profile_raw_version = comdat any
+$__llvm_profile_sampling = comdat any
+
+@odd = common dso_local local_unnamed_addr global i32 0, align 4
+@even = common dso_local local_unnamed_addr global i32 0, align 4
+@__llvm_profile_filename = local_unnamed_addr constant [25 x i8] c"pass2/default_%m.profraw\00", comdat
+@__llvm_profile_raw_version = local_unnamed_addr constant i64 216172782113783812, comdat
+@__llvm_profile_sampling = thread_local global i16 0, comdat
+@llvm.used = appending global [1 x i8*] [i8* bitcast (i64* @__llvm_profile_sampling to i8*)], section "llvm.metadata"
+
+define dso_local void @bar(i32 %n) !prof !30 {
+entry:
+  %call = tail call fastcc i32 @cond(i32 %n)
+  %tobool = icmp eq i32 %call, 0
+  br i1 %tobool, label %if.else, label %if.then, !prof !31
+
+if.then:
+  %0 = load i32, i32* @odd, align 4, !tbaa !32
+  %inc = add i32 %0, 1
+  store i32 %inc, i32* @odd, align 4, !tbaa !32
+  br label %if.end
+
+if.else:
+  %1 = load i32, i32* @even, align 4, !tbaa !32
+  %inc1 = add i32 %1, 1
+  store i32 %inc1, i32* @even, align 4, !tbaa !32
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define internal fastcc i32 @cond(i32 %i) #1 !prof !30 !PGOFuncName !36 {
+entry:
+  %rem = srem i32 %i, 2
+  ret i32 %rem
+}
+
+attributes #1 = { inlinehint noinline }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"EnableSplitLTOUnit", i32 0}
+!2 = !{i32 1, !"ProfileSummary", !3}
+!3 = !{!4, !5, !6, !7, !8, !9, !10, !11}
+!4 = !{!"ProfileFormat", !"InstrProf"}
+!5 = !{!"TotalCount", i64 500002}
+!6 = !{!"MaxCount", i64 200000}
+!7 = !{!"MaxInternalCount", i64 100000}
+!8 = !{!"MaxFunctionCount", i64 200000}
+!9 = !{!"NumCounts", i64 6}
+!10 = !{!"NumFunctions", i64 4}
+!11 = !{!"DetailedSummary", !12}
+!12 = !{!13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28}
+!13 = !{i32 10000, i64 200000, i32 1}
+!14 = !{i32 100000, i64 200000, i32 1}
+!15 = !{i32 200000, i64 200000, i32 1}
+!16 = !{i32 300000, i64 200000, i32 1}
+!17 = !{i32 400000, i64 200000, i32 1}
+!18 = !{i32 500000, i64 100000, i32 4}
+!19 = !{i32 600000, i64 100000, i32 4}
+!20 = !{i32 700000, i64 100000, i32 4}
+!21 = !{i32 800000, i64 100000, i32 4}
+!22 = !{i32 900000, i64 100000, i32 4}
+!23 = !{i32 950000, i64 100000, i32 4}
+!24 = !{i32 990000, i64 100000, i32 4}
+!25 = !{i32 999000, i64 100000, i32 4}
+!26 = !{i32 999900, i64 100000, i32 4}
+!27 = !{i32 999990, i64 100000, i32 4}
+!28 = !{i32 999999, i64 1, i32 6}
+!30 = !{!"function_entry_count", i64 200000}
+!31 = !{!"branch_weights", i32 100000, i32 100000}
+!32 = !{!33, !33, i64 0}
+!33 = !{!"int", !34, i64 0}
+!34 = !{!"omnipotent char", !35, i64 0}
+!35 = !{!"Simple C/C++ TBAA"}
+!36 = !{!"cspgo_bar.c:cond"}
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo_sampling.ll b/llvm/test/Transforms/PGOProfile/counter_promo_sampling.ll
new file mode 100644
index 0000000000000..9d083fe04015e
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/counter_promo_sampling.ll
@@ -0,0 +1,78 @@
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof --do-counter-promotion=true --sampled-instrumentation=true --skip-ret-exit-block=0 -S | FileCheck --check-prefixes=SAMPLING,PROMO %s
+
+; SAMPLING: $__llvm_profile_sampling = comdat any
+; SAMPLING: @__llvm_profile_sampling = thread_local global i16 0, comdat
+
+define void @foo(i32 %n, i32 %N) {
+; SAMPLING-LABEL: @foo
+; SAMPLING:  %[[VV0:[0-9]+]] = load i16, ptr @__llvm_profile_sampling, align 2
+; SAMPLING:  %[[VV1:[0-9]+]] = icmp ule i16 %[[VV0]], 200
+; SAMPLING:  br i1 %[[VV1]], label {{.*}}, label {{.*}}, !prof !0
+; SAMPLING: {{.*}} = load {{.*}} @__profc_foo{{.*}} 3)
+; SAMPLING-NEXT: add
+; SAMPLING-NEXT: store {{.*}}@__profc_foo{{.*}}3)
+bb:
+  %tmp = add nsw i32 %n, 1
+  %tmp1 = add nsw i32 %n, -1
+  br label %bb2
+
+bb2:
+; PROMO: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+  %i.0 = phi i32 [ 0, %bb ], [ %tmp10, %bb9 ]
+  %tmp3 = icmp slt i32 %i.0, %tmp
+  br i1 %tmp3, label %bb4, label %bb5
+
+bb4:
+  tail call void @bar(i32 1)
+  br label %bb9
+
+bb5:
+  %tmp6 = icmp slt i32 %i.0, %tmp1
+  br i1 %tmp6, label %bb7, label %bb8
+
+bb7:
+  tail call void @bar(i32 2)
+  br label %bb9
+
+bb8:
+  tail call void @bar(i32 3)
+  br label %bb9
+
+bb9:
+; SAMPLING:       phi {{.*}}
+; SAMPLING-NEXT:  %[[V1:[0-9]+]] = add i16 {{.*}}, 1
+; SAMPLING-NEXT:  store i16 %[[V1]], ptr @__llvm_profile_sampling, align 2
+; SAMPLING:       phi {{.*}}
+; SAMPLING-NEXT:  %[[V2:[0-9]+]] = add i16 {{.*}}, 1
+; SAMPLING-NEXT:  store i16 %[[V2]], ptr @__llvm_profile_sampling, align 2
+; SAMPLING:       phi {{.*}}
+; SAMPLING-NEXT:  %[[V3:[0-9]+]] = add i16 {{.*}}, 1
+; SAMPLING-NEXT:  store i16 %[[V3]], ptr @__llvm_profile_sampling, align 2
+; PROMO: %[[LIVEOUT3:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[LIVEOUT2:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[LIVEOUT1:[a-z0-9]+]] = phi {{.*}}
+  %tmp10 = add nsw i32 %i.0, 1
+  %tmp11 = icmp slt i32 %tmp10, %N
+  br i1 %tmp11, label %bb2, label %bb12
+
+bb12:
+  ret void
+; PROMO: %[[CHECK1:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}}
+; PROMO-NEXT: add {{.*}} %[[CHECK1]], %[[LIVEOUT1]]
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}
+; PROMO-NEXT: %[[CHECK2:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 1)
+; PROMO-NEXT: add {{.*}} %[[CHECK2]], %[[LIVEOUT2]]
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT: %[[CHECK3:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 2)
+; PROMO-NEXT: add {{.*}} %[[CHECK3]], %[[LIVEOUT3]]
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2)
+; PROMO-NOT: @__profc_foo{{.*}})
+
+}
+
+declare void @bar(i32)
+
+; SAMPLING: !0 = !{!"branch_weights", i32 200, i32 65336}
diff --git a/llvm/test/Transforms/PGOProfile/cspgo_sample.ll b/llvm/test/Transforms/PGOProfile/cspgo_sample.ll
new file mode 100644
index 0000000000000..97ad4d00c9d9c
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/cspgo_sample.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; REQUIRES: x86-registered-target
+
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: opt -module-summary %S/Inputs/cspgo_bar_sample.ll -o %t2.bc
+; RUN: llvm-lto2 run -lto-cspgo-profile-file=alloc -enable-sampled-instrumentation -lto-cspgo-gen -save-temps -o %t %t1.bc %t2.bc \
+; RUN:   -r=%t1.bc,foo,pl \
+; RUN:   -r=%t1.bc,bar,l \
+; RUN:   -r=%t1.bc,main,plx \
+; RUN:   -r=%t1.bc,__llvm_profile_filename,plx \
+; RUN:   -r=%t1.bc,__llvm_profile_raw_version,plx \
+; RUN:   -r=%t1.bc,__llvm_profile_sampling,pl \
+; RUN:   -r=%t2.bc,bar,pl \
+; RUN:   -r=%t2.bc,odd,pl \
+; RUN:   -r=%t2.bc,even,pl \
+; RUN:   -r=%t2.bc,__llvm_profile_filename,x \
+; RUN:   -r=%t2.bc,__llvm_profile_raw_version,x \
+; RUN:   -r=%t2.bc,__llvm_profile_sampling,
+; RUN: llvm-dis %t.1.4.opt.bc -o - | FileCheck %s --check-prefix=CSGEN
+
+; CSGEN: @__llvm_profile_sampling = thread_local global i16 0, comdat
+; CSGEN: @__profc_
+; CSGEN: @__profd_
+
+source_filename = "cspgo.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$__llvm_profile_filename = comdat any
+$__llvm_profile_raw_version = comdat any
+$__llvm_profile_sampling = comdat any
+@__llvm_profile_filename = local_unnamed_addr constant [25 x i8] c"pass2/default_%m.profraw\00", comdat
+@__llvm_profile_raw_version = local_unnamed_addr constant i64 216172782113783812, comdat
+@__llvm_profile_sampling = thread_local global i16 0, comdat
+@llvm.used = appending global [1 x i8*] [i8* bitcast (i64* @__llvm_profile_sampling to i8*)], section "llvm.metadata"
+
+define dso_local void @foo() #0 !prof !30 {
+entry:
+  br label %for.body
+
+for.body:
+  %i.06 = phi i32 [ 0, %entry ], [ %add1, %for.body ]
+  tail call void @bar(i32 %i.06) #3
+  %add = or i32 %i.06, 1
+  tail call void @bar(i32 %add) #3
+  %add1 = add nuw nsw i32 %i.06, 2
+  %cmp = icmp ult i32 %add1, 200000
+  br i1 %cmp, label %for.body, label %for.end, !prof !31
+
+for.end:
+  ret void
+}
+
+; CSGEN-LABEL: @foo
+; CSGEN:        [[TMP0:%.*]]  = load i16, ptr @__llvm_profile_sampling, align 2
+; CSGEN-NEXT:   [[TMP1:%.*]] = icmp ult i16 [[TMP0]], 201
+; CSGEN-NEXT:   br i1 [[TMP1]], label %{{.*}}, label %{{.*}}, !prof [[PROF:![0-9]+]]
+; CSGEN:        [[TMP2:%.*]] = add i16 {{.*}}, 1
+; CSGEN-NEXT:   store i16 [[TMP2]], ptr @__llvm_profile_sampling, align 2
+
+declare dso_local void @bar(i32)
+
+define dso_local i32 @main() !prof !30 {
+entry:
+  tail call void @foo()
+  ret i32 0
+}
+; CSGEN-LABEL: @main
+; CSGEN:        [[TMP0:%.*]]  = load i16, ptr @__llvm_profile_sampling, align 2
+; CSGEN-NEXT:   [[TMP1:%.*]] = icmp ult i16 [[TMP0]], 201
+; CSGEN-NEXT:   br i1 [[TMP1]], label %{{.*}}, label %{{.*}}, !prof [[PROF:![0-9]+]]
+; CSGEN:        [[TMP2:%.*]] = add i16 {{.*}}, 1
+; CSGEN-NEXT:   store i16 [[TMP2]], ptr @__llvm_profile_sampling, align 2
+
+attributes #0 = { "target-cpu"="x86-64" }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"EnableSplitLTOUnit", i32 0}
+!2 = !{i32 1, !"ProfileSummary", !3}
+!3 = !{!4, !5, !6, !7, !8, !9, !10, !11}
+!4 = !{!"ProfileFormat", !"InstrProf"}
+!5 = !{!"TotalCount", i64 500002}
+!6 = !{!"MaxCount", i64 200000}
+!7 = !{!"MaxInternalCount", i64 100000}
+!8 = !{!"MaxFunctionCount", i64 200000}
+!9 = !{!"NumCounts", i64 6}
+!10 = !{!"NumFunctions", i64 4}
+!11 = !{!"DetailedSummary", !12}
+!12 = !{!13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28}
+!13 = !{i32 10000, i64 200000, i32 1}
+!14 = !{i32 100000, i64 200000, i32 1}
+!15 = !{i32 200000, i64 200000, i32 1}
+!16 = !{i32 300000, i64 200000, i32 1}
+!17 = !{i32 400000, i64 200000, i32 1}
+!18 = !{i32 500000, i64 100000, i32 4}
+!19 = !{i32 600000, i64 100000, i32 4}
+!20 = !{i32 700000, i64 100000, i32 4}
+!21 = !{i32 800000, i64 100000, i32 4}
+!22 = !{i32 900000, i64 100000, i32 4}
+!23 = !{i32 950000, i64 100000, i32 4}
+!24 = !{i32 990000, i64 100000, i32 4}
+!25 = !{i32 999000, i64 100000, i32 4}
+!26 = !{i32 999900, i64 100000, i32 4}
+!27 = !{i32 999990, i64 100000, i32 4}
+!28 = !{i32 999999, i64 1, i32 6}
+!30 = !{!"function_entry_count", i64 1}
+!31 = !{!"branch_weights", i32 100000, i32 1}
+
+; CSGEN: [[PROF]] = !{!"branch_weights", i32 200, i32 65336}
+
diff --git a/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_fast.ll b/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_fast.ll
new file mode 100644
index 0000000000000..dcc1e805ba6f6
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_fast.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s --passes=instrprof --sampled-instrumentation -S | FileCheck %s --check-prefixes=SAMPLE-VAR,SAMPLE-CODE,SAMPLE-DURATION,SAMPLE-WEIGHT
+; RUN: opt < %s --passes=instrprof --sampled-instrumentation --sampled-instr-burst-duration=100 -S | FileCheck %s --check-prefixes=SAMPLE-VAR,SAMPLE-CODE,SAMPLE-DURATION100,SAMPLE-WEIGHT100
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$__llvm_profile_raw_version = comdat any
+
+; SAMPLE-VAR: $__llvm_profile_sampling = comdat any
+
+@__llvm_profile_raw_version = constant i64 72057594037927940, comdat
+@__profn_f = private constant [1 x i8] c"f"
+
+; SAMPLE-VAR: @__llvm_profile_sampling = thread_local global i16 0, comdat
+; SAMPLE-VAR: @__profc_f = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
+; SAMPLE-VAR: @__profd_f = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 -3706093650706652785, i64 12884901887, i64 sub (i64 ptrtoint (ptr @__profc_f to i64), i64 ptrtoint (ptr @__profd_f to i64)), i64 0, ptr @f.local, ptr null, i32 1, [3 x i16] zeroinitializer, i32 0 }, section "__llvm_prf_data", comdat($__profc_f), align 8
+; SAMPLE-VAR: @__llvm_prf_nm = private constant {{.*}}, section "__llvm_prf_names", align 1
+; SAMPLE-VAR: @llvm.compiler.used = appending global [2 x ptr] [ptr @__llvm_profile_sampling, ptr @__profd_f], section "llvm.metadata"
+; SAMPLE-VAR: @llvm.used = appending global [1 x ptr] [ptr @__llvm_prf_nm], section "llvm.metadata"
+
+
+define void @f() {
+; SAMPLE-CODE-LABEL: @f(
+; SAMPLE-CODE:  entry:
+; SAMPLE-CODE-NEXT:    [[TMP0:%.*]] = load i16, ptr @__llvm_profile_sampling, align 2
+; SAMPLE-DURATION:         [[TMP1:%.*]] = icmp ule i16 [[TMP0]], 200
+; SAMPLE-DURATION100:     [[TMP1:%.*]] = icmp ule i16 [[TMP0]], 100
+; SAMPLE-CODE:         br i1 [[TMP1]], label %[[TMP2:.*]], label %[[TMP4:.*]], !prof !0
+; SAMPLE-CODE:       [[TMP2]]:
+; SAMPLE-CODE-NEXT:    [[PGOCOUNT:%.*]] = load i64, ptr @__profc_f
+; SAMPLE-CODE-NEXT:    [[TMP3:%.*]] = add i64 [[PGOCOUNT]], 1
+; SAMPLE-CODE-NEXT:    store i64 [[TMP3]], ptr @__profc_f
+; SAMPLE-CODE-NEXT:    br label %[[TMP4]]
+; SAMPLE-CODE:       [[TMP4]]:
+; SAMPLE-CODE-NEXT:    [[TMP5:%.*]] = add i16 [[TMP0]], 1
+; SAMPLE-CODE-NEXT:    store i16 [[TMP5]],  ptr @__llvm_profile_sampling, align 2
+; SAMPLE-CODE-NEXT:    ret void
+;
+entry:
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_f, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
+  ret void
+}
+
+; SAMPLE-WEIGHT: !0 = !{!"branch_weights", i32 200, i32 65336}
+; SAMPLE-WEIGHT100: !0 = !{!"branch_weights", i32 100, i32 65436}
+
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32)
diff --git a/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_full.ll b/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_full.ll
new file mode 100644
index 0000000000000..57d1a0cd33fbe
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_full.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s --passes=instrprof -sampled-instrumentation --sampled-instr-period=1009 --sampled-instr-burst-duration=32 -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$__llvm_profile_raw_version = comdat any
+
+@__llvm_profile_raw_version = constant i64 72057594037927940, comdat
+@__profn_f = private constant [1 x i8] c"f"
+
+define void @f() {
+; CHECK-LABEL: define void @f() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr @__llvm_profile_sampling, align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i16 [[TMP0]], 32
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[BB2:.*]], label %[[BB4:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    [[PGOCOUNT:%.*]] = load i64, ptr @__profc_f, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[PGOCOUNT]], 1
+; CHECK-NEXT:    store i64 [[TMP3]], ptr @__profc_f, align 8
+; CHECK-NEXT:    br label %[[BB4]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = add i16 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp uge i16 [[TMP5]], 1009
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    store i16 0, ptr @__llvm_profile_sampling, align 2
+; CHECK-NEXT:    br label %[[BB9:.*]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    store i16 [[TMP5]], ptr @__llvm_profile_sampling, align 2
+; CHECK-NEXT:    br label %[[BB9]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_f, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
+  ret void
+}
+
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32)
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 32, i32 978}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1009}
+;.
diff --git a/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_full_intsize.ll b/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_full_intsize.ll
new file mode 100644
index 0000000000000..1ad889524bc6a
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_full_intsize.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s --passes=instrprof --sampled-instrumentation --sampled-instr-period=1000019 --sampled-instr-burst-duration=3000 -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$__llvm_profile_raw_version = comdat any
+
+@__llvm_profile_raw_version = constant i64 72057594037927940, comdat
+@__profn_f = private constant [1 x i8] c"f"
+
+define void @f() {
+; CHECK-LABEL: define void @f() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr @__llvm_profile_sampling, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i32 [[TMP0]], 3000
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[BB2:.*]], label %[[BB4:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    [[PGOCOUNT:%.*]] = load i64, ptr @__profc_f, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[PGOCOUNT]], 1
+; CHECK-NEXT:    store i64 [[TMP3]], ptr @__profc_f, align 8
+; CHECK-NEXT:    br label %[[BB4]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp uge i32 [[TMP5]], 1000019
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    store i32 0, ptr @__llvm_profile_sampling, align 4
+; CHECK-NEXT:    br label %[[BB9:.*]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    store i32 [[TMP5]], ptr @__llvm_profile_sampling, align 4
+; CHECK-NEXT:    br label %[[BB9]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_f, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
+  ret void
+}
+
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32)
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 3000, i32 997020}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1000019}
+;.
diff --git a/llvm/test/Transforms/PGOProfile/instrprof_simple_sampling.ll b/llvm/test/Transforms/PGOProfile/instrprof_simple_sampling.ll
new file mode 100644
index 0000000000000..8e846bbf1d982
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/instrprof_simple_sampling.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s --passes=instrprof --sampled-instrumentation --sampled-instr-burst-duration=1 --sampled-instr-period=1009 -S | FileCheck %s --check-prefix=PERIOD1009
+; RUN: opt < %s --passes=instrprof --sampled-instrumentation --sampled-instr-burst-duration=1 -S | FileCheck %s --check-prefix=DEFAULTPERIOD
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$__llvm_profile_raw_version = comdat any
+
+@__llvm_profile_raw_version = constant i64 72057594037927940, comdat
+@__profn_f = private constant [1 x i8] c"f"
+
+define void @f() {
+; PERIOD1009-LABEL: define void @f() {
+; PERIOD1009-NEXT:  [[ENTRY:.*:]]
+; PERIOD1009-NEXT:    [[TMP0:%.*]] = load i16, ptr @__llvm_profile_sampling, align 2
+; PERIOD1009-NEXT:    [[TMP1:%.*]] = add i16 [[TMP0]], 1
+; PERIOD1009-NEXT:    [[TMP2:%.*]] = icmp uge i16 [[TMP1]], 1009
+; PERIOD1009-NEXT:    br i1 [[TMP2]], label %[[BB3:.*]], label %[[BB5:.*]], !prof [[PROF0:![0-9]+]]
+; PERIOD1009:       [[BB3]]:
+; PERIOD1009-NEXT:    [[PGOCOUNT:%.*]] = load i64, ptr @__profc_f, align 8
+; PERIOD1009-NEXT:    [[TMP4:%.*]] = add i64 [[PGOCOUNT]], 1
+; PERIOD1009-NEXT:    store i64 [[TMP4]], ptr @__profc_f, align 8
+; PERIOD1009-NEXT:    store i16 0, ptr @__llvm_profile_sampling, align 2
+; PERIOD1009-NEXT:    br label %[[BB6:.*]]
+; PERIOD1009:       [[BB5]]:
+; PERIOD1009-NEXT:    store i16 [[TMP1]], ptr @__llvm_profile_sampling, align 2
+; PERIOD1009-NEXT:    br label %[[BB6]]
+; PERIOD1009:       [[BB6]]:
+; PERIOD1009-NEXT:    ret void
+;
+; DEFAULTPERIOD-LABEL: define void @f() {
+; DEFAULTPERIOD-NEXT:  [[ENTRY:.*:]]
+; DEFAULTPERIOD-NEXT:    [[TMP0:%.*]] = load i16, ptr @__llvm_profile_sampling, align 2
+; DEFAULTPERIOD-NEXT:    [[TMP1:%.*]] = add i16 [[TMP0]], 1
+; DEFAULTPERIOD-NEXT:    [[TMP2:%.*]] = icmp uge i16 [[TMP1]], -1
+; DEFAULTPERIOD-NEXT:    br i1 [[TMP2]], label %[[BB3:.*]], label %[[BB5:.*]], !prof [[PROF0:![0-9]+]]
+; DEFAULTPERIOD:       [[BB3]]:
+; DEFAULTPERIOD-NEXT:    [[PGOCOUNT:%.*]] = load i64, ptr @__profc_f, align 8
+; DEFAULTPERIOD-NEXT:    [[TMP4:%.*]] = add i64 [[PGOCOUNT]], 1
+; DEFAULTPERIOD-NEXT:    store i64 [[TMP4]], ptr @__profc_f, align 8
+; DEFAULTPERIOD-NEXT:    store i16 0, ptr @__llvm_profile_sampling, align 2
+; DEFAULTPERIOD-NEXT:    br label %[[BB6:.*]]
+; DEFAULTPERIOD:       [[BB5]]:
+; DEFAULTPERIOD-NEXT:    store i16 [[TMP1]], ptr @__llvm_profile_sampling, align 2
+; DEFAULTPERIOD-NEXT:    br label %[[BB6]]
+; DEFAULTPERIOD:       [[BB6]]:
+; DEFAULTPERIOD-NEXT:    ret void
+;
+entry:
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_f, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
+  ret void
+}
+
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32)
+;.
+; PERIOD1009: [[PROF0]] = !{!"branch_weights", i32 1, i32 1009}
+;.
+; DEFAULTPERIOD: [[PROF0]] = !{!"branch_weights", i32 1, i32 65535}
+;.

From 2c92335eb78f93271e9cefdf1f80740e286d801d Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 22 Jul 2024 09:21:01 -0700
Subject: [PATCH 441/486] [RISCV] Copy call frame size when splitting basic
 block in emitSelectPseudo. (#99823)

Fixes #97304.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp |  5 ++
 llvm/test/CodeGen/RISCV/pr97304.ll          | 51 +++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/pr97304.ll

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e938454b8e642..585d8a060fe94 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -18478,6 +18478,11 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
   F->insert(I, IfFalseMBB);
   F->insert(I, TailMBB);
 
+  // Set the call frame size on entry to the new basic blocks.
+  unsigned CallFrameSize = TII.getCallFrameSizeAt(MI);
+  IfFalseMBB->setCallFrameSize(CallFrameSize);
+  TailMBB->setCallFrameSize(CallFrameSize);
+
   // Transfer debug instructions associated with the selects to TailMBB.
   for (MachineInstr *DebugInstr : SelectDebugValues) {
     TailMBB->push_back(DebugInstr->removeFromParent());
diff --git a/llvm/test/CodeGen/RISCV/pr97304.ll b/llvm/test/CodeGen/RISCV/pr97304.ll
new file mode 100644
index 0000000000000..120a0e787384d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/pr97304.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv64 -verify-machineinstrs -stop-after=finalize-isel | FileCheck %s
+
+define i32 @_ZNK2cv12LMSolverImpl3runERKNS_17_InputOutputArrayE(i1 %cmp436) {
+  ; CHECK-LABEL: name: _ZNK2cv12LMSolverImpl3runERKNS_17_InputOutputArrayE
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $x10
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY [[COPY]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.for.cond:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[ANDI:%[0-9]+]]:gpr = ANDI [[COPY1]], 1
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 8, 0, implicit-def dead $x2, implicit $x2
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x2
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gprjalr = COPY $x0
+  ; CHECK-NEXT:   SD [[COPY3]], [[COPY2]], 0 :: (store (s64))
+  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+  ; CHECK-NEXT:   [[ADDI1:%[0-9]+]]:gpr = ADDI $x0, 32
+  ; CHECK-NEXT:   BNE [[ANDI]], $x0, %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.for.cond (call-frame-size 8):
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3.for.cond (call-frame-size 8):
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr = PHI [[ADDI1]], %bb.1, [[ADDI]], %bb.2
+  ; CHECK-NEXT:   $x10 = COPY [[COPY3]]
+  ; CHECK-NEXT:   $x11 = COPY [[PHI]]
+  ; CHECK-NEXT:   $x12 = COPY [[COPY3]]
+  ; CHECK-NEXT:   $x13 = COPY [[COPY3]]
+  ; CHECK-NEXT:   $x14 = COPY [[COPY3]]
+  ; CHECK-NEXT:   $x15 = COPY [[COPY3]]
+  ; CHECK-NEXT:   $x16 = COPY [[COPY3]]
+  ; CHECK-NEXT:   $x17 = COPY [[COPY3]]
+  ; CHECK-NEXT:   PseudoCALLIndirect [[COPY3]], csr_ilp32_lp64, implicit-def dead $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x2, implicit-def $x10
+  ; CHECK-NEXT:   ADJCALLSTACKUP 8, 0, implicit-def dead $x2, implicit $x2
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY $x10
+  ; CHECK-NEXT:   PseudoBR %bb.1
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %conv = select i1 %cmp436, i32 32, i32 1
+  %call479 = call i32 (ptr, ...) null(ptr null, i32 %conv, i32 0, i32 0, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00)
+  br label %for.cond
+}

From 6911f823ad0e1471f523205daf976c3cf6f96bfd Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 22 Jul 2024 11:18:53 -0500
Subject: [PATCH 442/486] [libc] Fix invalid format specifier in benchmark

Summary:
This value is a uint32_t but is printed as a uint64_t, leading to
invalid offsets when done on AMDGPU due to its packed format extending
past the buffer.
---
 libc/benchmarks/gpu/LibcGpuBenchmark.cpp | 27 ++++++++++--------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
index ac139cc21b0b6..031ad163c20da 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
@@ -74,17 +74,10 @@ struct AtomicBenchmarkSums {
 };
 
 AtomicBenchmarkSums all_results;
-const char *header_format_string =
-    "Benchmark            |  Cycles |     Min |     Max | Iterations |        "
-    "Time |   Stddev |  Threads |\n";
-const char *output_format_string =
-    "%-20s |%8ld |%8ld |%8ld |%11ld |%9ld %2s |%9ld |%9d |\n";
-
 constexpr auto GREEN = "\033[32m";
 constexpr auto RESET = "\033[0m";
 
 void print_results(Benchmark *b) {
-
   BenchmarkResult result;
   cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
   int num_threads = all_results.active_threads.load(cpp::MemoryOrder::RELAXED);
@@ -104,16 +97,16 @@ void print_results(Benchmark *b) {
   const uint64_t duration_us = duration_ns / 1000;
   const uint64_t duration_ms = duration_ns / (1000 * 1000);
   uint64_t converted_duration = duration_ns;
-  cpp::string time_unit;
+  const char *time_unit;
   if (duration_ms != 0) {
     converted_duration = duration_ms;
-    time_unit = cpp::string("ms");
+    time_unit = "ms";
   } else if (duration_us != 0) {
     converted_duration = duration_us;
-    time_unit = cpp::string("us");
+    time_unit = "us";
   } else {
     converted_duration = duration_ns;
-    time_unit = cpp::string("ns");
+    time_unit = "ns";
   }
   result.total_time = converted_duration;
   // result.total_time =
@@ -121,10 +114,10 @@ void print_results(Benchmark *b) {
   cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
 
   LIBC_NAMESPACE::printf(
-      output_format_string, b->get_test_name().data(), result.cycles,
-      result.min, result.max, result.total_iterations, result.total_time,
-      time_unit.data(), static_cast<uint64_t>(result.standard_deviation),
-      num_threads);
+      "%-20s |%8ld |%8ld |%8ld |%11d |%9ld %2s |%9ld |%9d |\n",
+      b->get_test_name().data(), result.cycles, result.min, result.max,
+      result.total_iterations, result.total_time, time_unit,
+      static_cast<uint64_t>(result.standard_deviation), num_threads);
 }
 
 void print_header() {
@@ -132,7 +125,9 @@ void print_header() {
   LIBC_NAMESPACE::printf("Running Suite: %-10s\n",
                          benchmarks[0]->get_suite_name().data());
   LIBC_NAMESPACE::printf("%s", RESET);
-  LIBC_NAMESPACE::printf(header_format_string);
+  LIBC_NAMESPACE::printf("Benchmark            |  Cycles |     Min |     Max | "
+                         "Iterations |        "
+                         "Time |   Stddev |  Threads |\n");
   LIBC_NAMESPACE::printf(
       "---------------------------------------------------------------------"
       "--------------------------------\n");

From 0b12e185bd81b2199f95e211743f011084a79195 Mon Sep 17 00:00:00 2001
From: Raphael Isemann <teemperor@gmail.com>
Date: Mon, 22 Jul 2024 18:29:34 +0200
Subject: [PATCH 443/486] [clang-fuzzer-dictionary] Fix build failure with
 libfuzzer (#99871)

---
 clang/tools/clang-fuzzer/dictionary/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/tools/clang-fuzzer/dictionary/CMakeLists.txt b/clang/tools/clang-fuzzer/dictionary/CMakeLists.txt
index ee4aa587ea54d..6b72b98f5e1c4 100644
--- a/clang/tools/clang-fuzzer/dictionary/CMakeLists.txt
+++ b/clang/tools/clang-fuzzer/dictionary/CMakeLists.txt
@@ -1,3 +1,4 @@
+set(CMAKE_CXX_FLAGS ${CXX_FLAGS_NOFUZZ})
 add_clang_executable(clang-fuzzer-dictionary
   dictionary.c
   )

From ec966f699d26410d5d9a705af570fa339df0e78c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Mon, 22 Jul 2024 19:31:47 +0300
Subject: [PATCH 444/486] [libcxx] [test] Make indentation more consistent in
 thousands_sep. NFC. (#99844)

This was made inconsistent recently in
f114eddb1923289b696f1b0980cc22c4dbaafa22.
---
 .../locale.numpunct.byname/thousands_sep.pass.cpp         | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/thousands_sep.pass.cpp b/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/thousands_sep.pass.cpp
index f368b1069c063..850352b3bc1ec 100644
--- a/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/thousands_sep.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/thousands_sep.pass.cpp
@@ -64,7 +64,7 @@ int main(int, char**)
         std::locale l(LOCALE_fr_FR_UTF_8);
         {
 #if defined(_CS_GNU_LIBC_VERSION) || defined(_WIN32) || defined(_AIX)
-          const char sep = ' ';
+            const char sep = ' ';
 #else
             const char sep = ',';
 #endif
@@ -77,11 +77,11 @@ int main(int, char**)
 #if defined(_CS_GNU_LIBC_VERSION)
             const wchar_t wsep = glibc_version_less_than("2.27") ? L' ' : L'\u202f';
 #  elif defined(_AIX)
-          const wchar_t wsep = L'\u202F';
+            const wchar_t wsep = L'\u202F';
 #  elif defined(_WIN32)
-          const wchar_t wsep = L'\u00A0';
+            const wchar_t wsep = L'\u00A0';
 #  else
-          const wchar_t wsep = L',';
+            const wchar_t wsep = L',';
 #  endif
             typedef wchar_t C;
             const std::numpunct<C>& np = std::use_facet<std::numpunct<C> >(l);

From 3d7622ea0bd443bb6ccb58d6b33e8cf52a8f0f4e Mon Sep 17 00:00:00 2001
From: Xiaoyang Liu <siujoeng.lau@gmail.com>
Date: Tue, 23 Jul 2024 01:32:37 +0900
Subject: [PATCH 445/486] [libc++][ranges] LWG3618: Unnecessary `iter_move` for
 `transform_view::iterator` (#91809)

## Introduction

This patch implements LWG3618: Unnecessary `iter_move` for
`transform_view::iterator`.

`transform_view`'s iterator currently specifies a customization point
for `iter_move`. This customization point does the same thing that the
default implementation would do, but its sole purpose is to ensure the
appropriate conditional `noexcept` specification.

## Reference

-
[[range.transform.iterator]](https://eel.is/c++draft/range.transform.iterator)
- [LWG3618](https://cplusplus.github.io/LWG/issue3618)
---
 libcxx/docs/Status/Cxx23Issues.csv            |  2 +-
 libcxx/include/__ranges/transform_view.h      |  7 ---
 .../range.transform/iterator/deref.pass.cpp   |  2 +-
 .../iterator/iter_move.pass.cpp               | 49 -------------------
 4 files changed, 2 insertions(+), 58 deletions(-)
 delete mode 100644 libcxx/test/std/ranges/range.adaptors/range.transform/iterator/iter_move.pass.cpp

diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index 50104052effad..1f35ca5713a9b 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -146,7 +146,7 @@
 "`3610 <https://wg21.link/LWG3610>`__","``iota_view::size`` sometimes rejects integer-class types","February 2022","","","|ranges|"
 "`3612 <https://wg21.link/LWG3612>`__","Inconsistent pointer alignment in ``std::format`` ","February 2022","|Complete|","14.0","|format|"
 "`3616 <https://wg21.link/LWG3616>`__","LWG 3498 seems to miss the non-member ``swap`` for ``basic_syncbuf`` ","February 2022","",""
-"`3618 <https://wg21.link/LWG3618>`__","Unnecessary ``iter_move`` for ``transform_view::iterator`` ","February 2022","","","|ranges|"
+"`3618 <https://wg21.link/LWG3618>`__","Unnecessary ``iter_move`` for ``transform_view::iterator`` ","February 2022","|Complete|","19.0","|ranges|"
 "`3619 <https://wg21.link/LWG3619>`__","Specification of ``vformat_to`` contains ill-formed ``formatted_size`` calls","February 2022","|Nothing to do|","","|format|"
 "`3621 <https://wg21.link/LWG3621>`__","Remove feature-test macro ``__cpp_lib_monadic_optional`` ","February 2022","|Complete|","15.0"
 "`3632 <https://wg21.link/LWG3632>`__","``unique_ptr`` ""Mandates: This constructor is not selected by class template argument deduction""","February 2022","|Nothing to do|",""
diff --git a/libcxx/include/__ranges/transform_view.h b/libcxx/include/__ranges/transform_view.h
index dc3aaa59ed8c3..bcce389c0e680 100644
--- a/libcxx/include/__ranges/transform_view.h
+++ b/libcxx/include/__ranges/transform_view.h
@@ -326,13 +326,6 @@ class transform_view<_View, _Fn>::__iterator : public __transform_view_iterator_
   {
     return __x.__current_ - __y.__current_;
   }
-
-  _LIBCPP_HIDE_FROM_ABI friend constexpr decltype(auto) iter_move(const __iterator& __i) noexcept(noexcept(*__i)) {
-    if constexpr (is_lvalue_reference_v<decltype(*__i)>)
-      return std::move(*__i);
-    else
-      return *__i;
-  }
 };
 
 #  if _LIBCPP_STD_VER >= 23
diff --git a/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/deref.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/deref.pass.cpp
index 06eb91dba22ff..5d8bf51c69696 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/deref.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/deref.pass.cpp
@@ -37,7 +37,7 @@ int main(int, char**) {
     using View = std::ranges::transform_view<MoveOnlyView, PlusOneNoexcept>;
     View transformView(MoveOnlyView{buff}, PlusOneNoexcept{});
     assert(*transformView.begin() == 1);
-    LIBCPP_ASSERT_NOEXCEPT(*std::declval<std::ranges::iterator_t<View>>());
+    ASSERT_NOEXCEPT(*std::declval<std::ranges::iterator_t<View>>());
     ASSERT_SAME_TYPE(int, decltype(*std::declval<View>().begin()));
   }
   {
diff --git a/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/iter_move.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/iter_move.pass.cpp
deleted file mode 100644
index ea1e4b61412c4..0000000000000
--- a/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/iter_move.pass.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// friend constexpr decltype(auto) iter_move(const iterator& i)
-//    noexcept(noexcept(invoke(i.parent_->fun_, *i.current_)))
-
-#include <ranges>
-
-#include "test_macros.h"
-#include "../types.h"
-
-constexpr bool test() {
-  int buff[8] = {0, 1, 2, 3, 4, 5, 6, 7};
-
-  {
-    std::ranges::transform_view transformView(MoveOnlyView{buff}, PlusOneMutable{});
-    auto iter = transformView.begin();
-    ASSERT_NOT_NOEXCEPT(std::ranges::iter_move(iter));
-
-    assert(std::ranges::iter_move(iter) == 1);
-    assert(std::ranges::iter_move(iter + 2) == 3);
-
-    ASSERT_SAME_TYPE(int, decltype(std::ranges::iter_move(iter)));
-    ASSERT_SAME_TYPE(int, decltype(std::ranges::iter_move(std::move(iter))));
-  }
-
-  {
-    LIBCPP_ASSERT_NOEXCEPT(std::ranges::iter_move(
-      std::declval<std::ranges::iterator_t<std::ranges::transform_view<MoveOnlyView, PlusOneNoexcept>>&>()));
-    ASSERT_NOT_NOEXCEPT(std::ranges::iter_move(
-      std::declval<std::ranges::iterator_t<std::ranges::transform_view<MoveOnlyView, PlusOneMutable>>&>()));
-  }
-
-  return true;
-}
-
-int main(int, char**) {
-  test();
-  static_assert(test());
-
-  return 0;
-}

From 418922623673a8044d30a57825b026a03f9d9354 Mon Sep 17 00:00:00 2001
From: nicole mazzuca <nicole@strega-nil.co>
Date: Mon, 22 Jul 2024 09:33:41 -0700
Subject: [PATCH 446/486] [libc++] Update some C++23 statuses to "Nothing to
 do" or "Complete" (#99621)

- [P2160R1][] "Locks lock lockables"
- [P2212R2][] "Relax Requirements for `time_point::clock`"
- [P1675R2][] "`rethrow_exception` must be allowed to copy"
- [P2340R1][] "Clarifying the status of the 'C headers'"
- [P2460R2][] "Relax requirements on `wchar_t` to match existing
practices"

Are all papers that change wording without changing implementation
behaviour.

Additionally, [P2736R2][] "Referencing The Unicode Standard", is an
already complete paper in 19.0 (as of [LLVM-86543][])

[P2160R1]: https://wg21.link/p2160r1
[P2212R2]: https://wg21.link/p2212r2
[P1675R2]: https://wg21.link/p1675r2
[P2340R1]: https://wg21.link/p2340r1
[P2460R2]: https://wg21.link/p2460r2
[P2736R2]: https://wg21.link/p2736r2
[LLVM-86543]: https://github.com/llvm/llvm-project/pull/86543
---
 libcxx/docs/Status/Cxx23Papers.csv | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index f8970320ce1fa..92f4908487ae7 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -6,9 +6,9 @@
 "","","","","","",""
 "`P1682R3 <https://wg21.link/P1682R3>`__","LWG","std::to_underlying for enumerations","February 2021","|Complete|","13.0"
 "`P2017R1 <https://wg21.link/P2017R1>`__","LWG","Conditionally borrowed ranges","February 2021","|Complete|","16.0","|ranges|"
-"`P2160R1 <https://wg21.link/P2160R1>`__","LWG","Locks lock lockables","February 2021","",""
+"`P2160R1 <https://wg21.link/P2160R1>`__","LWG","Locks lock lockables","February 2021","Nothing to do",""
 "`P2162R2 <https://wg21.link/P2162R2>`__","LWG","Inheriting from std::variant","February 2021","|Complete|","13.0"
-"`P2212R2 <https://wg21.link/P2212R2>`__","LWG","Relax Requirements for time_point::clock","February 2021","",""
+"`P2212R2 <https://wg21.link/P2212R2>`__","LWG","Relax Requirements for time_point::clock","February 2021","Nothing to do",""
 "`P2259R1 <https://wg21.link/P2259R1>`__","LWG","Repairing input range adaptors and counted_iterator","February 2021","","","|ranges|"
 "","","","","","",""
 "`P0401R6 <https://wg21.link/P0401R6>`__","LWG","Providing size feedback in the Allocator interface","June 2021","|Complete|","15.0"
@@ -29,12 +29,12 @@
 "`P1072R10 <https://wg21.link/P1072R10>`__","LWG","``basic_string::resize_and_overwrite``","October 2021","|Complete|","14.0"
 "`P1147R1 <https://wg21.link/P1147R1>`__","LWG","Printing ``volatile`` Pointers","October 2021","|Complete|","14.0"
 "`P1272R4 <https://wg21.link/P1272R4>`__","LWG","Byteswapping for fun&&nuf","October 2021","|Complete|","14.0"
-"`P1675R2 <https://wg21.link/P1675R2>`__","LWG","``rethrow_exception`` must be allowed to copy","October 2021","",""
+"`P1675R2 <https://wg21.link/P1675R2>`__","LWG","``rethrow_exception`` must be allowed to copy","October 2021","Nothing to do",""
 "`P2077R3 <https://wg21.link/P2077R3>`__","LWG","Heterogeneous erasure overloads for associative containers","October 2021","",""
 "`P2251R1 <https://wg21.link/P2251R1>`__","LWG","Require ``span`` & ``basic_string_view`` to be Trivially Copyable","October 2021","|Complete|","14.0"
 "`P2301R1 <https://wg21.link/P2301R1>`__","LWG","Add a ``pmr`` alias for ``std::stacktrace``","October 2021","",""
 "`P2321R2 <https://wg21.link/P2321R2>`__","LWG","``zip``","October 2021","|In Progress|","","|ranges|"
-"`P2340R1 <https://wg21.link/P2340R1>`__","LWG","Clarifying the status of the 'C headers'","October 2021","",""
+"`P2340R1 <https://wg21.link/P2340R1>`__","LWG","Clarifying the status of the 'C headers'","October 2021","Nothing to do",""
 "`P2393R1 <https://wg21.link/P2393R1>`__","LWG","Cleaning up ``integer``-class types","October 2021","",""
 "`P2401R0 <https://wg21.link/P2401R0>`__","LWG","Add a conditional ``noexcept`` specification to ``std::exchange``","October 2021","|Complete|","14.0"
 "","","","","","",""
@@ -74,7 +74,7 @@
 "`P2438R2 <https://wg21.link/P2438R2>`__","LWG","``std::string::substr() &&``","July 2022","|Complete|","16.0"
 "`P2445R1 <https://wg21.link/P2445R1>`__","LWG","``forward_like``","July 2022","|Complete|","16.0"
 "`P2446R2 <https://wg21.link/P2446R2>`__","LWG","``views::as_rvalue``","July 2022","|Complete|","16.0","|ranges|"
-"`P2460R2 <https://wg21.link/P2460R2>`__","LWG","Relax requirements on ``wchar_t`` to match existing practices","July 2022","",""
+"`P2460R2 <https://wg21.link/P2460R2>`__","LWG","Relax requirements on ``wchar_t`` to match existing practices","July 2022","Nothing to do",""
 "`P2465R3 <https://wg21.link/P2465R3>`__","LWG","Standard Library Modules ``std`` and ``std.compat``","July 2022","|Complete|","19.0",""
 "`P2467R1 <https://wg21.link/P2467R1>`__","LWG","Support exclusive mode for ``fstreams``","July 2022","|Complete|","18.0",""
 "`P2474R2 <https://wg21.link/P2474R2>`__","LWG","``views::repeat``","July 2022","|Complete|","17.0","|ranges|"
@@ -120,4 +120,4 @@
 "`P2614R2 <https://wg21.link/P2614R2>`__","LWG", "Deprecate ``numeric_limits::has_denorm``","February 2023","|Complete|","18.0",""
 "`P2588R3 <https://wg21.link/P2588R3>`__","LWG", "``barrier``’s phase completion guarantees","February 2023","","",""
 "`P2763R1 <https://wg21.link/P2763R1>`__","LWG", "``layout_stride`` static extents default constructor fix","February 2023","","",""
-"`P2736R2 <https://wg21.link/P2736R2>`__","CWG","Referencing The Unicode Standard","February 2023","","","|format|"
+"`P2736R2 <https://wg21.link/P2736R2>`__","CWG","Referencing The Unicode Standard","February 2023","Complete","19.0","|format|"

From 70843bf658004cc6997e4978e625140abce77427 Mon Sep 17 00:00:00 2001
From: OverMighty <its.overmighty@gmail.com>
Date: Mon, 22 Jul 2024 18:37:44 +0200
Subject: [PATCH 447/486] [libc][math] Optimize copysign{,f,f16} and
 fabs{,f,f16} with builtins when available (#99037)

---
 .../modules/LLVMLibCCompileOptionRules.cmake  |  4 ++
 libc/cmake/modules/LLVMLibCFlagRules.cmake    |  3 ++
 libc/src/math/generic/CMakeLists.txt          | 20 +++++++--
 libc/src/math/generic/copysign.cpp            |  4 ++
 libc/src/math/generic/copysignf.cpp           |  4 ++
 libc/src/math/generic/copysignf16.cpp         |  4 ++
 libc/src/math/generic/fabs.cpp                |  8 +++-
 libc/src/math/generic/fabsf.cpp               |  8 +++-
 libc/src/math/generic/fabsf16.cpp             | 13 +++++-
 .../BinaryOpSingleOutputPerf.h                |  7 +--
 .../math/performance_testing/CMakeLists.txt   | 19 ++++++++
 .../SingleInputSingleOutputPerf.h             | 44 ++++++++++++-------
 .../misc_basic_ops_perf.cpp                   | 41 +++++++++++++++++
 13 files changed, 155 insertions(+), 24 deletions(-)
 create mode 100644 libc/test/src/math/performance_testing/misc_basic_ops_perf.cpp

diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index a722fe709af5f..97d1c7262d24d 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -6,6 +6,7 @@ function(_get_compile_options_from_flags output_var)
   endif()
   check_flag(ADD_ROUND_OPT_FLAG ${ROUND_OPT_FLAG} ${ARGN})
   check_flag(ADD_EXPLICIT_SIMD_OPT_FLAG ${EXPLICIT_SIMD_OPT_FLAG} ${ARGN})
+  check_flag(ADD_MISC_MATH_BASIC_OPS_OPT_FLAG ${MISC_MATH_BASIC_OPS_OPT_FLAG} ${ARGN})
 
   if(LLVM_COMPILER_IS_GCC_COMPATIBLE)
     if(ADD_FMA_FLAG)
@@ -37,6 +38,9 @@ function(_get_compile_options_from_flags output_var)
     if(ADD_EXPLICIT_SIMD_OPT_FLAG)
       list(APPEND compile_options "-D__LIBC_EXPLICIT_SIMD_OPT")
     endif()
+    if(ADD_MISC_MATH_BASIC_OPS_OPT_FLAG)
+      list(APPEND compile_options "-D__LIBC_MISC_MATH_BASIC_OPS_OPT")
+    endif()
   elseif(MSVC)
     if(ADD_FMA_FLAG)
       list(APPEND compile_options "/arch:AVX2")
diff --git a/libc/cmake/modules/LLVMLibCFlagRules.cmake b/libc/cmake/modules/LLVMLibCFlagRules.cmake
index eca7ba8d183e6..4398fe55db5aa 100644
--- a/libc/cmake/modules/LLVMLibCFlagRules.cmake
+++ b/libc/cmake/modules/LLVMLibCFlagRules.cmake
@@ -263,6 +263,9 @@ set(FMA_OPT_FLAG "FMA_OPT")
 set(ROUND_OPT_FLAG "ROUND_OPT")
 # This flag controls whether we use explicit SIMD instructions or not.
 set(EXPLICIT_SIMD_OPT_FLAG "EXPLICIT_SIMD_OPT")
+# This flag controls whether we use compiler builtin functions to implement
+# various basic math operations or not.
+set(MISC_MATH_BASIC_OPS_OPT_FLAG "MISC_MATH_BASIC_OPS_OPT")
 
 # Skip FMA_OPT flag for targets that don't support fma.
 if(NOT((LIBC_TARGET_ARCHITECTURE_IS_X86 AND (LIBC_CPU_FEATURES MATCHES "FMA")) OR
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 0513aa6cebd2d..74360edff3f9a 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -424,7 +424,9 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
   COMPILE_OPTIONS
-    -O2
+    -O3
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
 )
 
 add_entrypoint_object(
@@ -436,7 +438,9 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
   COMPILE_OPTIONS
-    -O2
+    -O3
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
 )
 
 add_entrypoint_object(
@@ -448,7 +452,7 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
   COMPILE_OPTIONS
-    -O2
+    -O3
 )
 
 add_entrypoint_object(
@@ -460,8 +464,12 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.macros.properties.architectures
+    libc.src.__support.macros.properties.compiler
   COMPILE_OPTIONS
     -O3
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
 )
 
 add_entrypoint_object(
@@ -1443,6 +1451,8 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
   COMPILE_OPTIONS
     -O3
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
 )
 
 add_entrypoint_object(
@@ -1455,6 +1465,8 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
   COMPILE_OPTIONS
     -O3
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
 )
 
 add_entrypoint_object(
@@ -1480,6 +1492,8 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
   COMPILE_OPTIONS
     -O3
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
 )
 
 add_entrypoint_object(
diff --git a/libc/src/math/generic/copysign.cpp b/libc/src/math/generic/copysign.cpp
index 149d725af08e2..186bb2c5983f4 100644
--- a/libc/src/math/generic/copysign.cpp
+++ b/libc/src/math/generic/copysign.cpp
@@ -14,7 +14,11 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(double, copysign, (double x, double y)) {
+#ifdef __LIBC_MISC_MATH_BASIC_OPS_OPT
+  return __builtin_copysign(x, y);
+#else
   return fputil::copysign(x, y);
+#endif
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/copysignf.cpp b/libc/src/math/generic/copysignf.cpp
index 17cd70d37c308..c79e50b61ebda 100644
--- a/libc/src/math/generic/copysignf.cpp
+++ b/libc/src/math/generic/copysignf.cpp
@@ -14,7 +14,11 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(float, copysignf, (float x, float y)) {
+#ifdef __LIBC_MISC_MATH_BASIC_OPS_OPT
+  return __builtin_copysignf(x, y);
+#else
   return fputil::copysign(x, y);
+#endif
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/copysignf16.cpp b/libc/src/math/generic/copysignf16.cpp
index 42695b3b4a6de..546622f049ebe 100644
--- a/libc/src/math/generic/copysignf16.cpp
+++ b/libc/src/math/generic/copysignf16.cpp
@@ -14,7 +14,11 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(float16, copysignf16, (float16 x, float16 y)) {
+#ifdef __LIBC_MISC_MATH_BASIC_OPS_OPT
+  return __builtin_copysignf16(x, y);
+#else
   return fputil::copysign(x, y);
+#endif
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fabs.cpp b/libc/src/math/generic/fabs.cpp
index 472297aecb2f7..55fa958cd7c00 100644
--- a/libc/src/math/generic/fabs.cpp
+++ b/libc/src/math/generic/fabs.cpp
@@ -13,6 +13,12 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, fabs, (double x)) { return fputil::abs(x); }
+LLVM_LIBC_FUNCTION(double, fabs, (double x)) {
+#ifdef __LIBC_MISC_MATH_BASIC_OPS_OPT
+  return __builtin_fabs(x);
+#else
+  return fputil::abs(x);
+#endif
+}
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fabsf.cpp b/libc/src/math/generic/fabsf.cpp
index ad4fcb30c795d..2ba18d09bbd5b 100644
--- a/libc/src/math/generic/fabsf.cpp
+++ b/libc/src/math/generic/fabsf.cpp
@@ -13,6 +13,12 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float, fabsf, (float x)) { return fputil::abs(x); }
+LLVM_LIBC_FUNCTION(float, fabsf, (float x)) {
+#ifdef __LIBC_MISC_MATH_BASIC_OPS_OPT
+  return __builtin_fabsf(x);
+#else
+  return fputil::abs(x);
+#endif
+}
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fabsf16.cpp b/libc/src/math/generic/fabsf16.cpp
index 57671fb6067e2..02e11330db718 100644
--- a/libc/src/math/generic/fabsf16.cpp
+++ b/libc/src/math/generic/fabsf16.cpp
@@ -10,9 +10,20 @@
 #include "src/__support/FPUtil/BasicOperations.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/architectures.h"
+#include "src/__support/macros/properties/compiler.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float16, fabsf16, (float16 x)) { return fputil::abs(x); }
+LLVM_LIBC_FUNCTION(float16, fabsf16, (float16 x)) {
+  // For x86, GCC generates better code from the generic implementation.
+  // https://godbolt.org/z/K9orM4hTa
+#if defined(__LIBC_MISC_MATH_BASIC_OPS_OPT) &&                                 \
+    !(defined(LIBC_TARGET_ARCH_IS_X86) && defined(LIBC_COMPILER_IS_GCC))
+  return __builtin_fabsf16(x);
+#else
+  return fputil::abs(x);
+#endif
+}
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/math/performance_testing/BinaryOpSingleOutputPerf.h b/libc/test/src/math/performance_testing/BinaryOpSingleOutputPerf.h
index 995e41ba84b03..63d9768e21899 100644
--- a/libc/test/src/math/performance_testing/BinaryOpSingleOutputPerf.h
+++ b/libc/test/src/math/performance_testing/BinaryOpSingleOutputPerf.h
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/macros/config.h"
 #include "test/src/math/performance_testing/Timer.h"
@@ -28,11 +29,11 @@ template <typename T> class BinaryOpSingleOutputPerf {
   static void run_perf_in_range(Func myFunc, Func otherFunc,
                                 StorageType startingBit, StorageType endingBit,
                                 size_t N, size_t rounds, std::ofstream &log) {
-    if (endingBit - startingBit < N)
-      N = endingBit - startingBit;
+    if (sizeof(StorageType) <= sizeof(size_t))
+      N = cpp::min(N, static_cast<size_t>(endingBit - startingBit));
 
     auto runner = [=](Func func) {
-      volatile T result;
+      [[maybe_unused]] volatile T result;
       if (endingBit < startingBit) {
         return;
       }
diff --git a/libc/test/src/math/performance_testing/CMakeLists.txt b/libc/test/src/math/performance_testing/CMakeLists.txt
index 2ec5364059b17..72e1a730ac756 100644
--- a/libc/test/src/math/performance_testing/CMakeLists.txt
+++ b/libc/test/src/math/performance_testing/CMakeLists.txt
@@ -95,6 +95,9 @@ add_header_library(
   single_input_single_output_diff
   HDRS
     SingleInputSingleOutputPerf.h
+  DEPENDS
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
 )
 
 add_header_library(
@@ -102,6 +105,7 @@ add_header_library(
   HDRS
     BinaryOpSingleOutputPerf.h
   DEPENDS
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -402,3 +406,18 @@ add_perf_binary(
   LINK_LIBRARIES
     LibcFPTestHelpers
 )
+
+add_perf_binary(
+  misc_basic_ops_perf
+  SRCS
+    misc_basic_ops_perf.cpp
+  DEPENDS
+    .binary_op_single_output_diff
+    .single_input_single_output_diff
+    libc.src.math.copysignf
+    libc.src.math.copysignf16
+    libc.src.math.fabsf
+    libc.src.math.fabsf16
+  COMPILE_OPTIONS
+    -fno-builtin
+)
diff --git a/libc/test/src/math/performance_testing/SingleInputSingleOutputPerf.h b/libc/test/src/math/performance_testing/SingleInputSingleOutputPerf.h
index 48ae43d6315e3..efad1259d6bf1 100644
--- a/libc/test/src/math/performance_testing/SingleInputSingleOutputPerf.h
+++ b/libc/test/src/math/performance_testing/SingleInputSingleOutputPerf.h
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/macros/config.h"
 #include "test/src/math/performance_testing/Timer.h"
@@ -26,16 +27,21 @@ template <typename T> class SingleInputSingleOutputPerf {
 
   static void runPerfInRange(Func myFunc, Func otherFunc,
                              StorageType startingBit, StorageType endingBit,
-                             std::ofstream &log) {
+                             size_t rounds, std::ofstream &log) {
+    size_t n = 10'010'001;
+    if (sizeof(StorageType) <= sizeof(size_t))
+      n = cpp::min(n, static_cast<size_t>(endingBit - startingBit));
+
     auto runner = [=](Func func) {
-      constexpr StorageType N = 10'010'001;
-      StorageType step = (endingBit - startingBit) / N;
+      StorageType step = (endingBit - startingBit) / n;
       if (step == 0)
         step = 1;
-      volatile T result;
-      for (StorageType bits = startingBit; bits < endingBit; bits += step) {
-        T x = FPBits(bits).get_val();
-        result = func(x);
+      [[maybe_unused]] volatile T result;
+      for (size_t i = 0; i < rounds; i++) {
+        for (StorageType bits = startingBit; bits < endingBit; bits += step) {
+          T x = FPBits(bits).get_val();
+          result = func(x);
+        }
       }
     };
 
@@ -44,8 +50,7 @@ template <typename T> class SingleInputSingleOutputPerf {
     runner(myFunc);
     timer.stop();
 
-    StorageType numberOfRuns = endingBit - startingBit + 1;
-    double myAverage = static_cast<double>(timer.nanoseconds()) / numberOfRuns;
+    double myAverage = static_cast<double>(timer.nanoseconds()) / n / rounds;
     log << "-- My function --\n";
     log << "     Total time      : " << timer.nanoseconds() << " ns \n";
     log << "     Average runtime : " << myAverage << " ns/op \n";
@@ -56,8 +61,7 @@ template <typename T> class SingleInputSingleOutputPerf {
     runner(otherFunc);
     timer.stop();
 
-    double otherAverage =
-        static_cast<double>(timer.nanoseconds()) / numberOfRuns;
+    double otherAverage = static_cast<double>(timer.nanoseconds()) / n / rounds;
     log << "-- Other function --\n";
     log << "     Total time      : " << timer.nanoseconds() << " ns \n";
     log << "     Average runtime : " << otherAverage << " ns/op \n";
@@ -68,15 +72,18 @@ template <typename T> class SingleInputSingleOutputPerf {
     log << "     Mine / Other's  : " << myAverage / otherAverage << " \n";
   }
 
-  static void runPerf(Func myFunc, Func otherFunc, const char *logFile) {
+  static void runPerf(Func myFunc, Func otherFunc, size_t rounds,
+                      const char *logFile) {
     std::ofstream log(logFile);
     log << " Performance tests with inputs in denormal range:\n";
     runPerfInRange(myFunc, otherFunc, /* startingBit= */ StorageType(0),
-                   /* endingBit= */ FPBits::max_subnormal().uintval(), log);
+                   /* endingBit= */ FPBits::max_subnormal().uintval(), rounds,
+                   log);
     log << "\n Performance tests with inputs in normal range:\n";
     runPerfInRange(myFunc, otherFunc,
                    /* startingBit= */ FPBits::min_normal().uintval(),
-                   /* endingBit= */ FPBits::max_normal().uintval(), log);
+                   /* endingBit= */ FPBits::max_normal().uintval(), rounds,
+                   log);
   }
 };
 
@@ -86,6 +93,13 @@ template <typename T> class SingleInputSingleOutputPerf {
 #define SINGLE_INPUT_SINGLE_OUTPUT_PERF(T, myFunc, otherFunc, filename)        \
   int main() {                                                                 \
     LIBC_NAMESPACE::testing::SingleInputSingleOutputPerf<T>::runPerf(          \
-        &myFunc, &otherFunc, filename);                                        \
+        &myFunc, &otherFunc, 1, filename);                                     \
     return 0;                                                                  \
   }
+
+#define SINGLE_INPUT_SINGLE_OUTPUT_PERF_EX(T, myFunc, otherFunc, rounds,       \
+                                           filename)                           \
+  {                                                                            \
+    LIBC_NAMESPACE::testing::SingleInputSingleOutputPerf<T>::runPerf(          \
+        &myFunc, &otherFunc, rounds, filename);                                \
+  }
diff --git a/libc/test/src/math/performance_testing/misc_basic_ops_perf.cpp b/libc/test/src/math/performance_testing/misc_basic_ops_perf.cpp
new file mode 100644
index 0000000000000..ace1d21c62c32
--- /dev/null
+++ b/libc/test/src/math/performance_testing/misc_basic_ops_perf.cpp
@@ -0,0 +1,41 @@
+//===-- Performance test for miscellaneous basic operations ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "BinaryOpSingleOutputPerf.h"
+#include "SingleInputSingleOutputPerf.h"
+#include "src/math/copysignf.h"
+#include "src/math/copysignf16.h"
+#include "src/math/fabsf.h"
+#include "src/math/fabsf16.h"
+
+#include <math.h>
+
+static constexpr size_t FLOAT16_ROUNDS = 20'000;
+static constexpr size_t FLOAT_ROUNDS = 40;
+
+// LLVM libc might be the only libc implementation with support for float16 math
+// functions currently. We can't compare our float16 functions against the
+// system libc, so we compare them against this placeholder function.
+float16 placeholder_unaryf16(float16 x) { return x; }
+float16 placeholder_binaryf16(float16 x, float16 y) { return x; }
+
+int main() {
+  SINGLE_INPUT_SINGLE_OUTPUT_PERF_EX(float16, LIBC_NAMESPACE::fabsf16,
+                                     placeholder_unaryf16, FLOAT16_ROUNDS,
+                                     "fabsf16_perf.log")
+  BINARY_OP_SINGLE_OUTPUT_PERF_EX(float16, LIBC_NAMESPACE::copysignf16,
+                                  placeholder_binaryf16, FLOAT16_ROUNDS,
+                                  "copysignf16_perf.log")
+
+  SINGLE_INPUT_SINGLE_OUTPUT_PERF_EX(float, LIBC_NAMESPACE::fabsf, fabsf,
+                                     FLOAT_ROUNDS, "fabsf_perf.log")
+  BINARY_OP_SINGLE_OUTPUT_PERF_EX(float, LIBC_NAMESPACE::copysignf, copysignf,
+                                  FLOAT_ROUNDS, "copysignf_perf.log")
+
+  return 0;
+}

From c80b799e90c6279ced025c44b9177c6e5f0f2ada Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Mon, 22 Jul 2024 18:40:47 +0200
Subject: [PATCH 448/486] [libc] No need to use recursion in fcntl (#99893)

This patch removes the recursion in fcntl introduced by PR #99675 as it is not required and may be dangerous in some cases: some toolchains define F_GETLK == F_GETLK64 causing infinite recursion.
---
 libc/src/__support/OSUtil/linux/fcntl.cpp | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/libc/src/__support/OSUtil/linux/fcntl.cpp b/libc/src/__support/OSUtil/linux/fcntl.cpp
index 7731aa04a178d..4742b2a00220b 100644
--- a/libc/src/__support/OSUtil/linux/fcntl.cpp
+++ b/libc/src/__support/OSUtil/linux/fcntl.cpp
@@ -33,7 +33,8 @@ int fcntl(int fd, int cmd, void *arg) {
 #error "fcntl and fcntl64 syscalls not available."
 #endif
 
-  switch (cmd) {
+  int new_cmd = cmd;
+  switch (new_cmd) {
   case F_OFD_SETLKW: {
     struct flock *flk = reinterpret_cast<struct flock *>(arg);
     // convert the struct to a flock64
@@ -44,7 +45,8 @@ int fcntl(int fd, int cmd, void *arg) {
     flk64.l_len = flk->l_len;
     flk64.l_pid = flk->l_pid;
     // create a syscall
-    return LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, cmd, &flk64);
+    return LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, new_cmd,
+                                             &flk64);
   }
   case F_OFD_GETLK:
   case F_OFD_SETLK: {
@@ -57,8 +59,8 @@ int fcntl(int fd, int cmd, void *arg) {
     flk64.l_len = flk->l_len;
     flk64.l_pid = flk->l_pid;
     // create a syscall
-    int retVal =
-        LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, cmd, &flk64);
+    int retVal = LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd,
+                                                   new_cmd, &flk64);
     // On failure, return
     if (retVal == -1)
       return -1;
@@ -89,22 +91,22 @@ int fcntl(int fd, int cmd, void *arg) {
 #ifdef SYS_fcntl64
   case F_GETLK: {
     if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64)
-      return fcntl(fd, F_GETLK64, arg);
+      new_cmd = F_GETLK64;
     break;
   }
   case F_SETLK: {
     if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64)
-      return fcntl(fd, F_SETLK64, arg);
+      new_cmd = F_SETLK64;
     break;
   }
   case F_SETLKW: {
     if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64)
-      return fcntl(fd, F_SETLKW64, arg);
+      new_cmd = F_SETLKW64;
     break;
   }
 #endif
   }
-  int retVal = LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, cmd,
+  int retVal = LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, new_cmd,
                                                  reinterpret_cast<void *>(arg));
   if (retVal >= 0) {
     return retVal;

From 1c798e0b077f062dbe56603021a9b67c7621ffe0 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 22 Jul 2024 09:41:57 -0700
Subject: [PATCH 449/486] [SelectionDAGBuilder][RISCV] Fix crash when using a
 memory constraint with scalable vector type. (#99821)

We need to use the minimum size of the scalable type and the correct
stack ID.

The code in the PR is still invalid because the instruction used doesn't
have a pointer operand. This is diagnosed later when the assembler
parses it.

Fixes #99782
---
 .../SelectionDAG/SelectionDAGBuilder.cpp       | 11 ++++++++---
 llvm/test/CodeGen/RISCV/rvv/pr99782.ll         | 18 ++++++++++++++++++
 2 files changed, 26 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/pr99782.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 98a795edb7a03..37b1131d2f8a3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -9566,10 +9566,15 @@ static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
   // Otherwise, create a stack slot and emit a store to it before the asm.
   Type *Ty = OpVal->getType();
   auto &DL = DAG.getDataLayout();
-  uint64_t TySize = DL.getTypeAllocSize(Ty);
+  TypeSize TySize = DL.getTypeAllocSize(Ty);
   MachineFunction &MF = DAG.getMachineFunction();
-  int SSFI = MF.getFrameInfo().CreateStackObject(
-      TySize, DL.getPrefTypeAlign(Ty), false);
+  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  int StackID = 0;
+  if (TySize.isScalable())
+    StackID = TFI->getStackIDForScalableVectors();
+  int SSFI = MF.getFrameInfo().CreateStackObject(TySize.getKnownMinValue(),
+                                                 DL.getPrefTypeAlign(Ty), false,
+                                                 nullptr, StackID);
   SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
   Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
                             MachinePointerInfo::getFixedStack(MF, SSFI),
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr99782.ll b/llvm/test/CodeGen/RISCV/rvv/pr99782.ll
new file mode 100644
index 0000000000000..92c40f41f02b5
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/pr99782.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v -stop-after=finalize-isel | FileCheck %s
+
+define void @vslidedown() {
+  ; CHECK-LABEL: name: vslidedown
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI %stack.0.v, 0
+  ; CHECK-NEXT:   [[VL8RE8_V:%[0-9]+]]:vrm8 = VL8RE8_V killed [[ADDI]] :: (load (<vscale x 1 x s512>) from %ir.v, align 1)
+  ; CHECK-NEXT:   [[ADDI1:%[0-9]+]]:gpr = ADDI %stack.1, 0
+  ; CHECK-NEXT:   VS8R_V killed [[VL8RE8_V]], killed [[ADDI1]] :: (store (<vscale x 1 x s512>) into %stack.1)
+  ; CHECK-NEXT:   INLINEASM &"vadd.vv $0, $0, $0", 25 /* sideeffect mayload maystore attdialect */, 262166 /* mem:m */, %stack.0.v, 0, 262166 /* mem:m */, %stack.1, 0
+  ; CHECK-NEXT:   PseudoRET
+entry:
+  %v = alloca <vscale x 64 x i8>, align 1
+  %0 = load <vscale x 64 x i8>, ptr %v, align 1
+  call void asm sideeffect "vadd.vv $0, $0, $0", "=*imr,imr"(ptr elementtype(<vscale x 64 x i8>) %v, <vscale x 64 x i8> %0)
+  ret void
+}

From 90569e02e63ff5d0915446919f564e9b3638fe2a Mon Sep 17 00:00:00 2001
From: Daniel Bertalan <dani@danielbertalan.dev>
Date: Mon, 22 Jul 2024 19:06:43 +0200
Subject: [PATCH 450/486] [Support] Add Arm NEON implementation for
 `llvm::xxh3_64bits` (#99634)

Compared to the generic scalar code, using Arm NEON instructions yields
a ~11x speedup: 31 vs 339.5 ms to hash 1 GiB of random data on the Apple
M1.

This follows the upstream implementation closely, with some
simplifications made:
- Removed workarounds for suboptimal codegen on older GCC
- Removed instruction reordering barriers which seem to have a
negligible impact according to my measurements
- We do not support WebAssembly's mostly NEON-compatible API
- There is no configurable mixing of SIMD and scalar code; according to
the upstream comments, this is only relevant for smaller Cortex cores
which can dispatch relatively few NEON micro-ops per cycle.

This commit intends to use only standard ACLE intrinsics and datatypes,
so it should build with all supported versions of GCC, Clang and MSVC.

This feature is enabled by default when targeting AArch64, but the
`LLVM_XXH_USE_NEON=0` macro can be set to explicitly disable it.

XXH3 is used for ICF, string deduplication and computing the UUID in
ld64.lld; this commit results in a -1.77% +/- 0.59% speed improvement
for a `--threads=8` link of Chromium.framework.
---
 llvm/benchmarks/CMakeLists.txt |   3 +-
 llvm/benchmarks/xxhash.cpp     |  29 +++++
 llvm/lib/Support/xxhash.cpp    | 192 +++++++++++++++++++++++++++++----
 3 files changed, 203 insertions(+), 21 deletions(-)
 create mode 100644 llvm/benchmarks/xxhash.cpp

diff --git a/llvm/benchmarks/CMakeLists.txt b/llvm/benchmarks/CMakeLists.txt
index 43f88f7257924..52d726451ada9 100644
--- a/llvm/benchmarks/CMakeLists.txt
+++ b/llvm/benchmarks/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
   Support)
 
-add_benchmark(DummyYAML DummyYAML.cpp)
+add_benchmark(DummyYAML DummyYAML.cpp PARTIAL_SOURCES_INTENDED)
+add_benchmark(xxhash xxhash.cpp PARTIAL_SOURCES_INTENDED)
diff --git a/llvm/benchmarks/xxhash.cpp b/llvm/benchmarks/xxhash.cpp
new file mode 100644
index 0000000000000..429cbc0fa87d4
--- /dev/null
+++ b/llvm/benchmarks/xxhash.cpp
@@ -0,0 +1,29 @@
+#include "llvm/Support/xxhash.h"
+#include "benchmark/benchmark.h"
+
+#include <memory>
+
+static uint32_t xorshift(uint32_t State) {
+  State ^= State << 13;
+  State ^= State >> 17;
+  State ^= State << 5;
+  return State;
+}
+
+static void BM_xxh3_64bits(benchmark::State &State) {
+  std::unique_ptr<uint32_t[]> Data(new uint32_t[State.range(0) / 4]);
+
+  uint32_t Prev = 0xcafebabe;
+  for (int64_t I = 0; I < State.range(0) / 4; I++)
+    Data[I] = Prev = xorshift(Prev);
+
+  llvm::ArrayRef DataRef =
+      llvm::ArrayRef(reinterpret_cast<uint8_t *>(Data.get()), State.range(0));
+
+  for (auto _ : State)
+    llvm::xxh3_64bits(DataRef);
+}
+
+BENCHMARK(BM_xxh3_64bits)->Arg(32)->Arg(512)->Arg(64 * 1024)->Arg(1024 * 1024);
+
+BENCHMARK_MAIN();
diff --git a/llvm/lib/Support/xxhash.cpp b/llvm/lib/Support/xxhash.cpp
index 607789b391381..cdb76d57e2c1d 100644
--- a/llvm/lib/Support/xxhash.cpp
+++ b/llvm/lib/Support/xxhash.cpp
@@ -47,6 +47,19 @@
 
 #include <stdlib.h>
 
+#if !defined(LLVM_XXH_USE_NEON)
+#if (defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) &&      \
+    !defined(__ARM_BIG_ENDIAN)
+#define LLVM_XXH_USE_NEON 1
+#else
+#define LLVM_XXH_USE_NEON 0
+#endif
+#endif
+
+#if LLVM_XXH_USE_NEON
+#include <arm_neon.h>
+#endif
+
 using namespace llvm;
 using namespace support;
 
@@ -323,6 +336,144 @@ static uint64_t XXH3_len_129to240_64b(const uint8_t *input, size_t len,
   return XXH3_avalanche(acc);
 }
 
+#if LLVM_XXH_USE_NEON
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_neon
+#define XXH3_scrambleAcc XXH3_scrambleAcc_neon
+
+// NEON implementation based on commit a57f6cce2698049863af8c25787084ae0489d849
+// (July 2024), with the following removed:
+// - workaround for suboptimal codegen on older GCC
+// - compiler barriers against instruction reordering
+// - WebAssembly SIMD support
+// - configurable split between NEON and scalar lanes (benchmarking shows no
+//   penalty when fully doing SIMD on the Apple M1)
+
+#if defined(__GNUC__) || defined(__clang__)
+#define XXH_ALIASING __attribute__((__may_alias__))
+#else
+#define XXH_ALIASING /* nothing */
+#endif
+
+typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING;
+
+LLVM_ATTRIBUTE_ALWAYS_INLINE static uint64x2_t XXH_vld1q_u64(void const *ptr) {
+  return vreinterpretq_u64_u8(vld1q_u8((uint8_t const *)ptr));
+}
+
+LLVM_ATTRIBUTE_ALWAYS_INLINE
+static void XXH3_accumulate_512_neon(uint64_t *acc, const uint8_t *input,
+                                     const uint8_t *secret) {
+  xxh_aliasing_uint64x2_t *const xacc = (xxh_aliasing_uint64x2_t *)acc;
+
+#ifdef __clang__
+#pragma clang loop unroll(full)
+#endif
+  for (size_t i = 0; i < XXH_ACC_NB / 2; i += 2) {
+    /* data_vec = input[i]; */
+    uint64x2_t data_vec_1 = XXH_vld1q_u64(input + (i * 16));
+    uint64x2_t data_vec_2 = XXH_vld1q_u64(input + ((i + 1) * 16));
+
+    /* key_vec  = secret[i];  */
+    uint64x2_t key_vec_1 = XXH_vld1q_u64(secret + (i * 16));
+    uint64x2_t key_vec_2 = XXH_vld1q_u64(secret + ((i + 1) * 16));
+
+    /* data_swap = swap(data_vec) */
+    uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1);
+    uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1);
+
+    /* data_key = data_vec ^ key_vec; */
+    uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1);
+    uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2);
+
+    /*
+     * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a
+     * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to
+     * get one vector with the low 32 bits of each lane, and one vector
+     * with the high 32 bits of each lane.
+     *
+     * The intrinsic returns a double vector because the original ARMv7-a
+     * instruction modified both arguments in place. AArch64 and SIMD128 emit
+     * two instructions from this intrinsic.
+     *
+     *  [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ]
+     *  [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ]
+     */
+    uint32x4x2_t unzipped = vuzpq_u32(vreinterpretq_u32_u64(data_key_1),
+                                      vreinterpretq_u32_u64(data_key_2));
+
+    /* data_key_lo = data_key & 0xFFFFFFFF */
+    uint32x4_t data_key_lo = unzipped.val[0];
+    /* data_key_hi = data_key >> 32 */
+    uint32x4_t data_key_hi = unzipped.val[1];
+
+    /*
+     * Then, we can split the vectors horizontally and multiply which, as for
+     * most widening intrinsics, have a variant that works on both high half
+     * vectors for free on AArch64. A similar instruction is available on
+     * SIMD128.
+     *
+     * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi
+     */
+    uint64x2_t sum_1 = vmlal_u32(data_swap_1, vget_low_u32(data_key_lo),
+                                 vget_low_u32(data_key_hi));
+    uint64x2_t sum_2 = vmlal_u32(data_swap_2, vget_high_u32(data_key_lo),
+                                 vget_high_u32(data_key_hi));
+
+    /* xacc[i] = acc_vec + sum; */
+    xacc[i] = vaddq_u64(xacc[i], sum_1);
+    xacc[i + 1] = vaddq_u64(xacc[i + 1], sum_2);
+  }
+}
+
+LLVM_ATTRIBUTE_ALWAYS_INLINE
+static void XXH3_scrambleAcc_neon(uint64_t *acc, const uint8_t *secret) {
+  xxh_aliasing_uint64x2_t *const xacc = (xxh_aliasing_uint64x2_t *)acc;
+
+  /* { prime32_1, prime32_1 } */
+  uint32x2_t const kPrimeLo = vdup_n_u32(PRIME32_1);
+  /* { 0, prime32_1, 0, prime32_1 } */
+  uint32x4_t const kPrimeHi =
+      vreinterpretq_u32_u64(vdupq_n_u64((uint64_t)PRIME32_1 << 32));
+
+  for (size_t i = 0; i < XXH_ACC_NB / 2; ++i) {
+    /* xacc[i] ^= (xacc[i] >> 47); */
+    uint64x2_t acc_vec = XXH_vld1q_u64(acc + (2 * i));
+    uint64x2_t shifted = vshrq_n_u64(acc_vec, 47);
+    uint64x2_t data_vec = veorq_u64(acc_vec, shifted);
+
+    /* xacc[i] ^= secret[i]; */
+    uint64x2_t key_vec = XXH_vld1q_u64(secret + (i * 16));
+    uint64x2_t data_key = veorq_u64(data_vec, key_vec);
+
+    /*
+     * xacc[i] *= XXH_PRIME32_1
+     *
+     * Expanded version with portable NEON intrinsics
+     *
+     *    lo(x) * lo(y) + (hi(x) * lo(y) << 32)
+     *
+     * prod_hi = hi(data_key) * lo(prime) << 32
+     *
+     * Since we only need 32 bits of this multiply a trick can be used,
+     * reinterpreting the vector as a uint32x4_t and multiplying by
+     * { 0, prime, 0, prime } to cancel out the unwanted bits and avoid the
+     * shift.
+     */
+    uint32x4_t prod_hi = vmulq_u32(vreinterpretq_u32_u64(data_key), kPrimeHi);
+
+    /* Extract low bits for vmlal_u32  */
+    uint32x2_t data_key_lo = vmovn_u64(data_key);
+
+    /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */
+    xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo);
+  }
+}
+#else
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_scalar
+#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar
+
 LLVM_ATTRIBUTE_ALWAYS_INLINE
 static void XXH3_accumulate_512_scalar(uint64_t *acc, const uint8_t *input,
                                        const uint8_t *secret) {
@@ -335,20 +486,23 @@ static void XXH3_accumulate_512_scalar(uint64_t *acc, const uint8_t *input,
 }
 
 LLVM_ATTRIBUTE_ALWAYS_INLINE
-static void XXH3_accumulate_scalar(uint64_t *acc, const uint8_t *input,
-                                   const uint8_t *secret, size_t nbStripes) {
-  for (size_t n = 0; n < nbStripes; ++n)
-    XXH3_accumulate_512_scalar(acc, input + n * XXH_STRIPE_LEN,
-                               secret + n * XXH_SECRET_CONSUME_RATE);
-}
-
-static void XXH3_scrambleAcc(uint64_t *acc, const uint8_t *secret) {
+static void XXH3_scrambleAcc_scalar(uint64_t *acc, const uint8_t *secret) {
   for (size_t i = 0; i < XXH_ACC_NB; ++i) {
     acc[i] ^= acc[i] >> 47;
     acc[i] ^= endian::read64le(secret + 8 * i);
     acc[i] *= PRIME32_1;
   }
 }
+#endif
+
+LLVM_ATTRIBUTE_ALWAYS_INLINE
+static void XXH3_accumulate(uint64_t *acc, const uint8_t *input,
+                            const uint8_t *secret, size_t nbStripes) {
+  for (size_t n = 0; n < nbStripes; ++n) {
+    XXH3_accumulate_512(acc, input + n * XXH_STRIPE_LEN,
+                        secret + n * XXH_SECRET_CONSUME_RATE);
+  }
+}
 
 static uint64_t XXH3_mix2Accs(const uint64_t *acc, const uint8_t *secret) {
   return XXH3_mul128_fold64(acc[0] ^ endian::read64le(secret),
@@ -375,21 +529,20 @@ static uint64_t XXH3_hashLong_64b(const uint8_t *input, size_t len,
       PRIME64_4, PRIME32_2, PRIME64_5, PRIME32_1,
   };
   for (size_t n = 0; n < nb_blocks; ++n) {
-    XXH3_accumulate_scalar(acc, input + n * block_len, secret,
-                           nbStripesPerBlock);
+    XXH3_accumulate(acc, input + n * block_len, secret, nbStripesPerBlock);
     XXH3_scrambleAcc(acc, secret + secretSize - XXH_STRIPE_LEN);
   }
 
   /* last partial block */
   const size_t nbStripes = (len - 1 - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
   assert(nbStripes <= secretSize / XXH_SECRET_CONSUME_RATE);
-  XXH3_accumulate_scalar(acc, input + nb_blocks * block_len, secret, nbStripes);
+  XXH3_accumulate(acc, input + nb_blocks * block_len, secret, nbStripes);
 
   /* last stripe */
   constexpr size_t XXH_SECRET_LASTACC_START = 7;
-  XXH3_accumulate_512_scalar(acc, input + len - XXH_STRIPE_LEN,
-                             secret + secretSize - XXH_STRIPE_LEN -
-                                 XXH_SECRET_LASTACC_START);
+  XXH3_accumulate_512(acc, input + len - XXH_STRIPE_LEN,
+                      secret + secretSize - XXH_STRIPE_LEN -
+                          XXH_SECRET_LASTACC_START);
 
   /* converge into final hash */
   constexpr size_t XXH_SECRET_MERGEACCS_START = 11;
@@ -840,21 +993,20 @@ XXH3_hashLong_128b(const uint8_t *input, size_t len, const uint8_t *secret,
   };
 
   for (size_t n = 0; n < nb_blocks; ++n) {
-    XXH3_accumulate_scalar(acc, input + n * block_len, secret,
-                           nbStripesPerBlock);
+    XXH3_accumulate(acc, input + n * block_len, secret, nbStripesPerBlock);
     XXH3_scrambleAcc(acc, secret + secretSize - XXH_STRIPE_LEN);
   }
 
   /* last partial block */
   const size_t nbStripes = (len - 1 - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
   assert(nbStripes <= secretSize / XXH_SECRET_CONSUME_RATE);
-  XXH3_accumulate_scalar(acc, input + nb_blocks * block_len, secret, nbStripes);
+  XXH3_accumulate(acc, input + nb_blocks * block_len, secret, nbStripes);
 
   /* last stripe */
   constexpr size_t XXH_SECRET_LASTACC_START = 7;
-  XXH3_accumulate_512_scalar(acc, input + len - XXH_STRIPE_LEN,
-                             secret + secretSize - XXH_STRIPE_LEN -
-                                 XXH_SECRET_LASTACC_START);
+  XXH3_accumulate_512(acc, input + len - XXH_STRIPE_LEN,
+                      secret + secretSize - XXH_STRIPE_LEN -
+                          XXH_SECRET_LASTACC_START);
 
   /* converge into final hash */
   static_assert(sizeof(acc) == 64);

From f2eb7c7344a56381e373b110527e09dbed51da32 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 22 Jul 2024 10:26:05 -0700
Subject: [PATCH 451/486] [test] Delete a redundant mapping symbol test

Covered by llvm/test/tools/llvm-nm/special-syms-arm.test
---
 llvm/test/Object/ARM/nm-mapping-symbol.s | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 llvm/test/Object/ARM/nm-mapping-symbol.s

diff --git a/llvm/test/Object/ARM/nm-mapping-symbol.s b/llvm/test/Object/ARM/nm-mapping-symbol.s
deleted file mode 100644
index 6b41876c35e5e..0000000000000
--- a/llvm/test/Object/ARM/nm-mapping-symbol.s
+++ /dev/null
@@ -1,12 +0,0 @@
-// RUN: llvm-mc %s -o %t.o -filetype=obj -triple=armv7-pc-linux
-// RUN: llvm-readobj --symbols %t.o | FileCheck %s
-// RUN: llvm-nm %t.o | FileCheck -allow-empty --check-prefix=NM %s
-
-// Test that nm doesn't print the mapping symbols
-
-// CHECK: Name: $d.0
-// NM-NOT: $d.0
-
-        .section        .foobar,"",%progbits
-        .asciz  "foo"
-        nop

From 3f8d77bcc7f3c6ada0650d848d76ea9827e78de4 Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov@arm.com>
Date: Mon, 22 Jul 2024 17:22:56 +0000
Subject: [PATCH 452/486] Revert "[AArch64][SVE] Improve code quality of vector
 unsigned/signed add reductions. (#97339)"

This reverts commit b7b0071680e60c60da9d4d858f944fd95d76fd42.

The change caused regression in a performance testing.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  72 +------------
 llvm/test/CodeGen/AArch64/double_reduct.ll    |   9 +-
 llvm/test/CodeGen/AArch64/sve-doublereduct.ll |  41 ++++---
 .../CodeGen/AArch64/sve-fixed-vector-zext.ll  |  44 +++-----
 llvm/test/CodeGen/AArch64/sve-int-reduce.ll   |  88 ---------------
 ...-streaming-mode-fixed-length-reductions.ll | 102 ------------------
 llvm/test/CodeGen/AArch64/vecreduce-add.ll    |  52 ++++-----
 7 files changed, 78 insertions(+), 330 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index eae0200f37f04..99f9758a3716d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -17576,71 +17576,6 @@ static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
   return DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, UADDLP);
 }
 
-// Turn [sign|zero]_extend(vecreduce_add()) into SVE's  SADDV|UADDV
-// instructions.
-static SDValue
-performVecReduceAddExtCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
-                              const AArch64TargetLowering &TLI) {
-  if (N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND &&
-      N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND)
-    return SDValue();
-  bool IsSigned = N->getOperand(0).getOpcode() == ISD::SIGN_EXTEND;
-
-  SelectionDAG &DAG = DCI.DAG;
-  auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
-  SDValue VecOp = N->getOperand(0).getOperand(0);
-  EVT VecOpVT = VecOp.getValueType();
-  SDLoc DL(N);
-
-  // Split the input vectors if not legal, e.g.
-  // i32 (vecreduce_add (zext nxv32i8 %op to nxv32i32))
-  // ->
-  // i32 (add
-  //   (i32 vecreduce_add (zext nxv16i8 %op.lo to nxv16i32)),
-  //   (i32 vecreduce_add (zext nxv16i8 %op.hi to nxv16i32)))
-  if (TLI.getTypeAction(*DAG.getContext(), VecOpVT) ==
-      TargetLowering::TypeSplitVector) {
-    SDValue Lo, Hi;
-    std::tie(Lo, Hi) = DAG.SplitVector(VecOp, DL);
-    unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
-    EVT HalfVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
-        *DAG.getContext());
-    Lo = DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0),
-                     DAG.getNode(ExtOpc, DL, HalfVT, Lo));
-    Hi = DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0),
-                     DAG.getNode(ExtOpc, DL, HalfVT, Hi));
-    return DAG.getNode(ISD::ADD, DL, N->getValueType(0), Lo, Hi);
-  }
-
-  if (!TLI.isTypeLegal(VecOpVT))
-    return SDValue();
-
-  if (VecOpVT.isFixedLengthVector() &&
-      !TLI.useSVEForFixedLengthVectorVT(VecOpVT, !Subtarget.isNeonAvailable()))
-    return SDValue();
-
-  // The input type is legal so map VECREDUCE_ADD to UADDV/SADDV, e.g.
-  // i32 (vecreduce_add (zext nxv16i8 %op to nxv16i32))
-  // ->
-  // i32 (UADDV nxv16i8:%op)
-  EVT ElemType = N->getValueType(0);
-  SDValue Pg = getPredicateForVector(DAG, DL, VecOpVT);
-  if (VecOpVT.isFixedLengthVector()) {
-    EVT ContainerVT = getContainerForFixedLengthVector(DAG, VecOpVT);
-    VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
-  }
-  SDValue Res =
-      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64,
-                  DAG.getConstant(IsSigned ? Intrinsic::aarch64_sve_saddv
-                                           : Intrinsic::aarch64_sve_uaddv,
-                                  DL, MVT::i64),
-                  Pg, VecOp);
-  if (ElemType != MVT::i64)
-    Res = DAG.getAnyExtOrTrunc(Res, DL, ElemType);
-
-  return Res;
-}
-
 // Turn a v8i8/v16i8 extended vecreduce into a udot/sdot and vecreduce
 //   vecreduce.add(ext(A)) to vecreduce.add(DOT(zero, A, one))
 //   vecreduce.add(mul(ext(A), ext(B))) to vecreduce.add(DOT(zero, A, B))
@@ -25326,11 +25261,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
     return performInsertVectorEltCombine(N, DCI);
   case ISD::EXTRACT_VECTOR_ELT:
     return performExtractVectorEltCombine(N, DCI, Subtarget);
-  case ISD::VECREDUCE_ADD: {
-    if (SDValue Val = performVecReduceAddCombine(N, DCI.DAG, Subtarget))
-      return Val;
-    return performVecReduceAddExtCombine(N, DCI, *this);
-  }
+  case ISD::VECREDUCE_ADD:
+    return performVecReduceAddCombine(N, DCI.DAG, Subtarget);
   case AArch64ISD::UADDV:
     return performUADDVCombine(N, DAG);
   case AArch64ISD::SMULL:
diff --git a/llvm/test/CodeGen/AArch64/double_reduct.ll b/llvm/test/CodeGen/AArch64/double_reduct.ll
index cf5e15da0b173..b10114bc0ffa7 100644
--- a/llvm/test/CodeGen/AArch64/double_reduct.ll
+++ b/llvm/test/CodeGen/AArch64/double_reduct.ll
@@ -145,10 +145,11 @@ define i16 @add_ext_i16(<16 x i8> %a, <16 x i8> %b) {
 define i16 @add_ext_v32i16(<32 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: add_ext_v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uaddlp v1.8h, v1.16b
-; CHECK-NEXT:    uadalp v1.8h, v0.16b
-; CHECK-NEXT:    uadalp v1.8h, v2.16b
-; CHECK-NEXT:    addv h0, v1.8h
+; CHECK-NEXT:    uaddl2 v3.8h, v0.16b, v1.16b
+; CHECK-NEXT:    uaddl v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    add v0.8h, v0.8h, v3.8h
+; CHECK-NEXT:    uadalp v0.8h, v2.16b
+; CHECK-NEXT:    addv h0, v0.8h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
   %ae = zext <32 x i8> %a to <32 x i16>
diff --git a/llvm/test/CodeGen/AArch64/sve-doublereduct.ll b/llvm/test/CodeGen/AArch64/sve-doublereduct.ll
index b289dfbec527c..7bc31d44bb654 100644
--- a/llvm/test/CodeGen/AArch64/sve-doublereduct.ll
+++ b/llvm/test/CodeGen/AArch64/sve-doublereduct.ll
@@ -103,12 +103,17 @@ define i32 @add_i32(<vscale x 8 x i32> %a, <vscale x 4 x i32> %b) {
 define i16 @add_ext_i16(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: add_ext_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    uaddv d0, p0, z0.b
-; CHECK-NEXT:    uaddv d1, p0, z1.b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    fmov w9, s1
-; CHECK-NEXT:    add w0, w8, w9
+; CHECK-NEXT:    uunpkhi z2.h, z0.b
+; CHECK-NEXT:    uunpklo z0.h, z0.b
+; CHECK-NEXT:    uunpkhi z3.h, z1.b
+; CHECK-NEXT:    uunpklo z1.h, z1.b
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    add z0.h, z0.h, z2.h
+; CHECK-NEXT:    add z1.h, z1.h, z3.h
+; CHECK-NEXT:    add z0.h, z0.h, z1.h
+; CHECK-NEXT:    uaddv d0, p0, z0.h
+; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
   %ae = zext <vscale x 16 x i8> %a to <vscale x 16 x i16>
   %be = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>
@@ -121,15 +126,21 @@ define i16 @add_ext_i16(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 define i16 @add_ext_v32i16(<vscale x 32 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: add_ext_v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    uaddv d1, p0, z1.b
-; CHECK-NEXT:    uaddv d0, p0, z0.b
-; CHECK-NEXT:    uaddv d2, p0, z2.b
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    fmov w9, s2
-; CHECK-NEXT:    add w0, w8, w9
+; CHECK-NEXT:    uunpklo z3.h, z1.b
+; CHECK-NEXT:    uunpklo z4.h, z0.b
+; CHECK-NEXT:    uunpkhi z1.h, z1.b
+; CHECK-NEXT:    uunpkhi z0.h, z0.b
+; CHECK-NEXT:    uunpkhi z5.h, z2.b
+; CHECK-NEXT:    uunpklo z2.h, z2.b
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    add z0.h, z0.h, z1.h
+; CHECK-NEXT:    add z1.h, z4.h, z3.h
+; CHECK-NEXT:    add z0.h, z1.h, z0.h
+; CHECK-NEXT:    add z1.h, z2.h, z5.h
+; CHECK-NEXT:    add z0.h, z0.h, z1.h
+; CHECK-NEXT:    uaddv d0, p0, z0.h
+; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
   %ae = zext <vscale x 32 x i8> %a to <vscale x 32 x i16>
   %be = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-vector-zext.ll b/llvm/test/CodeGen/AArch64/sve-fixed-vector-zext.ll
index 24c817d410301..1ab2589bccd5b 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-vector-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-vector-zext.ll
@@ -1,4 +1,3 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 
 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v1 -O3 -aarch64-sve-vector-bits-min=256 -verify-machineinstrs | FileCheck %s --check-prefixes=SVE256
 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v1 -O3 -aarch64-sve-vector-bits-min=128 -verify-machineinstrs | FileCheck %s --check-prefixes=NEON
@@ -7,31 +6,24 @@
 
 define internal i32 @test(ptr nocapture readonly %p1, i32 %i1, ptr nocapture readonly %p2, i32 %i2) {
 ; SVE256-LABEL: test:
-; SVE256:       // %bb.0: // %L.entry
-; SVE256-NEXT:    ptrue p0.h, vl16
-; SVE256-NEXT:    mov w9, wzr
-; SVE256-NEXT:    mov w10, wzr
-; SVE256-NEXT:    mov w8, wzr
-; SVE256-NEXT:    mov w11, #-16 // =0xfffffff0
-; SVE256-NEXT:    .p2align 5, , 16
-; SVE256-NEXT:  .LBB0_1: // %L1
-; SVE256-NEXT:    // =>This Inner Loop Header: Depth=1
-; SVE256-NEXT:    sxtw x12, w9
-; SVE256-NEXT:    sxtw x13, w10
-; SVE256-NEXT:    adds w11, w11, #1
-; SVE256-NEXT:    add w10, w10, w3
-; SVE256-NEXT:    ld1b { z0.h }, p0/z, [x0, x12]
-; SVE256-NEXT:    ld1b { z1.h }, p0/z, [x2, x13]
-; SVE256-NEXT:    add w9, w9, w1
-; SVE256-NEXT:    sub z0.h, z0.h, z1.h
-; SVE256-NEXT:    saddv d0, p0, z0.h
-; SVE256-NEXT:    fmov w12, s0
-; SVE256-NEXT:    add w8, w12, w8
-; SVE256-NEXT:    b.lo .LBB0_1
-; SVE256-NEXT:  // %bb.2: // %L2
-; SVE256-NEXT:    mov w0, w8
-; SVE256-NEXT:    ret
+; SVE256:       ld1b    { z0.h }, p0/z,
+; SVE256:       ld1b    { z1.h }, p0/z,
+; SVE256:       sub z0.h, z0.h, z1.h
+; SVE256-NEXT:  sunpklo z1.s, z0.h
+; SVE256-NEXT:  ext z0.b, z0.b, z0.b, #16
+; SVE256-NEXT:  sunpklo z0.s, z0.h
+; SVE256-NEXT:  add z0.s, z1.s, z0.s
+; SVE256-NEXT:  uaddv   d0, p1, z0.s
 
+; NEON-LABEL: test:
+; NEON:       ldr q0, [x0, w9, sxtw]
+; NEON:       ldr q1, [x2, w10, sxtw]
+; NEON:       usubl2  v2.8h, v0.16b, v1.16b
+; NEON-NEXT:  usubl   v0.8h, v0.8b, v1.8b
+; NEON:       saddl2  v1.4s, v0.8h, v2.8h
+; NEON-NEXT:  saddl   v0.4s, v0.4h, v2.4h
+; NEON-NEXT:  add v0.4s, v0.4s, v1.4s
+; NEON-NEXT:  addv    s0, v0.4s
 
 L.entry:
   br label %L1
@@ -63,5 +55,3 @@ L2:                                          ; preds = %L1
 }
 
 declare  i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; NEON: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/sve-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve-int-reduce.ll
index c8dd719aa03c6..8c1b5225b7f25 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-reduce.ll
@@ -188,94 +188,6 @@ define i64 @uaddv_nxv2i64(<vscale x 2 x i64> %a) {
   ret i64 %res
 }
 
-define i32 @uaddv_nxv16i8_nxv16i32(<vscale x 16 x i8> %a) {
-; CHECK-LABEL: uaddv_nxv16i8_nxv16i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    uaddv d0, p0, z0.b
-; CHECK-NEXT:    fmov x0, d0
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-  %1 = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>
-  %2 = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> %1)
-  ret i32 %2
-}
-
-define i64 @uaddv_nxv16i16_nxv16i64(<vscale x 16 x i16> %a) {
-; CHECK-LABEL: uaddv_nxv16i16_nxv16i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    uaddv d1, p0, z1.h
-; CHECK-NEXT:    uaddv d0, p0, z0.h
-; CHECK-NEXT:    fmov x8, d1
-; CHECK-NEXT:    fmov x9, d0
-; CHECK-NEXT:    add x0, x9, x8
-; CHECK-NEXT:    ret
-  %1 = zext <vscale x 16 x i16> %a to <vscale x 16 x i64>
-  %2 = call i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64> %1)
-  ret i64 %2
-}
-
-define i32 @uaddv_nxv16i16_nxv16i32(<vscale x 32 x i16> %a) {
-; CHECK-LABEL: uaddv_nxv16i16_nxv16i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    uaddv d3, p0, z3.h
-; CHECK-NEXT:    uaddv d2, p0, z2.h
-; CHECK-NEXT:    uaddv d1, p0, z1.h
-; CHECK-NEXT:    uaddv d0, p0, z0.h
-; CHECK-NEXT:    fmov w8, s3
-; CHECK-NEXT:    fmov w9, s2
-; CHECK-NEXT:    fmov w10, s1
-; CHECK-NEXT:    fmov w11, s0
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w9, w11, w10
-; CHECK-NEXT:    add w0, w9, w8
-; CHECK-NEXT:    ret
-  %1 = zext <vscale x 32 x i16> %a to <vscale x 32 x i32>
-  %2 = call i32 @llvm.vector.reduce.add.nxv32i64(<vscale x 32 x i32> %1)
-  ret i32 %2
-}
-
-define i32 @saddv_nxv16i8_nxv16i32(<vscale x 16 x i8> %a) {
-; CHECK-LABEL: saddv_nxv16i8_nxv16i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    saddv d0, p0, z0.b
-; CHECK-NEXT:    fmov x0, d0
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
-  %1 = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>
-  %2 = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> %1)
-  ret i32 %2
-}
-
-define i32 @uaddv_nxv32i16_nxv32i32(ptr %a) {
-; CHECK-LABEL: uaddv_nxv32i16_nxv32i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0, #3, mul vl]
-; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT:    ld1h { z2.h }, p0/z, [x0, #1, mul vl]
-; CHECK-NEXT:    ld1h { z3.h }, p0/z, [x0]
-; CHECK-NEXT:    uaddv d0, p0, z0.h
-; CHECK-NEXT:    uaddv d1, p0, z1.h
-; CHECK-NEXT:    uaddv d2, p0, z2.h
-; CHECK-NEXT:    uaddv d3, p0, z3.h
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    fmov w9, s1
-; CHECK-NEXT:    fmov w10, s2
-; CHECK-NEXT:    fmov w11, s3
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w9, w11, w10
-; CHECK-NEXT:    add w0, w9, w8
-; CHECK-NEXT:    ret
-  %1 = load <vscale x 32 x i16>, ptr %a, align 16
-  %2 = zext <vscale x 32 x i16> %1 to <vscale x 32 x i32>
-  %3 = call i32 @llvm.vector.reduce.add.nxv32i32(<vscale x 32 x i32> %2)
-  ret i32 %3
-}
-
 ; UMINV
 
 define i8 @umin_nxv16i8(<vscale x 16 x i8> %a) {
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll
deleted file mode 100644
index 608b3bdeac75a..0000000000000
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll
+++ /dev/null
@@ -1,102 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve < %s | FileCheck %s -check-prefixes=CHECK,NO_STREAMING
-; RUN: llc -mattr=+sve -force-streaming-compatible -aarch64-sve-vector-bits-min=128 -aarch64-sve-vector-bits-max=128  < %s | FileCheck %s -check-prefixes=CHECK,SVE_128
-; RUN: llc -mattr=+sve -force-streaming-compatible -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,SVE_MIN_256
-
-target triple = "aarch64-unknown-linux-gnu"
-
-define i32 @reduce_uadd_v16i8(<32 x i8> %a) #0 {
-; NO_STREAMING-LABEL: reduce_uadd_v16i8:
-; NO_STREAMING:       // %bb.0:
-; NO_STREAMING-NEXT:    ushll2 v2.8h, v1.16b, #0
-; NO_STREAMING-NEXT:    ushll v1.8h, v1.8b, #0
-; NO_STREAMING-NEXT:    ushll2 v3.8h, v0.16b, #0
-; NO_STREAMING-NEXT:    ushll v0.8h, v0.8b, #0
-; NO_STREAMING-NEXT:    uaddl2 v4.4s, v1.8h, v2.8h
-; NO_STREAMING-NEXT:    uaddl v1.4s, v1.4h, v2.4h
-; NO_STREAMING-NEXT:    uaddl2 v2.4s, v0.8h, v3.8h
-; NO_STREAMING-NEXT:    uaddl v0.4s, v0.4h, v3.4h
-; NO_STREAMING-NEXT:    add v1.4s, v1.4s, v4.4s
-; NO_STREAMING-NEXT:    add v0.4s, v0.4s, v2.4s
-; NO_STREAMING-NEXT:    add v0.4s, v0.4s, v1.4s
-; NO_STREAMING-NEXT:    addv s0, v0.4s
-; NO_STREAMING-NEXT:    fmov w0, s0
-; NO_STREAMING-NEXT:    ret
-;
-; SVE_128-LABEL: reduce_uadd_v16i8:
-; SVE_128:       // %bb.0:
-; SVE_128-NEXT:    ptrue p0.b
-; SVE_128-NEXT:    // kill: def $q1 killed $q1 def $z1
-; SVE_128-NEXT:    // kill: def $q0 killed $q0 def $z0
-; SVE_128-NEXT:    uaddv d1, p0, z1.b
-; SVE_128-NEXT:    uaddv d0, p0, z0.b
-; SVE_128-NEXT:    fmov x8, d1
-; SVE_128-NEXT:    fmov x9, d0
-; SVE_128-NEXT:    add w0, w9, w8
-; SVE_128-NEXT:    ret
-;
-; SVE_MIN_256-LABEL: reduce_uadd_v16i8:
-; SVE_MIN_256:       // %bb.0:
-; SVE_MIN_256-NEXT:    ptrue p0.b, vl16
-; SVE_MIN_256-NEXT:    // kill: def $q0 killed $q0 def $z0
-; SVE_MIN_256-NEXT:    // kill: def $q1 killed $q1 def $z1
-; SVE_MIN_256-NEXT:    splice z0.b, p0, z0.b, z1.b
-; SVE_MIN_256-NEXT:    ptrue p0.b, vl32
-; SVE_MIN_256-NEXT:    uaddv d0, p0, z0.b
-; SVE_MIN_256-NEXT:    fmov x0, d0
-; SVE_MIN_256-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; SVE_MIN_256-NEXT:    ret
-  %1 = zext <32 x i8> %a to <32 x i32>
-  %2 = call i32 @llvm.vector.reduce.add.v16i32(<32 x i32> %1)
-  ret i32 %2
-}
-
-define i32 @reduce_sadd_v16i8(<32 x i8> %a) #0 {
-; NO_STREAMING-LABEL: reduce_sadd_v16i8:
-; NO_STREAMING:       // %bb.0:
-; NO_STREAMING-NEXT:    sshll2 v2.8h, v1.16b, #0
-; NO_STREAMING-NEXT:    sshll v1.8h, v1.8b, #0
-; NO_STREAMING-NEXT:    sshll2 v3.8h, v0.16b, #0
-; NO_STREAMING-NEXT:    sshll v0.8h, v0.8b, #0
-; NO_STREAMING-NEXT:    saddl2 v4.4s, v1.8h, v2.8h
-; NO_STREAMING-NEXT:    saddl v1.4s, v1.4h, v2.4h
-; NO_STREAMING-NEXT:    saddl2 v2.4s, v0.8h, v3.8h
-; NO_STREAMING-NEXT:    saddl v0.4s, v0.4h, v3.4h
-; NO_STREAMING-NEXT:    add v1.4s, v1.4s, v4.4s
-; NO_STREAMING-NEXT:    add v0.4s, v0.4s, v2.4s
-; NO_STREAMING-NEXT:    add v0.4s, v0.4s, v1.4s
-; NO_STREAMING-NEXT:    addv s0, v0.4s
-; NO_STREAMING-NEXT:    fmov w0, s0
-; NO_STREAMING-NEXT:    ret
-;
-; SVE_128-LABEL: reduce_sadd_v16i8:
-; SVE_128:       // %bb.0:
-; SVE_128-NEXT:    ptrue p0.b
-; SVE_128-NEXT:    // kill: def $q1 killed $q1 def $z1
-; SVE_128-NEXT:    // kill: def $q0 killed $q0 def $z0
-; SVE_128-NEXT:    saddv d1, p0, z1.b
-; SVE_128-NEXT:    saddv d0, p0, z0.b
-; SVE_128-NEXT:    fmov x8, d1
-; SVE_128-NEXT:    fmov x9, d0
-; SVE_128-NEXT:    add w0, w9, w8
-; SVE_128-NEXT:    ret
-;
-; SVE_MIN_256-LABEL: reduce_sadd_v16i8:
-; SVE_MIN_256:       // %bb.0:
-; SVE_MIN_256-NEXT:    ptrue p0.b, vl16
-; SVE_MIN_256-NEXT:    // kill: def $q0 killed $q0 def $z0
-; SVE_MIN_256-NEXT:    // kill: def $q1 killed $q1 def $z1
-; SVE_MIN_256-NEXT:    splice z0.b, p0, z0.b, z1.b
-; SVE_MIN_256-NEXT:    ptrue p0.b, vl32
-; SVE_MIN_256-NEXT:    saddv d0, p0, z0.b
-; SVE_MIN_256-NEXT:    fmov x0, d0
-; SVE_MIN_256-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; SVE_MIN_256-NEXT:    ret
-  %1 = sext <32 x i8> %a to <32 x i32>
-  %2 = call i32 @llvm.vector.reduce.add.v16i32(<32 x i32> %1)
-  ret i32 %2
-}
-
-attributes #0 = { "target-features"="+sve" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
index 27e786eb1ced1..c81fd26a77525 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -1968,9 +1968,10 @@ define i32 @test_udot_v16i8(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-SD-BASE:       // %bb.0: // %entry
 ; CHECK-SD-BASE-NEXT:    umull2 v2.8h, v1.16b, v0.16b
 ; CHECK-SD-BASE-NEXT:    umull v0.8h, v1.8b, v0.8b
-; CHECK-SD-BASE-NEXT:    uaddlp v1.4s, v2.8h
-; CHECK-SD-BASE-NEXT:    uadalp v1.4s, v0.8h
-; CHECK-SD-BASE-NEXT:    addv s0, v1.4s
+; CHECK-SD-BASE-NEXT:    uaddl2 v1.4s, v0.8h, v2.8h
+; CHECK-SD-BASE-NEXT:    uaddl v0.4s, v0.4h, v2.4h
+; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-BASE-NEXT:    addv s0, v0.4s
 ; CHECK-SD-BASE-NEXT:    fmov w0, s0
 ; CHECK-SD-BASE-NEXT:    ret
 ;
@@ -2295,9 +2296,10 @@ define i32 @test_sdot_v16i8(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-SD-BASE:       // %bb.0: // %entry
 ; CHECK-SD-BASE-NEXT:    smull2 v2.8h, v1.16b, v0.16b
 ; CHECK-SD-BASE-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-SD-BASE-NEXT:    saddlp v1.4s, v2.8h
-; CHECK-SD-BASE-NEXT:    sadalp v1.4s, v0.8h
-; CHECK-SD-BASE-NEXT:    addv s0, v1.4s
+; CHECK-SD-BASE-NEXT:    saddl2 v1.4s, v0.8h, v2.8h
+; CHECK-SD-BASE-NEXT:    saddl v0.4s, v0.4h, v2.4h
+; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-BASE-NEXT:    addv s0, v0.4s
 ; CHECK-SD-BASE-NEXT:    fmov w0, s0
 ; CHECK-SD-BASE-NEXT:    ret
 ;
@@ -3866,9 +3868,10 @@ entry:
 define i16 @add_v32i8_v32i16_zext(<32 x i8> %x) {
 ; CHECK-SD-LABEL: add_v32i8_v32i16_zext:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    uaddlp v1.8h, v1.16b
-; CHECK-SD-NEXT:    uadalp v1.8h, v0.16b
-; CHECK-SD-NEXT:    addv h0, v1.8h
+; CHECK-SD-NEXT:    uaddl2 v2.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT:    uaddl v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    add v0.8h, v0.8h, v2.8h
+; CHECK-SD-NEXT:    addv h0, v0.8h
 ; CHECK-SD-NEXT:    fmov w0, s0
 ; CHECK-SD-NEXT:    ret
 ;
@@ -3991,9 +3994,10 @@ entry:
 define i16 @add_v32i8_v32i16_sext(<32 x i8> %x) {
 ; CHECK-SD-LABEL: add_v32i8_v32i16_sext:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    saddlp v1.8h, v1.16b
-; CHECK-SD-NEXT:    sadalp v1.8h, v0.16b
-; CHECK-SD-NEXT:    addv h0, v1.8h
+; CHECK-SD-NEXT:    saddl2 v2.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT:    saddl v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    add v0.8h, v0.8h, v2.8h
+; CHECK-SD-NEXT:    addv h0, v0.8h
 ; CHECK-SD-NEXT:    fmov w0, s0
 ; CHECK-SD-NEXT:    ret
 ;
@@ -4234,14 +4238,14 @@ define i32 @add_v32i8_v32i32_zext(<32 x i8> %x) {
 ; CHECK-SD-BASE-LABEL: add_v32i8_v32i32_zext:
 ; CHECK-SD-BASE:       // %bb.0: // %entry
 ; CHECK-SD-BASE-NEXT:    ushll2 v2.8h, v1.16b, #0
-; CHECK-SD-BASE-NEXT:    ushll v1.8h, v1.8b, #0
 ; CHECK-SD-BASE-NEXT:    ushll2 v3.8h, v0.16b, #0
+; CHECK-SD-BASE-NEXT:    ushll v1.8h, v1.8b, #0
 ; CHECK-SD-BASE-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-SD-BASE-NEXT:    uaddl2 v4.4s, v1.8h, v2.8h
-; CHECK-SD-BASE-NEXT:    uaddl v1.4s, v1.4h, v2.4h
-; CHECK-SD-BASE-NEXT:    uaddl2 v2.4s, v0.8h, v3.8h
-; CHECK-SD-BASE-NEXT:    uaddl v0.4s, v0.4h, v3.4h
-; CHECK-SD-BASE-NEXT:    add v1.4s, v1.4s, v4.4s
+; CHECK-SD-BASE-NEXT:    uaddl2 v4.4s, v3.8h, v2.8h
+; CHECK-SD-BASE-NEXT:    uaddl v2.4s, v3.4h, v2.4h
+; CHECK-SD-BASE-NEXT:    uaddl2 v5.4s, v0.8h, v1.8h
+; CHECK-SD-BASE-NEXT:    uaddl v0.4s, v0.4h, v1.4h
+; CHECK-SD-BASE-NEXT:    add v1.4s, v5.4s, v4.4s
 ; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-BASE-NEXT:    addv s0, v0.4s
@@ -4507,14 +4511,14 @@ define i32 @add_v32i8_v32i32_sext(<32 x i8> %x) {
 ; CHECK-SD-BASE-LABEL: add_v32i8_v32i32_sext:
 ; CHECK-SD-BASE:       // %bb.0: // %entry
 ; CHECK-SD-BASE-NEXT:    sshll2 v2.8h, v1.16b, #0
-; CHECK-SD-BASE-NEXT:    sshll v1.8h, v1.8b, #0
 ; CHECK-SD-BASE-NEXT:    sshll2 v3.8h, v0.16b, #0
+; CHECK-SD-BASE-NEXT:    sshll v1.8h, v1.8b, #0
 ; CHECK-SD-BASE-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-SD-BASE-NEXT:    saddl2 v4.4s, v1.8h, v2.8h
-; CHECK-SD-BASE-NEXT:    saddl v1.4s, v1.4h, v2.4h
-; CHECK-SD-BASE-NEXT:    saddl2 v2.4s, v0.8h, v3.8h
-; CHECK-SD-BASE-NEXT:    saddl v0.4s, v0.4h, v3.4h
-; CHECK-SD-BASE-NEXT:    add v1.4s, v1.4s, v4.4s
+; CHECK-SD-BASE-NEXT:    saddl2 v4.4s, v3.8h, v2.8h
+; CHECK-SD-BASE-NEXT:    saddl v2.4s, v3.4h, v2.4h
+; CHECK-SD-BASE-NEXT:    saddl2 v5.4s, v0.8h, v1.8h
+; CHECK-SD-BASE-NEXT:    saddl v0.4s, v0.4h, v1.4h
+; CHECK-SD-BASE-NEXT:    add v1.4s, v5.4s, v4.4s
 ; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-SD-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-BASE-NEXT:    addv s0, v0.4s

From fa5971c298c96c965ba6d8fb855867f23518aad4 Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Mon, 22 Jul 2024 18:32:48 +0100
Subject: [PATCH 453/486] [flang][debug] Generate correct name for external
 function. (#99510)

The `ExternalNameConversion` will add an _ at the end of the external
functions. We extract the real function name to use in the debug info.
The convention is to use the real name of function in the `name` field
and mangled name with extra _ at the end in the `linkageName` field.

Fixes #92391.
---
 .../lib/Optimizer/Transforms/AddDebugInfo.cpp  |  8 ++++++--
 flang/test/Transforms/debug-92391.fir          | 18 ++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Transforms/debug-92391.fir

diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 8bb24fb6c8078..685a8645fa2fc 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -262,9 +262,13 @@ void AddDebugInfoPass::runOnOperation() {
 
     mlir::StringAttr fullName =
         mlir::StringAttr::get(context, funcOp.getName());
-    auto result = fir::NameUniquer::deconstruct(funcOp.getName());
+    mlir::Attribute attr = funcOp->getAttr(fir::getInternalFuncNameAttrName());
     mlir::StringAttr funcName =
-        mlir::StringAttr::get(context, result.second.name);
+        (attr) ? mlir::cast<mlir::StringAttr>(attr)
+               : mlir::StringAttr::get(context, funcOp.getName());
+
+    auto result = fir::NameUniquer::deconstruct(funcName);
+    funcName = mlir::StringAttr::get(context, result.second.name);
 
     llvm::SmallVector<mlir::LLVM::DITypeAttr> types;
     fir::DebugTypeGenerator typeGen(module);
diff --git a/flang/test/Transforms/debug-92391.fir b/flang/test/Transforms/debug-92391.fir
new file mode 100644
index 0000000000000..2d83be995a0f6
--- /dev/null
+++ b/flang/test/Transforms/debug-92391.fir
@@ -0,0 +1,18 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>, #dlti.dl_entry<"dlti.endianness", "little">>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"} {
+  func.func @my_square_(%arg0: !fir.ref<i32> {fir.bindc_name = "x"} ) -> i32 attributes {fir.internal_name = "_QPmy_square"} {
+    %0 = fir.undefined !fir.dscope
+    %1 = fir.alloca i32 {bindc_name = "my_square", uniq_name = "_QFmy_squareEmy_square"}
+    %2 = fircg.ext_declare %1 {uniq_name = "_QFmy_squareEmy_square"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %3 = fircg.ext_declare %arg0 dummy_scope %0 {uniq_name = "_QFmy_squareEx"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+    %4 = fir.load %3 : !fir.ref<i32>
+    %5 = arith.muli %4, %4 : i32
+    fir.store %5 to %2 : !fir.ref<i32>
+    %6 = fir.load %2 : !fir.ref<i32>
+    return %6 : i32
+  } loc(#loc1)
+}
+#loc1 = loc("92391.f90":15:1)
+
+// CHECK: #di_subprogram = #llvm.di_subprogram<{{.*}}name = "my_square", linkageName = "my_square_"{{.*}}>

From d221662ed0cc758331ea4277d46f5983ea76f36d Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 22 Jul 2024 10:00:51 -0700
Subject: [PATCH 454/486] [RISCV] In emitSelectPseudo, copy call frame size
 from LastSelectPseudo instead of MI.

The split point is LastSelectPseudo. If MI is earlier, we might
sink it to LastSelectPseudo.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 585d8a060fe94..12d0e1d6ccb48 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -18479,7 +18479,7 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
   F->insert(I, TailMBB);
 
   // Set the call frame size on entry to the new basic blocks.
-  unsigned CallFrameSize = TII.getCallFrameSizeAt(MI);
+  unsigned CallFrameSize = TII.getCallFrameSizeAt(*LastSelectPseudo);
   IfFalseMBB->setCallFrameSize(CallFrameSize);
   TailMBB->setCallFrameSize(CallFrameSize);
 

From 0950533cff55478af0eb802c3698ff5a4e94bb73 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 22 Jul 2024 10:10:06 -0700
Subject: [PATCH 455/486] [RISCV] Move call to EmitLoweredCascadedSelect above
 some variable declarations. NFC

These variables aren't used if we call EmitLoweredCascadedSelect
so move the call above them.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 12d0e1d6ccb48..823fb428472ef 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -18418,6 +18418,15 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
   // Select_FPRX_ (rs1, rs2, imm, rs4, (Select_FPRX_ rs1, rs2, imm, rs4, rs5))
   // is checked here and handled by a separate function -
   // EmitLoweredCascadedSelect.
+
+  auto Next = next_nodbg(MI.getIterator(), BB->instr_end());
+  if ((MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
+       MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
+      Next != BB->end() && Next->getOpcode() == MI.getOpcode() &&
+      Next->getOperand(5).getReg() == MI.getOperand(0).getReg() &&
+      Next->getOperand(5).isKill())
+    return EmitLoweredCascadedSelect(MI, *Next, BB, Subtarget);
+
   Register LHS = MI.getOperand(1).getReg();
   Register RHS;
   if (MI.getOperand(2).isReg())
@@ -18429,15 +18438,6 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
   SelectDests.insert(MI.getOperand(0).getReg());
 
   MachineInstr *LastSelectPseudo = &MI;
-  auto Next = next_nodbg(MI.getIterator(), BB->instr_end());
-  if ((MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
-       MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
-      Next != BB->end() && Next->getOpcode() == MI.getOpcode() &&
-      Next->getOperand(5).getReg() == MI.getOperand(0).getReg() &&
-      Next->getOperand(5).isKill()) {
-    return EmitLoweredCascadedSelect(MI, *Next, BB, Subtarget);
-  }
-
   for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
        SequenceMBBI != E; ++SequenceMBBI) {
     if (SequenceMBBI->isDebugInstr())

From bee2654300a8f524e05dd4cad41411d597246ac0 Mon Sep 17 00:00:00 2001
From: Wei Wang <apollo.mobility@gmail.com>
Date: Mon, 22 Jul 2024 10:53:40 -0700
Subject: [PATCH 456/486] [Asan] Skip pre-split coroutine and noop coroutine
 frame (#99415)

CoroSplit expects the second parameter of `llvm.coro.id` to be the
promise alloca. Applying Asan on a pre-split coroutine breaks this
assumption and causes split to fail. This should be NFC because asan
pass happens late in the pipeline where all coroutines are split. This
is to prevent crash in case the order of passes are switched.

Also `NoopCoro.Frame.Const` is a special coroutine frame that does
nothing when resumed or destroyed. There is no point to do
instrumentation on it.
---
 llvm/lib/Transforms/Coroutines/CoroEarly.cpp  |  1 +
 .../Instrumentation/AddressSanitizer.cpp      |  2 ++
 .../AddressSanitizer/skip-coro.ll             | 34 +++++++++++++++++++
 3 files changed, 37 insertions(+)
 create mode 100644 llvm/test/Instrumentation/AddressSanitizer/skip-coro.ll

diff --git a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
index 489106422e199..d8e827e9cebcf 100644
--- a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
@@ -148,6 +148,7 @@ void Lowerer::lowerCoroNoop(IntrinsicInst *II) {
     NoopCoro = new GlobalVariable(M, NoopCoroConst->getType(), /*isConstant=*/true,
                                 GlobalVariable::PrivateLinkage, NoopCoroConst,
                                 "NoopCoro.Frame.Const");
+    cast<GlobalVariable>(NoopCoro)->setNoSanitizeMetadata();
   }
 
   Builder.SetInsertPoint(II);
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 149866a8e4200..9fb1df7ab2b79 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -2947,6 +2947,8 @@ bool AddressSanitizer::instrumentFunction(Function &F,
   if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) return false;
   if (!ClDebugFunc.empty() && ClDebugFunc == F.getName()) return false;
   if (F.getName().starts_with("__asan_")) return false;
+  if (F.isPresplitCoroutine())
+    return false;
 
   bool FunctionModified = false;
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/skip-coro.ll b/llvm/test/Instrumentation/AddressSanitizer/skip-coro.ll
new file mode 100644
index 0000000000000..65b27ee2241dd
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/skip-coro.ll
@@ -0,0 +1,34 @@
+; Tests that asan skips pre-split coroutine and NoopCoro.Frame
+; RUN: opt < %s -S -passes=coro-early,asan | FileCheck %s
+
+; CHECK: %NoopCoro.Frame = type { ptr, ptr }
+; CHECK: @NoopCoro.Frame.Const = private constant %NoopCoro.Frame { ptr @__NoopCoro_ResumeDestroy, ptr @__NoopCoro_ResumeDestroy }
+; CHECK-NOT: @0 = private alias { %NoopCoro.Frame,
+
+%struct.Promise = type { %"struct.std::__n4861::coroutine_handle" }
+%"struct.std::__n4861::coroutine_handle" = type { ptr }
+
+; CHECK-LABEL: @foo(
+define ptr @foo() #0 {
+; CHECK-NEXT: entry
+; CHECK-NOT: %asan_local_stack_base
+entry:
+  %__promise = alloca %struct.Promise, align 8
+  %0 = call token @llvm.coro.id(i32 16, ptr nonnull %__promise, ptr null, ptr null)
+  %1 = call ptr @llvm.coro.noop()
+  ret ptr %1
+}
+
+declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr)
+declare ptr @llvm.coro.noop()
+
+attributes #0 = { sanitize_address presplitcoroutine }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "hand-written", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}

From 146fd7cd454d71afb94d5d7ad7a3dbd6f3344d04 Mon Sep 17 00:00:00 2001
From: Daniil Kovalev <dkovalev@accesssoftek.com>
Date: Mon, 22 Jul 2024 21:18:39 +0300
Subject: [PATCH 457/486] [PAC][Driver] Support `pauthtest` ABI for AArch64
 Linux triples (#97237)

When `pauthtest` is either passed as environment part of AArch64 Linux
triple
or passed via `-mabi=`, enable the following ptrauth flags:

- `intrinsics`;
- `calls`;
- `returns`;
- `auth-traps`;
- `vtable-pointer-address-discrimination`;
- `vtable-pointer-type-discrimination`;
- `init-fini`.

Some related stuff is still subject to change, and the ABI itself might
be changed, so end users are not expected to use this and the ABI name
has 'test' suffix.

If `-mabi=pauthtest` option is used, it's normalized to effective
triple.

When the environment part of the effective triple is `pauthtest`, try
to use `aarch64-linux-pauthtest` as multilib directory.

The following is not supported:
- combination of `pauthtest` ABI with any branch protection scheme
except BTI;
- explicit set of environment part of the triple to a value different
  from `pauthtest` in combination with `-mabi=pauthtest`;
- usage on non-Linux OS.

---------

Co-authored-by: Anatoly Trosinenko <atrosinenko@accesssoftek.com>
---
 clang/lib/Basic/Targets/AArch64.cpp           |  9 +-
 clang/lib/CodeGen/CodeGenModule.cpp           |  2 +
 clang/lib/CodeGen/TargetInfo.h                |  1 +
 clang/lib/Driver/ToolChain.cpp                |  5 +-
 clang/lib/Driver/ToolChains/Arch/AArch64.cpp  | 21 +++++
 clang/lib/Driver/ToolChains/Arch/AArch64.h    |  3 +
 clang/lib/Driver/ToolChains/Clang.cpp         | 60 ++++++++++++-
 clang/lib/Driver/ToolChains/Linux.cpp         |  3 +
 .../usr/include/aarch64-linux-gnu/.keep       |  0
 .../usr/include/aarch64-linux-pauthtest/.keep |  0
 clang/test/Driver/aarch64-multilib-pauthabi.c |  4 +
 clang/test/Driver/aarch64-ptrauth.c           | 85 +++++++++++++++++--
 llvm/include/llvm/TargetParser/Triple.h       |  6 +-
 llvm/lib/TargetParser/Triple.cpp              |  3 +
 llvm/unittests/TargetParser/TripleTest.cpp    |  6 ++
 15 files changed, 191 insertions(+), 17 deletions(-)
 create mode 100644 clang/test/Driver/Inputs/multilib_aarch64_linux_tree/usr/include/aarch64-linux-gnu/.keep
 create mode 100644 clang/test/Driver/Inputs/multilib_aarch64_linux_tree/usr/include/aarch64-linux-pauthtest/.keep
 create mode 100644 clang/test/Driver/aarch64-multilib-pauthabi.c

diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 6349fcf3dadd7..6ba31cc05a0d7 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -204,7 +204,8 @@ AArch64TargetInfo::AArch64TargetInfo(const llvm::Triple &Triple,
 StringRef AArch64TargetInfo::getABI() const { return ABI; }
 
 bool AArch64TargetInfo::setABI(const std::string &Name) {
-  if (Name != "aapcs" && Name != "aapcs-soft" && Name != "darwinpcs")
+  if (Name != "aapcs" && Name != "aapcs-soft" && Name != "darwinpcs" &&
+      Name != "pauthtest")
     return false;
 
   ABI = Name;
@@ -218,6 +219,12 @@ bool AArch64TargetInfo::validateTarget(DiagnosticsEngine &Diags) const {
     Diags.Report(diag::err_target_unsupported_abi_with_fpu) << ABI;
     return false;
   }
+  if (getTriple().getEnvironment() == llvm::Triple::PAuthTest &&
+      getTriple().getOS() != llvm::Triple::Linux) {
+    Diags.Report(diag::err_target_unsupported_abi_for_triple)
+        << getTriple().getEnvironmentName() << getTriple().getTriple();
+    return false;
+  }
   return true;
 }
 
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 71192cb0e8c4a..63ed5b4dd0c31 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -149,6 +149,8 @@ createTargetCodeGenInfo(CodeGenModule &CGM) {
       return createWindowsAArch64TargetCodeGenInfo(CGM, AArch64ABIKind::Win64);
     else if (Target.getABI() == "aapcs-soft")
       Kind = AArch64ABIKind::AAPCSSoft;
+    else if (Target.getABI() == "pauthtest")
+      Kind = AArch64ABIKind::PAuthTest;
 
     return createAArch64TargetCodeGenInfo(CGM, Kind);
   }
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index 0925609cc74aa..2f2138582ba1e 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -437,6 +437,7 @@ enum class AArch64ABIKind {
   DarwinPCS,
   Win64,
   AAPCSSoft,
+  PAuthTest,
 };
 
 std::unique_ptr<TargetCodeGenInfo>
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 85ae4d2a26fee..20a555afb8092 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1031,11 +1031,12 @@ std::string ToolChain::ComputeLLVMTriple(const ArgList &Args,
   }
   case llvm::Triple::aarch64: {
     llvm::Triple Triple = getTriple();
+    tools::aarch64::setPAuthABIInTriple(getDriver(), Args, Triple);
     if (!Triple.isOSBinFormatMachO())
-      return getTripleString();
+      return Triple.getTriple();
 
     if (Triple.isArm64e())
-      return getTripleString();
+      return Triple.getTriple();
 
     // FIXME: older versions of ld64 expect the "arm64" component in the actual
     // triple string and query it to determine whether an LTO file can be
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index 5fbf38cdda12b..f083e40df1314 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -449,3 +449,24 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
   if (Args.getLastArg(options::OPT_mno_bti_at_return_twice))
     Features.push_back("+no-bti-at-return-twice");
 }
+
+void aarch64::setPAuthABIInTriple(const Driver &D, const ArgList &Args,
+                                  llvm::Triple &Triple) {
+  Arg *ABIArg = Args.getLastArg(options::OPT_mabi_EQ);
+  bool HasPAuthABI =
+      ABIArg ? (StringRef(ABIArg->getValue()) == "pauthtest") : false;
+
+  switch (Triple.getEnvironment()) {
+  case llvm::Triple::UnknownEnvironment:
+    if (HasPAuthABI)
+      Triple.setEnvironment(llvm::Triple::PAuthTest);
+    break;
+  case llvm::Triple::PAuthTest:
+    break;
+  default:
+    if (HasPAuthABI)
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+          << ABIArg->getAsString(Args) << Triple.getTriple();
+    break;
+  }
+}
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.h b/clang/lib/Driver/ToolChains/Arch/AArch64.h
index d47c402d4a42d..6d071167bd392 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.h
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.h
@@ -28,6 +28,9 @@ void getAArch64TargetFeatures(const Driver &D, const llvm::Triple &Triple,
 std::string getAArch64TargetCPU(const llvm::opt::ArgList &Args,
                                 const llvm::Triple &Triple, llvm::opt::Arg *&A);
 
+void setPAuthABIInTriple(const Driver &D, const llvm::opt::ArgList &Args,
+                         llvm::Triple &triple);
+
 } // end namespace aarch64
 } // end namespace target
 } // end namespace driver
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 601f7a355297d..f240a47504ef6 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1486,6 +1486,41 @@ void AddUnalignedAccessWarning(ArgStringList &CmdArgs) {
 }
 }
 
+// Each combination of options here forms a signing schema, and in most cases
+// each signing schema is its own incompatible ABI. The default values of the
+// options represent the default signing schema.
+static void handlePAuthABI(const ArgList &DriverArgs, ArgStringList &CC1Args) {
+  if (!DriverArgs.hasArg(options::OPT_fptrauth_intrinsics,
+                         options::OPT_fno_ptrauth_intrinsics))
+    CC1Args.push_back("-fptrauth-intrinsics");
+
+  if (!DriverArgs.hasArg(options::OPT_fptrauth_calls,
+                         options::OPT_fno_ptrauth_calls))
+    CC1Args.push_back("-fptrauth-calls");
+
+  if (!DriverArgs.hasArg(options::OPT_fptrauth_returns,
+                         options::OPT_fno_ptrauth_returns))
+    CC1Args.push_back("-fptrauth-returns");
+
+  if (!DriverArgs.hasArg(options::OPT_fptrauth_auth_traps,
+                         options::OPT_fno_ptrauth_auth_traps))
+    CC1Args.push_back("-fptrauth-auth-traps");
+
+  if (!DriverArgs.hasArg(
+          options::OPT_fptrauth_vtable_pointer_address_discrimination,
+          options::OPT_fno_ptrauth_vtable_pointer_address_discrimination))
+    CC1Args.push_back("-fptrauth-vtable-pointer-address-discrimination");
+
+  if (!DriverArgs.hasArg(
+          options::OPT_fptrauth_vtable_pointer_type_discrimination,
+          options::OPT_fno_ptrauth_vtable_pointer_type_discrimination))
+    CC1Args.push_back("-fptrauth-vtable-pointer-type-discrimination");
+
+  if (!DriverArgs.hasArg(options::OPT_fptrauth_init_fini,
+                         options::OPT_fno_ptrauth_init_fini))
+    CC1Args.push_back("-fptrauth-init-fini");
+}
+
 static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
                                     ArgStringList &CmdArgs, bool isAArch64) {
   const Arg *A = isAArch64
@@ -1548,16 +1583,30 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
 
   CmdArgs.push_back(
       Args.MakeArgString(Twine("-msign-return-address=") + Scope));
-  if (Scope != "none")
+  if (Scope != "none") {
+    if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+          << A->getAsString(Args) << Triple.getTriple();
     CmdArgs.push_back(
         Args.MakeArgString(Twine("-msign-return-address-key=") + Key));
-  if (BranchProtectionPAuthLR)
+  }
+  if (BranchProtectionPAuthLR) {
+    if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+          << A->getAsString(Args) << Triple.getTriple();
     CmdArgs.push_back(
         Args.MakeArgString(Twine("-mbranch-protection-pauth-lr")));
+  }
   if (IndirectBranches)
     CmdArgs.push_back("-mbranch-target-enforce");
-  if (GuardedControlStack)
+  // GCS is currently untested with PAuthABI, but enabling this could be allowed
+  // in future after testing with a suitable system.
+  if (GuardedControlStack) {
+    if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+          << A->getAsString(Args) << Triple.getTriple();
     CmdArgs.push_back("-mguarded-control-stack");
+  }
 }
 
 void Clang::AddARMTargetArgs(const llvm::Triple &Triple, const ArgList &Args,
@@ -1701,6 +1750,8 @@ void RenderAArch64ABI(const llvm::Triple &Triple, const ArgList &Args,
     ABIName = A->getValue();
   else if (Triple.isOSDarwin())
     ABIName = "darwinpcs";
+  else if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
+    ABIName = "pauthtest";
   else
     ABIName = "aapcs";
 
@@ -1737,6 +1788,9 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
   // Enable/disable return address signing and indirect branch targets.
   CollectARMPACBTIOptions(getToolChain(), Args, CmdArgs, true /*isAArch64*/);
 
+  if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
+    handlePAuthABI(Args, CmdArgs);
+
   // Handle -msve_vector_bits=<bits>
   if (Arg *A = Args.getLastArg(options::OPT_msve_vector_bits_EQ)) {
     StringRef Val = A->getValue();
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 98a878e1d764d..2265138edbffb 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -86,6 +86,9 @@ std::string Linux::getMultiarchTriple(const Driver &D,
   case llvm::Triple::aarch64:
     if (IsAndroid)
       return "aarch64-linux-android";
+    if (hasEffectiveTriple() &&
+        getEffectiveTriple().getEnvironment() == llvm::Triple::PAuthTest)
+      return "aarch64-linux-pauthtest";
     return "aarch64-linux-gnu";
   case llvm::Triple::aarch64_be:
     return "aarch64_be-linux-gnu";
diff --git a/clang/test/Driver/Inputs/multilib_aarch64_linux_tree/usr/include/aarch64-linux-gnu/.keep b/clang/test/Driver/Inputs/multilib_aarch64_linux_tree/usr/include/aarch64-linux-gnu/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/multilib_aarch64_linux_tree/usr/include/aarch64-linux-pauthtest/.keep b/clang/test/Driver/Inputs/multilib_aarch64_linux_tree/usr/include/aarch64-linux-pauthtest/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/aarch64-multilib-pauthabi.c b/clang/test/Driver/aarch64-multilib-pauthabi.c
new file mode 100644
index 0000000000000..3046cb856e83c
--- /dev/null
+++ b/clang/test/Driver/aarch64-multilib-pauthabi.c
@@ -0,0 +1,4 @@
+// RUN: %clang --target=aarch64-linux-pauthtest       --sysroot=%S/Inputs/multilib_aarch64_linux_tree -### -c %s 2>&1 | FileCheck %s
+// RUN: %clang --target=aarch64-linux -mabi=pauthtest --sysroot=%S/Inputs/multilib_aarch64_linux_tree -### -c %s 2>&1 | FileCheck %s
+
+// CHECK: "-internal-externc-isystem" "{{.*}}/usr/include/aarch64-linux-pauthtest"
diff --git a/clang/test/Driver/aarch64-ptrauth.c b/clang/test/Driver/aarch64-ptrauth.c
index fa0125f4b22a9..d13930e8f4b37 100644
--- a/clang/test/Driver/aarch64-ptrauth.c
+++ b/clang/test/Driver/aarch64-ptrauth.c
@@ -1,5 +1,7 @@
+// REQUIRES: aarch64-registered-target
+
 // RUN: %clang -### -c --target=aarch64 %s 2>&1 | FileCheck %s --check-prefix NONE
-// NONE: "-cc1"
+// NONE:     "-cc1"
 // NONE-NOT: "-fptrauth-
 
 // RUN: %clang -### -c --target=aarch64 \
@@ -13,13 +15,78 @@
 // RUN:   %s 2>&1 | FileCheck %s --check-prefix=ALL
 // ALL: "-cc1"{{.*}} "-fptrauth-intrinsics" "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-init-fini"
 
+// RUN: %clang -### -c --target=aarch64-linux -mabi=pauthtest %s 2>&1 | FileCheck %s --check-prefix=PAUTHABI1
+// RUN: %clang -### -c --target=aarch64-linux-pauthtest %s 2>&1 | FileCheck %s --check-prefix=PAUTHABI1
+// PAUTHABI1:      "-cc1"{{.*}} "-triple" "aarch64-unknown-linux-pauthtest"
+// PAUTHABI1-SAME: "-target-abi" "pauthtest"
+// PAUTHABI1-SAME: "-fptrauth-intrinsics" "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-init-fini"
+
+// RUN: %clang -### -c --target=aarch64 -mabi=pauthtest -fno-ptrauth-intrinsics \
+// RUN:   -fno-ptrauth-calls -fno-ptrauth-returns -fno-ptrauth-auth-traps \
+// RUN:   -fno-ptrauth-vtable-pointer-address-discrimination -fno-ptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fno-ptrauth-init-fini %s 2>&1 | FileCheck %s --check-prefix=PAUTHABI2
+// RUN: %clang -### -c --target=aarch64-pauthtest -fno-ptrauth-intrinsics \
+// RUN:   -fno-ptrauth-calls -fno-ptrauth-returns -fno-ptrauth-auth-traps \
+// RUN:   -fno-ptrauth-vtable-pointer-address-discrimination -fno-ptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fno-ptrauth-init-fini %s 2>&1 | FileCheck %s --check-prefix=PAUTHABI2
+// PAUTHABI2:     "-cc1"
+// PAUTHABI2-NOT: "-fptrauth-
+
 // RUN: not %clang -### -c --target=x86_64 -fptrauth-intrinsics -fptrauth-calls -fptrauth-returns -fptrauth-auth-traps \
 // RUN:   -fptrauth-vtable-pointer-address-discrimination -fptrauth-vtable-pointer-type-discrimination \
-// RUN:   -fptrauth-init-fini %s 2>&1 | FileCheck %s --check-prefix=ERR
-// ERR:      error: unsupported option '-fptrauth-intrinsics' for target '{{.*}}'
-// ERR-NEXT: error: unsupported option '-fptrauth-calls' for target '{{.*}}'
-// ERR-NEXT: error: unsupported option '-fptrauth-returns' for target '{{.*}}'
-// ERR-NEXT: error: unsupported option '-fptrauth-auth-traps' for target '{{.*}}'
-// ERR-NEXT: error: unsupported option '-fptrauth-vtable-pointer-address-discrimination' for target '{{.*}}'
-// ERR-NEXT: error: unsupported option '-fptrauth-vtable-pointer-type-discrimination' for target '{{.*}}'
-// ERR-NEXT: error: unsupported option '-fptrauth-init-fini' for target '{{.*}}'
+// RUN:   -fptrauth-init-fini %s 2>&1 | FileCheck %s --check-prefix=ERR1
+// ERR1:      error: unsupported option '-fptrauth-intrinsics' for target '{{.*}}'
+// ERR1-NEXT: error: unsupported option '-fptrauth-calls' for target '{{.*}}'
+// ERR1-NEXT: error: unsupported option '-fptrauth-returns' for target '{{.*}}'
+// ERR1-NEXT: error: unsupported option '-fptrauth-auth-traps' for target '{{.*}}'
+// ERR1-NEXT: error: unsupported option '-fptrauth-vtable-pointer-address-discrimination' for target '{{.*}}'
+// ERR1-NEXT: error: unsupported option '-fptrauth-vtable-pointer-type-discrimination' for target '{{.*}}'
+// ERR1-NEXT: error: unsupported option '-fptrauth-init-fini' for target '{{.*}}'
+
+//// Only support PAuth ABI for Linux as for now.
+// RUN: not %clang -c --target=aarch64-unknown -mabi=pauthtest %s 2>&1 | FileCheck %s --check-prefix=ERR2
+// RUN: not %clang -c --target=aarch64-unknown-pauthtest       %s 2>&1 | FileCheck %s --check-prefix=ERR2
+// ERR2: error: ABI 'pauthtest' is not supported for 'aarch64-unknown-unknown-pauthtest'
+
+//// PAuth ABI is encoded as environment part of the triple, so don't allow to explicitly set other environments.
+// RUN: not %clang -c --target=aarch64-linux-gnu -mabi=pauthtest %s 2>&1 | FileCheck %s --check-prefix=ERR3
+// ERR3: error: unsupported option '-mabi=pauthtest' for target 'aarch64-unknown-linux-gnu'
+// RUN: %clang -c --target=aarch64-linux-pauthtest -mabi=pauthtest %s
+
+//// The only branch protection option compatible with PAuthABI is BTI.
+// RUN: not %clang -### -c --target=aarch64-linux -mabi=pauthtest -mbranch-protection=pac-ret %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR4
+// RUN: not %clang -### -c --target=aarch64-linux-pauthtest       -mbranch-protection=pac-ret %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR4
+// ERR4: error: unsupported option '-mbranch-protection=pac-ret' for target 'aarch64-unknown-linux-pauthtest'
+
+// RUN: not %clang -### -c --target=aarch64-linux -mabi=pauthtest -mbranch-protection=gcs %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR5
+// RUN: not %clang -### -c --target=aarch64-linux-pauthtest       -mbranch-protection=gcs %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR5
+// ERR5: error: unsupported option '-mbranch-protection=gcs' for target 'aarch64-unknown-linux-pauthtest'
+
+// RUN: not %clang -### -c --target=aarch64-linux -mabi=pauthtest -mbranch-protection=standard %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR6
+// RUN: not %clang -### -c --target=aarch64-linux-pauthtest       -mbranch-protection=standard %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR6
+// ERR6: error: unsupported option '-mbranch-protection=standard' for target 'aarch64-unknown-linux-pauthtest'
+
+// RUN: not %clang -### -c --target=aarch64-linux -mabi=pauthtest -msign-return-address=all %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR7
+// RUN: not %clang -### -c --target=aarch64-linux-pauthtest       -msign-return-address=all %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR7
+// ERR7: error: unsupported option '-msign-return-address=all' for target 'aarch64-unknown-linux-pauthtest'
+
+// RUN: not %clang -### -c --target=aarch64-linux -mabi=pauthtest -msign-return-address=non-leaf %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR8
+// RUN: not %clang -### -c --target=aarch64-linux-pauthtest       -msign-return-address=non-leaf %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR8
+// ERR8: error: unsupported option '-msign-return-address=non-leaf' for target 'aarch64-unknown-linux-pauthtest'
+
+// RUN: %clang -### -c --target=aarch64-linux -mabi=pauthtest -msign-return-address=none %s
+// RUN: %clang -### -c --target=aarch64-linux-pauthtest       -msign-return-address=none %s
+// RUN: %clang -### -c --target=aarch64-linux -mabi=pauthtest -mbranch-protection=bti %s
+// RUN: %clang -### -c --target=aarch64-linux-pauthtest       -mbranch-protection=bti %s
+// RUN: %clang -### -c --target=aarch64-linux -mabi=pauthtest -mbranch-protection=none %s
+// RUN: %clang -### -c --target=aarch64-linux-pauthtest       -mbranch-protection=none %s
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index b3bb354b38ff5..ebd92f264d904 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -268,7 +268,7 @@ class Triple {
     Cygnus,
     CoreCLR,
     Simulator, // Simulator variants of other systems, e.g., Apple's iOS
-    MacABI, // Mac Catalyst variant of Apple's iOS deployment target.
+    MacABI,    // Mac Catalyst variant of Apple's iOS deployment target.
 
     // Shader Stages
     // The order of these values matters, and must be kept in sync with the
@@ -292,7 +292,9 @@ class Triple {
     OpenCL,
     OpenHOS,
 
-    LastEnvironmentType = OpenHOS
+    PAuthTest,
+
+    LastEnvironmentType = PAuthTest
   };
   enum ObjectFormatType {
     UnknownObjectFormat,
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index 4fc1ff5aaa051..bf89aace65e58 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -352,6 +352,8 @@ StringRef Triple::getEnvironmentTypeName(EnvironmentType Kind) {
   case OpenCL:
     return "opencl";
   case OpenHOS: return "ohos";
+  case PAuthTest:
+    return "pauthtest";
   }
 
   llvm_unreachable("Invalid EnvironmentType!");
@@ -728,6 +730,7 @@ static Triple::EnvironmentType parseEnvironment(StringRef EnvironmentName) {
       .StartsWith("amplification", Triple::Amplification)
       .StartsWith("opencl", Triple::OpenCL)
       .StartsWith("ohos", Triple::OpenHOS)
+      .StartsWith("pauthtest", Triple::PAuthTest)
       .Default(Triple::UnknownEnvironment);
 }
 
diff --git a/llvm/unittests/TargetParser/TripleTest.cpp b/llvm/unittests/TargetParser/TripleTest.cpp
index f93dc3671197a..0aecfc64da208 100644
--- a/llvm/unittests/TargetParser/TripleTest.cpp
+++ b/llvm/unittests/TargetParser/TripleTest.cpp
@@ -1169,6 +1169,12 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::Serenity, T.getOS());
   EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
 
+  T = Triple("aarch64-unknown-linux-pauthtest");
+  EXPECT_EQ(Triple::aarch64, T.getArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::PAuthTest, T.getEnvironment());
+
   T = Triple("huh");
   EXPECT_EQ(Triple::UnknownArch, T.getArch());
 }

From 511e93b96ee74438cbd643cec63281aff0663933 Mon Sep 17 00:00:00 2001
From: James Y Knight <jyknight@google.com>
Date: Mon, 22 Jul 2024 14:20:25 -0400
Subject: [PATCH 458/486] Handle constant "pointers" for
 `__atomic_always_lock_free`/`__atomic_is_lock_free`. (#99340)

The second argument passed to these builtins is used to validate whether
the object's alignment is sufficient for atomic operations of the given
size.

Currently, the builtins can be folded at compile time only when the
argument is 0/nullptr, or if the _type_ of the pointer guarantees
appropriate alignment.

This change allows the compiler to also evaluate non-null constant
pointers, which enables callers to check a specified alignment, instead
of only the type or an exact object. E.g.:
 `__atomic_is_lock_free(sizeof(T), (void*)4)`
can be potentially evaluated to true at compile time, instead of
generating a libcall. This is also supported by GCC, and used by
libstdc++, and is also useful for libc++'s atomic_ref.

Also helps with (but doesn't fix) issue #75081.

This also fixes a crash bug, when the second argument was a non-pointer
implicitly convertible to a pointer (such as an array, or a function).
---
 clang/docs/ReleaseNotes.rst    |  7 +++++++
 clang/lib/AST/ExprConstant.cpp | 36 ++++++++++++++++++++++++----------
 clang/test/Sema/atomic-ops.c   | 25 +++++++++++++++++++++++
 3 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 8e24087b3dcdb..9a07ad42b8f6f 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -440,6 +440,13 @@ Non-comprehensive list of changes in this release
   pointers, enabling more powerful alias analysis when accessing pointer types.
   The new behavior can be enabled using ``-fpointer-tbaa``.
 
+- The ``__atomic_always_lock_free`` and ``__atomic_is_lock_free``
+  builtins may now return true if the pointer argument is a
+  compile-time constant (e.g. ``(void*)4``), and constant pointer is
+  sufficiently-aligned for the access requested. Previously, only the
+  type of the pointer was taken into account. This improves
+  compatibility with GCC's libstdc++.
+
 New Compiler Flags
 ------------------
 - ``-fsanitize=implicit-bitfield-conversion`` checks implicit truncation and
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 5af712dd7257b..fcb382474ea62 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -12949,19 +12949,35 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
           Info.Ctx.getTargetInfo().getMaxAtomicInlineWidth();
       if (Size <= Info.Ctx.toCharUnitsFromBits(InlineWidthBits)) {
         if (BuiltinOp == Builtin::BI__c11_atomic_is_lock_free ||
-            Size == CharUnits::One() ||
-            E->getArg(1)->isNullPointerConstant(Info.Ctx,
-                                                Expr::NPC_NeverValueDependent))
-          // OK, we will inline appropriately-aligned operations of this size,
-          // and _Atomic(T) is appropriately-aligned.
+            Size == CharUnits::One())
           return Success(1, E);
 
-        QualType PointeeType = E->getArg(1)->IgnoreImpCasts()->getType()->
-          castAs<PointerType>()->getPointeeType();
-        if (!PointeeType->isIncompleteType() &&
-            Info.Ctx.getTypeAlignInChars(PointeeType) >= Size) {
-          // OK, we will inline operations on this object.
+        // If the pointer argument can be evaluated to a compile-time constant
+        // integer (or nullptr), check if that value is appropriately aligned.
+        const Expr *PtrArg = E->getArg(1);
+        Expr::EvalResult ExprResult;
+        APSInt IntResult;
+        if (PtrArg->EvaluateAsRValue(ExprResult, Info.Ctx) &&
+            ExprResult.Val.toIntegralConstant(IntResult, PtrArg->getType(),
+                                              Info.Ctx) &&
+            IntResult.isAligned(Size.getAsAlign()))
           return Success(1, E);
+
+        // Otherwise, check if the type's alignment against Size.
+        if (auto *ICE = dyn_cast<ImplicitCastExpr>(PtrArg)) {
+          // Drop the potential implicit-cast to 'const volatile void*', getting
+          // the underlying type.
+          if (ICE->getCastKind() == CK_BitCast)
+            PtrArg = ICE->getSubExpr();
+        }
+
+        if (auto PtrTy = PtrArg->getType()->getAs<PointerType>()) {
+          QualType PointeeType = PtrTy->getPointeeType();
+          if (!PointeeType->isIncompleteType() &&
+              Info.Ctx.getTypeAlignInChars(PointeeType) >= Size) {
+            // OK, we will inline operations on this object.
+            return Success(1, E);
+          }
         }
       }
     }
diff --git a/clang/test/Sema/atomic-ops.c b/clang/test/Sema/atomic-ops.c
index 9b82d82ff8269..2405f804d0da5 100644
--- a/clang/test/Sema/atomic-ops.c
+++ b/clang/test/Sema/atomic-ops.c
@@ -124,6 +124,31 @@ _Static_assert(__atomic_always_lock_free(4, &i64), "");
 _Static_assert(!__atomic_always_lock_free(8, &i32), "");
 _Static_assert(__atomic_always_lock_free(8, &i64), "");
 
+// Validate use with fake pointers constants. This mechanism is used to allow
+// validating atomicity of a given size and alignment.
+_Static_assert(__atomic_is_lock_free(1, (void*)1), "");
+_Static_assert(__atomic_is_lock_free(1, (void*)-1), "");
+_Static_assert(__atomic_is_lock_free(4, (void*)2), ""); // expected-error {{not an integral constant expression}}
+_Static_assert(__atomic_is_lock_free(4, (void*)-2), ""); // expected-error {{not an integral constant expression}}
+_Static_assert(__atomic_is_lock_free(4, (void*)4), "");
+_Static_assert(__atomic_is_lock_free(4, (void*)-4), "");
+
+_Static_assert(__atomic_always_lock_free(1, (void*)1), "");
+_Static_assert(__atomic_always_lock_free(1, (void*)-1), "");
+_Static_assert(!__atomic_always_lock_free(4, (void*)2), "");
+_Static_assert(!__atomic_always_lock_free(4, (void*)-2), "");
+_Static_assert(__atomic_always_lock_free(4, (void*)4), "");
+_Static_assert(__atomic_always_lock_free(4, (void*)-4), "");
+
+// Ensure that "weird" constants don't cause trouble.
+_Static_assert(__atomic_always_lock_free(1, "string"), "");
+_Static_assert(!__atomic_always_lock_free(2, "string"), "");
+_Static_assert(__atomic_always_lock_free(2, (int[2]){}), "");
+void dummyfn();
+_Static_assert(__atomic_always_lock_free(2, dummyfn) || 1, "");
+
+
+
 #define _AS1 __attribute__((address_space(1)))
 #define _AS2 __attribute__((address_space(2)))
 

From 315561c867784ebd9ca387e94ea6597918e7cc1c Mon Sep 17 00:00:00 2001
From: Nicolas van Kempen <nvankemp@gmail.com>
Date: Mon, 22 Jul 2024 14:23:49 -0400
Subject: [PATCH 459/486] [run-clang-tidy.py] Refactor, add progress indicator,
 add type hints (#89490)

[There is
work](https://discourse.llvm.org/t/rfc-upgrading-llvms-minimum-required-python-version/67571)
to make Python 3.8 the minimum Python version for LLVM.

I edited this script because I wanted some indicator of progress while
going through files.
It now outputs `[XX/YYY]` with the number of processed and total files
after each completion.

The current version of this script is compatible downto Python 3.6 (this
is PyYAML's minimum version).
It would probably work with older Python 3 versions with an older PyYAML
or when YAML is disabled.

With the updates here, it is compatible downto Python 3.7. Python 3.7
was released June 2018.

https://github.com/llvm/llvm-project/pull/89302 is also touching this
file, I don't mind rebasing on top of that work if needed.

### Summary

 -  Add type annotations.
 -  Replace `threading` + `queue` with `asyncio`.
- **Add indicator of processed files over total files**. This is what I
set out to do initially.
- Only print the filename after completion, not the entire Clang-Tidy
invocation command. I find this neater but the behavior can easily be
restored.
---
 .../clang-tidy/tool/run-clang-tidy.py         | 323 ++++++++++--------
 clang-tools-extra/docs/ReleaseNotes.rst       |   2 +
 2 files changed, 185 insertions(+), 140 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
index 0dc35ad587362..48401ba5ea42a 100755
--- a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
+++ b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
@@ -34,29 +34,32 @@
 http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
 """
 
-from __future__ import print_function
-
 import argparse
+import asyncio
+from dataclasses import dataclass
 import glob
 import json
 import multiprocessing
 import os
-import queue
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
-import threading
+import time
 import traceback
+from types import ModuleType
+from typing import Any, Awaitable, Callable, List, Optional, Tuple, TypeVar
+
 
+yaml: Optional[ModuleType] = None
 try:
     import yaml
 except ImportError:
     yaml = None
 
 
-def strtobool(val):
+def strtobool(val: str) -> bool:
     """Convert a string representation of truth to a bool following LLVM's CLI argument parsing."""
 
     val = val.lower()
@@ -67,11 +70,11 @@ def strtobool(val):
 
     # Return ArgumentTypeError so that argparse does not substitute its own error message
     raise argparse.ArgumentTypeError(
-        "'{}' is invalid value for boolean argument! Try 0 or 1.".format(val)
+        f"'{val}' is invalid value for boolean argument! Try 0 or 1."
     )
 
 
-def find_compilation_database(path):
+def find_compilation_database(path: str) -> str:
     """Adjusts the directory until a compilation database is found."""
     result = os.path.realpath("./")
     while not os.path.isfile(os.path.join(result, path)):
@@ -83,49 +86,43 @@ def find_compilation_database(path):
     return result
 
 
-def make_absolute(f, directory):
-    if os.path.isabs(f):
-        return f
-    return os.path.normpath(os.path.join(directory, f))
-
-
 def get_tidy_invocation(
-    f,
-    clang_tidy_binary,
-    checks,
-    tmpdir,
-    build_path,
-    header_filter,
-    allow_enabling_alpha_checkers,
-    extra_arg,
-    extra_arg_before,
-    quiet,
-    config_file_path,
-    config,
-    line_filter,
-    use_color,
-    plugins,
-    warnings_as_errors,
-    exclude_header_filter,
-    allow_no_checks,
-):
+    f: str,
+    clang_tidy_binary: str,
+    checks: str,
+    tmpdir: Optional[str],
+    build_path: str,
+    header_filter: Optional[str],
+    allow_enabling_alpha_checkers: bool,
+    extra_arg: List[str],
+    extra_arg_before: List[str],
+    quiet: bool,
+    config_file_path: str,
+    config: str,
+    line_filter: Optional[str],
+    use_color: bool,
+    plugins: List[str],
+    warnings_as_errors: Optional[str],
+    exclude_header_filter: Optional[str],
+    allow_no_checks: bool,
+) -> List[str]:
     """Gets a command line for clang-tidy."""
     start = [clang_tidy_binary]
     if allow_enabling_alpha_checkers:
         start.append("-allow-enabling-analyzer-alpha-checkers")
     if exclude_header_filter is not None:
-        start.append("--exclude-header-filter=" + exclude_header_filter)
+        start.append(f"--exclude-header-filter={exclude_header_filter}")
     if header_filter is not None:
-        start.append("-header-filter=" + header_filter)
+        start.append(f"-header-filter={header_filter}")
     if line_filter is not None:
-        start.append("-line-filter=" + line_filter)
+        start.append(f"-line-filter={line_filter}")
     if use_color is not None:
         if use_color:
             start.append("--use-color")
         else:
             start.append("--use-color=false")
     if checks:
-        start.append("-checks=" + checks)
+        start.append(f"-checks={checks}")
     if tmpdir is not None:
         start.append("-export-fixes")
         # Get a temporary file. We immediately close the handle so clang-tidy can
@@ -134,28 +131,29 @@ def get_tidy_invocation(
         os.close(handle)
         start.append(name)
     for arg in extra_arg:
-        start.append("-extra-arg=%s" % arg)
+        start.append(f"-extra-arg={arg}")
     for arg in extra_arg_before:
-        start.append("-extra-arg-before=%s" % arg)
-    start.append("-p=" + build_path)
+        start.append(f"-extra-arg-before={arg}")
+    start.append(f"-p={build_path}")
     if quiet:
         start.append("-quiet")
     if config_file_path:
-        start.append("--config-file=" + config_file_path)
+        start.append(f"--config-file={config_file_path}")
     elif config:
-        start.append("-config=" + config)
+        start.append(f"-config={config}")
     for plugin in plugins:
-        start.append("-load=" + plugin)
+        start.append(f"-load={plugin}")
     if warnings_as_errors:
-        start.append("--warnings-as-errors=" + warnings_as_errors)
+        start.append(f"--warnings-as-errors={warnings_as_errors}")
     if allow_no_checks:
         start.append("--allow-no-checks")
     start.append(f)
     return start
 
 
-def merge_replacement_files(tmpdir, mergefile):
+def merge_replacement_files(tmpdir: str, mergefile: str) -> None:
     """Merge all replacement files in a directory into a single file"""
+    assert yaml
     # The fixes suggested by clang-tidy >= 4.0.0 are given under
     # the top level key 'Diagnostics' in the output yaml files
     mergekey = "Diagnostics"
@@ -179,16 +177,14 @@ def merge_replacement_files(tmpdir, mergefile):
         open(mergefile, "w").close()
 
 
-def find_binary(arg, name, build_path):
+def find_binary(arg: str, name: str, build_path: str) -> str:
     """Get the path for a binary or exit"""
     if arg:
         if shutil.which(arg):
             return arg
         else:
             raise SystemExit(
-                "error: passed binary '{}' was not found or is not executable".format(
-                    arg
-                )
+                f"error: passed binary '{arg}' was not found or is not executable"
             )
 
     built_path = os.path.join(build_path, "bin", name)
@@ -196,66 +192,102 @@ def find_binary(arg, name, build_path):
     if binary:
         return binary
     else:
-        raise SystemExit(
-            "error: failed to find {} in $PATH or at {}".format(name, built_path)
-        )
+        raise SystemExit(f"error: failed to find {name} in $PATH or at {built_path}")
 
 
-def apply_fixes(args, clang_apply_replacements_binary, tmpdir):
+def apply_fixes(
+    args: argparse.Namespace, clang_apply_replacements_binary: str, tmpdir: str
+) -> None:
     """Calls clang-apply-fixes on a given directory."""
     invocation = [clang_apply_replacements_binary]
     invocation.append("-ignore-insert-conflict")
     if args.format:
         invocation.append("-format")
     if args.style:
-        invocation.append("-style=" + args.style)
+        invocation.append(f"-style={args.style}")
     invocation.append(tmpdir)
     subprocess.call(invocation)
 
 
-def run_tidy(args, clang_tidy_binary, tmpdir, build_path, queue, lock, failed_files):
-    """Takes filenames out of queue and runs clang-tidy on them."""
-    while True:
-        name = queue.get()
-        invocation = get_tidy_invocation(
-            name,
-            clang_tidy_binary,
-            args.checks,
-            tmpdir,
-            build_path,
-            args.header_filter,
-            args.allow_enabling_alpha_checkers,
-            args.extra_arg,
-            args.extra_arg_before,
-            args.quiet,
-            args.config_file,
-            args.config,
-            args.line_filter,
-            args.use_color,
-            args.plugins,
-            args.warnings_as_errors,
-            args.exclude_header_filter,
-            args.allow_no_checks,
-        )
+# FIXME Python 3.12: This can be simplified out with run_with_semaphore[T](...).
+T = TypeVar("T")
+
+
+async def run_with_semaphore(
+    semaphore: asyncio.Semaphore,
+    f: Callable[..., Awaitable[T]],
+    *args: Any,
+    **kwargs: Any,
+) -> T:
+    async with semaphore:
+        return await f(*args, **kwargs)
+
+
+@dataclass
+class ClangTidyResult:
+    filename: str
+    invocation: List[str]
+    returncode: int
+    stdout: str
+    stderr: str
+    elapsed: float
+
+
+async def run_tidy(
+    args: argparse.Namespace,
+    name: str,
+    clang_tidy_binary: str,
+    tmpdir: str,
+    build_path: str,
+) -> ClangTidyResult:
+    """
+    Runs clang-tidy on a single file and returns the result.
+    """
+    invocation = get_tidy_invocation(
+        name,
+        clang_tidy_binary,
+        args.checks,
+        tmpdir,
+        build_path,
+        args.header_filter,
+        args.allow_enabling_alpha_checkers,
+        args.extra_arg,
+        args.extra_arg_before,
+        args.quiet,
+        args.config_file,
+        args.config,
+        args.line_filter,
+        args.use_color,
+        args.plugins,
+        args.warnings_as_errors,
+        args.exclude_header_filter,
+        args.allow_no_checks,
+    )
 
-        proc = subprocess.Popen(
-            invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    try:
+        process = await asyncio.create_subprocess_exec(
+            *invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE
         )
-        output, err = proc.communicate()
-        if proc.returncode != 0:
-            if proc.returncode < 0:
-                msg = "%s: terminated by signal %d\n" % (name, -proc.returncode)
-                err += msg.encode("utf-8")
-            failed_files.append(name)
-        with lock:
-            sys.stdout.write(" ".join(invocation) + "\n" + output.decode("utf-8"))
-            if len(err) > 0:
-                sys.stdout.flush()
-                sys.stderr.write(err.decode("utf-8"))
-        queue.task_done()
-
-
-def main():
+        start = time.time()
+        stdout, stderr = await process.communicate()
+        end = time.time()
+    except asyncio.CancelledError:
+        process.terminate()
+        await process.wait()
+        raise
+
+    assert process.returncode is not None
+    return ClangTidyResult(
+        name,
+        invocation,
+        process.returncode,
+        stdout.decode("UTF-8"),
+        stderr.decode("UTF-8"),
+        end - start,
+    )
+
+
+async def main() -> None:
     parser = argparse.ArgumentParser(
         description="Runs clang-tidy over all files "
         "in a compilation database. Requires "
@@ -432,7 +464,7 @@ def main():
         )
 
     combine_fixes = False
-    export_fixes_dir = None
+    export_fixes_dir: Optional[str] = None
     delete_fixes_dir = False
     if args.export_fixes is not None:
         # if a directory is given, create it if it does not exist
@@ -490,10 +522,10 @@ def main():
         sys.exit(1)
 
     # Load the database and extract all files.
-    database = json.load(open(os.path.join(build_path, db_path)))
-    files = set(
-        [make_absolute(entry["file"], entry["directory"]) for entry in database]
-    )
+    with open(os.path.join(build_path, db_path)) as f:
+        database = json.load(f)
+    files = {os.path.abspath(os.path.join(e["directory"], e["file"])) for e in database}
+    number_files_in_database = len(files)
 
     # Filter source files from compilation database.
     if args.source_filter:
@@ -514,70 +546,81 @@ def main():
 
     # Build up a big regexy filter from all command line arguments.
     file_name_re = re.compile("|".join(args.files))
+    files = {f for f in files if file_name_re.search(f)}
+
+    print(
+        "Running clang-tidy for",
+        len(files),
+        "files out of",
+        number_files_in_database,
+        "in compilation database ...",
+    )
 
-    return_code = 0
-    try:
-        # Spin up a bunch of tidy-launching threads.
-        task_queue = queue.Queue(max_task)
-        # List of files with a non-zero return code.
-        failed_files = []
-        lock = threading.Lock()
-        for _ in range(max_task):
-            t = threading.Thread(
-                target=run_tidy,
-                args=(
-                    args,
-                    clang_tidy_binary,
-                    export_fixes_dir,
-                    build_path,
-                    task_queue,
-                    lock,
-                    failed_files,
-                ),
+    returncode = 0
+    semaphore = asyncio.Semaphore(max_task)
+    tasks = [
+        asyncio.create_task(
+            run_with_semaphore(
+                semaphore,
+                run_tidy,
+                args,
+                f,
+                clang_tidy_binary,
+                export_fixes_dir,
+                build_path,
             )
-            t.daemon = True
-            t.start()
-
-        # Fill the queue with files.
-        for name in files:
-            if file_name_re.search(name):
-                task_queue.put(name)
-
-        # Wait for all threads to be done.
-        task_queue.join()
-        if len(failed_files):
-            return_code = 1
-
-    except KeyboardInterrupt:
-        # This is a sad hack. Unfortunately subprocess goes
-        # bonkers with ctrl-c and we start forking merrily.
+        )
+        for f in files
+    ]
+
+    try:
+        for i, coro in enumerate(asyncio.as_completed(tasks)):
+            result = await coro
+            if result.returncode != 0:
+                returncode = 1
+                if result.returncode < 0:
+                    result.stderr += f"{result.filename}: terminated by signal {-result.returncode}\n"
+            progress = f"[{i + 1: >{len(f'{len(files)}')}}/{len(files)}]"
+            runtime = f"[{result.elapsed:.1f}s]"
+            print(f"{progress}{runtime} {' '.join(result.invocation)}")
+            if result.stdout:
+                print(result.stdout, end=("" if result.stderr else "\n"))
+            if result.stderr:
+                print(result.stderr)
+    except asyncio.CancelledError:
         print("\nCtrl-C detected, goodbye.")
+        for task in tasks:
+            task.cancel()
         if delete_fixes_dir:
+            assert export_fixes_dir
             shutil.rmtree(export_fixes_dir)
-        os.kill(0, 9)
+        return
 
     if combine_fixes:
-        print("Writing fixes to " + args.export_fixes + " ...")
+        print(f"Writing fixes to {args.export_fixes} ...")
         try:
+            assert export_fixes_dir
             merge_replacement_files(export_fixes_dir, args.export_fixes)
         except:
             print("Error exporting fixes.\n", file=sys.stderr)
             traceback.print_exc()
-            return_code = 1
+            returncode = 1
 
     if args.fix:
         print("Applying fixes ...")
         try:
+            assert export_fixes_dir
             apply_fixes(args, clang_apply_replacements_binary, export_fixes_dir)
         except:
             print("Error applying fixes.\n", file=sys.stderr)
             traceback.print_exc()
-            return_code = 1
+            returncode = 1
 
     if delete_fixes_dir:
+        assert export_fixes_dir
         shutil.rmtree(export_fixes_dir)
-    sys.exit(return_code)
+    sys.exit(returncode)
 
 
 if __name__ == "__main__":
-    main()
+    asyncio.run(main())
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index a23483e6df6d2..70fc91fc0f6d1 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -116,6 +116,8 @@ Improvements to clang-tidy
 - Improved :program:`run-clang-tidy.py` script. Added argument `-source-filter`
   to filter source files from the compilation database, via a RegEx. In a
   similar fashion to what `-header-filter` does for header files.
+  Added progress indicator with a number of processed files and the runtime of
+  each invocation after completion.
 
 - Improved :program:`check_clang_tidy.py` script. Added argument `-export-fixes`
   to aid in clang-tidy and test development.

From 9d76231fcd3f6ddf9bd6aafe7f143404cad7e4a1 Mon Sep 17 00:00:00 2001
From: matthew-f <matthewf@cat.co.za>
Date: Mon, 22 Jul 2024 20:26:54 +0200
Subject: [PATCH 460/486] [clang-tidy] Ensure functions are anchored in the
 global namespace (for cert-err-33) (#99380)

The regular expressions match functions that aren't anchored in the
global namespace. For example `::remove` matches any object with a
`removeXyz` method. This change is to remove these false positives
---
 .../clang-tidy/cert/CERTTidyModule.cpp        | 354 +++++++++---------
 1 file changed, 177 insertions(+), 177 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp b/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp
index ffb62b409b29b..8b5be9cd95f76 100644
--- a/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp
@@ -49,183 +49,183 @@ namespace {
 // with NULL argument and in this case the check is not applicable:
 // `mblen, mbrlen, mbrtowc, mbtowc, wctomb, wctomb_s`.
 // FIXME: The check can be improved to handle such cases.
-const llvm::StringRef CertErr33CCheckedFunctions = "::aligned_alloc;"
-                                                   "::asctime_s;"
-                                                   "::at_quick_exit;"
-                                                   "::atexit;"
-                                                   "::bsearch;"
-                                                   "::bsearch_s;"
-                                                   "::btowc;"
-                                                   "::c16rtomb;"
-                                                   "::c32rtomb;"
-                                                   "::calloc;"
-                                                   "::clock;"
-                                                   "::cnd_broadcast;"
-                                                   "::cnd_init;"
-                                                   "::cnd_signal;"
-                                                   "::cnd_timedwait;"
-                                                   "::cnd_wait;"
-                                                   "::ctime_s;"
-                                                   "::fclose;"
-                                                   "::fflush;"
-                                                   "::fgetc;"
-                                                   "::fgetpos;"
-                                                   "::fgets;"
-                                                   "::fgetwc;"
-                                                   "::fopen;"
-                                                   "::fopen_s;"
-                                                   "::fprintf;"
-                                                   "::fprintf_s;"
-                                                   "::fputc;"
-                                                   "::fputs;"
-                                                   "::fputwc;"
-                                                   "::fputws;"
-                                                   "::fread;"
-                                                   "::freopen;"
-                                                   "::freopen_s;"
-                                                   "::fscanf;"
-                                                   "::fscanf_s;"
-                                                   "::fseek;"
-                                                   "::fsetpos;"
-                                                   "::ftell;"
-                                                   "::fwprintf;"
-                                                   "::fwprintf_s;"
-                                                   "::fwrite;"
-                                                   "::fwscanf;"
-                                                   "::fwscanf_s;"
-                                                   "::getc;"
-                                                   "::getchar;"
-                                                   "::getenv;"
-                                                   "::getenv_s;"
-                                                   "::gets_s;"
-                                                   "::getwc;"
-                                                   "::getwchar;"
-                                                   "::gmtime;"
-                                                   "::gmtime_s;"
-                                                   "::localtime;"
-                                                   "::localtime_s;"
-                                                   "::malloc;"
-                                                   "::mbrtoc16;"
-                                                   "::mbrtoc32;"
-                                                   "::mbsrtowcs;"
-                                                   "::mbsrtowcs_s;"
-                                                   "::mbstowcs;"
-                                                   "::mbstowcs_s;"
-                                                   "::memchr;"
-                                                   "::mktime;"
-                                                   "::mtx_init;"
-                                                   "::mtx_lock;"
-                                                   "::mtx_timedlock;"
-                                                   "::mtx_trylock;"
-                                                   "::mtx_unlock;"
-                                                   "::printf_s;"
-                                                   "::putc;"
-                                                   "::putwc;"
-                                                   "::raise;"
-                                                   "::realloc;"
-                                                   "::remove;"
-                                                   "::rename;"
-                                                   "::scanf;"
-                                                   "::scanf_s;"
-                                                   "::setlocale;"
-                                                   "::setvbuf;"
-                                                   "::signal;"
-                                                   "::snprintf;"
-                                                   "::snprintf_s;"
-                                                   "::sprintf;"
-                                                   "::sprintf_s;"
-                                                   "::sscanf;"
-                                                   "::sscanf_s;"
-                                                   "::strchr;"
-                                                   "::strerror_s;"
-                                                   "::strftime;"
-                                                   "::strpbrk;"
-                                                   "::strrchr;"
-                                                   "::strstr;"
-                                                   "::strtod;"
-                                                   "::strtof;"
-                                                   "::strtoimax;"
-                                                   "::strtok;"
-                                                   "::strtok_s;"
-                                                   "::strtol;"
-                                                   "::strtold;"
-                                                   "::strtoll;"
-                                                   "::strtoul;"
-                                                   "::strtoull;"
-                                                   "::strtoumax;"
-                                                   "::strxfrm;"
-                                                   "::swprintf;"
-                                                   "::swprintf_s;"
-                                                   "::swscanf;"
-                                                   "::swscanf_s;"
-                                                   "::thrd_create;"
-                                                   "::thrd_detach;"
-                                                   "::thrd_join;"
-                                                   "::thrd_sleep;"
-                                                   "::time;"
-                                                   "::timespec_get;"
-                                                   "::tmpfile;"
-                                                   "::tmpfile_s;"
-                                                   "::tmpnam;"
-                                                   "::tmpnam_s;"
-                                                   "::tss_create;"
-                                                   "::tss_get;"
-                                                   "::tss_set;"
-                                                   "::ungetc;"
-                                                   "::ungetwc;"
-                                                   "::vfprintf;"
-                                                   "::vfprintf_s;"
-                                                   "::vfscanf;"
-                                                   "::vfscanf_s;"
-                                                   "::vfwprintf;"
-                                                   "::vfwprintf_s;"
-                                                   "::vfwscanf;"
-                                                   "::vfwscanf_s;"
-                                                   "::vprintf_s;"
-                                                   "::vscanf;"
-                                                   "::vscanf_s;"
-                                                   "::vsnprintf;"
-                                                   "::vsnprintf_s;"
-                                                   "::vsprintf;"
-                                                   "::vsprintf_s;"
-                                                   "::vsscanf;"
-                                                   "::vsscanf_s;"
-                                                   "::vswprintf;"
-                                                   "::vswprintf_s;"
-                                                   "::vswscanf;"
-                                                   "::vswscanf_s;"
-                                                   "::vwprintf_s;"
-                                                   "::vwscanf;"
-                                                   "::vwscanf_s;"
-                                                   "::wcrtomb;"
-                                                   "::wcschr;"
-                                                   "::wcsftime;"
-                                                   "::wcspbrk;"
-                                                   "::wcsrchr;"
-                                                   "::wcsrtombs;"
-                                                   "::wcsrtombs_s;"
-                                                   "::wcsstr;"
-                                                   "::wcstod;"
-                                                   "::wcstof;"
-                                                   "::wcstoimax;"
-                                                   "::wcstok;"
-                                                   "::wcstok_s;"
-                                                   "::wcstol;"
-                                                   "::wcstold;"
-                                                   "::wcstoll;"
-                                                   "::wcstombs;"
-                                                   "::wcstombs_s;"
-                                                   "::wcstoul;"
-                                                   "::wcstoull;"
-                                                   "::wcstoumax;"
-                                                   "::wcsxfrm;"
-                                                   "::wctob;"
-                                                   "::wctrans;"
-                                                   "::wctype;"
-                                                   "::wmemchr;"
-                                                   "::wprintf_s;"
-                                                   "::wscanf;"
-                                                   "::wscanf_s;";
+const llvm::StringRef CertErr33CCheckedFunctions = "^::aligned_alloc;"
+                                                   "^::asctime_s;"
+                                                   "^::at_quick_exit;"
+                                                   "^::atexit;"
+                                                   "^::bsearch;"
+                                                   "^::bsearch_s;"
+                                                   "^::btowc;"
+                                                   "^::c16rtomb;"
+                                                   "^::c32rtomb;"
+                                                   "^::calloc;"
+                                                   "^::clock;"
+                                                   "^::cnd_broadcast;"
+                                                   "^::cnd_init;"
+                                                   "^::cnd_signal;"
+                                                   "^::cnd_timedwait;"
+                                                   "^::cnd_wait;"
+                                                   "^::ctime_s;"
+                                                   "^::fclose;"
+                                                   "^::fflush;"
+                                                   "^::fgetc;"
+                                                   "^::fgetpos;"
+                                                   "^::fgets;"
+                                                   "^::fgetwc;"
+                                                   "^::fopen;"
+                                                   "^::fopen_s;"
+                                                   "^::fprintf;"
+                                                   "^::fprintf_s;"
+                                                   "^::fputc;"
+                                                   "^::fputs;"
+                                                   "^::fputwc;"
+                                                   "^::fputws;"
+                                                   "^::fread;"
+                                                   "^::freopen;"
+                                                   "^::freopen_s;"
+                                                   "^::fscanf;"
+                                                   "^::fscanf_s;"
+                                                   "^::fseek;"
+                                                   "^::fsetpos;"
+                                                   "^::ftell;"
+                                                   "^::fwprintf;"
+                                                   "^::fwprintf_s;"
+                                                   "^::fwrite;"
+                                                   "^::fwscanf;"
+                                                   "^::fwscanf_s;"
+                                                   "^::getc;"
+                                                   "^::getchar;"
+                                                   "^::getenv;"
+                                                   "^::getenv_s;"
+                                                   "^::gets_s;"
+                                                   "^::getwc;"
+                                                   "^::getwchar;"
+                                                   "^::gmtime;"
+                                                   "^::gmtime_s;"
+                                                   "^::localtime;"
+                                                   "^::localtime_s;"
+                                                   "^::malloc;"
+                                                   "^::mbrtoc16;"
+                                                   "^::mbrtoc32;"
+                                                   "^::mbsrtowcs;"
+                                                   "^::mbsrtowcs_s;"
+                                                   "^::mbstowcs;"
+                                                   "^::mbstowcs_s;"
+                                                   "^::memchr;"
+                                                   "^::mktime;"
+                                                   "^::mtx_init;"
+                                                   "^::mtx_lock;"
+                                                   "^::mtx_timedlock;"
+                                                   "^::mtx_trylock;"
+                                                   "^::mtx_unlock;"
+                                                   "^::printf_s;"
+                                                   "^::putc;"
+                                                   "^::putwc;"
+                                                   "^::raise;"
+                                                   "^::realloc;"
+                                                   "^::remove;"
+                                                   "^::rename;"
+                                                   "^::scanf;"
+                                                   "^::scanf_s;"
+                                                   "^::setlocale;"
+                                                   "^::setvbuf;"
+                                                   "^::signal;"
+                                                   "^::snprintf;"
+                                                   "^::snprintf_s;"
+                                                   "^::sprintf;"
+                                                   "^::sprintf_s;"
+                                                   "^::sscanf;"
+                                                   "^::sscanf_s;"
+                                                   "^::strchr;"
+                                                   "^::strerror_s;"
+                                                   "^::strftime;"
+                                                   "^::strpbrk;"
+                                                   "^::strrchr;"
+                                                   "^::strstr;"
+                                                   "^::strtod;"
+                                                   "^::strtof;"
+                                                   "^::strtoimax;"
+                                                   "^::strtok;"
+                                                   "^::strtok_s;"
+                                                   "^::strtol;"
+                                                   "^::strtold;"
+                                                   "^::strtoll;"
+                                                   "^::strtoul;"
+                                                   "^::strtoull;"
+                                                   "^::strtoumax;"
+                                                   "^::strxfrm;"
+                                                   "^::swprintf;"
+                                                   "^::swprintf_s;"
+                                                   "^::swscanf;"
+                                                   "^::swscanf_s;"
+                                                   "^::thrd_create;"
+                                                   "^::thrd_detach;"
+                                                   "^::thrd_join;"
+                                                   "^::thrd_sleep;"
+                                                   "^::time;"
+                                                   "^::timespec_get;"
+                                                   "^::tmpfile;"
+                                                   "^::tmpfile_s;"
+                                                   "^::tmpnam;"
+                                                   "^::tmpnam_s;"
+                                                   "^::tss_create;"
+                                                   "^::tss_get;"
+                                                   "^::tss_set;"
+                                                   "^::ungetc;"
+                                                   "^::ungetwc;"
+                                                   "^::vfprintf;"
+                                                   "^::vfprintf_s;"
+                                                   "^::vfscanf;"
+                                                   "^::vfscanf_s;"
+                                                   "^::vfwprintf;"
+                                                   "^::vfwprintf_s;"
+                                                   "^::vfwscanf;"
+                                                   "^::vfwscanf_s;"
+                                                   "^::vprintf_s;"
+                                                   "^::vscanf;"
+                                                   "^::vscanf_s;"
+                                                   "^::vsnprintf;"
+                                                   "^::vsnprintf_s;"
+                                                   "^::vsprintf;"
+                                                   "^::vsprintf_s;"
+                                                   "^::vsscanf;"
+                                                   "^::vsscanf_s;"
+                                                   "^::vswprintf;"
+                                                   "^::vswprintf_s;"
+                                                   "^::vswscanf;"
+                                                   "^::vswscanf_s;"
+                                                   "^::vwprintf_s;"
+                                                   "^::vwscanf;"
+                                                   "^::vwscanf_s;"
+                                                   "^::wcrtomb;"
+                                                   "^::wcschr;"
+                                                   "^::wcsftime;"
+                                                   "^::wcspbrk;"
+                                                   "^::wcsrchr;"
+                                                   "^::wcsrtombs;"
+                                                   "^::wcsrtombs_s;"
+                                                   "^::wcsstr;"
+                                                   "^::wcstod;"
+                                                   "^::wcstof;"
+                                                   "^::wcstoimax;"
+                                                   "^::wcstok;"
+                                                   "^::wcstok_s;"
+                                                   "^::wcstol;"
+                                                   "^::wcstold;"
+                                                   "^::wcstoll;"
+                                                   "^::wcstombs;"
+                                                   "^::wcstombs_s;"
+                                                   "^::wcstoul;"
+                                                   "^::wcstoull;"
+                                                   "^::wcstoumax;"
+                                                   "^::wcsxfrm;"
+                                                   "^::wctob;"
+                                                   "^::wctrans;"
+                                                   "^::wctype;"
+                                                   "^::wmemchr;"
+                                                   "^::wprintf_s;"
+                                                   "^::wscanf;"
+                                                   "^::wscanf_s;";
 
 } // namespace
 

From 9374216d4b2c408a4537bdb6af05f1bc1e5e562b Mon Sep 17 00:00:00 2001
From: dyung <douglas.yung@sony.com>
Date: Mon, 22 Jul 2024 11:28:11 -0700
Subject: [PATCH 461/486] Replace distutils.version with packaging.version
 since the former was deprecated in python 3.10 and removed in 3.12. (#99852)

Attempt to reland #99549, but using packaging.version instead of
looseversion, based on the usage used for LLDB in #93712.
---
 cross-project-tests/lit.cfg.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/cross-project-tests/lit.cfg.py b/cross-project-tests/lit.cfg.py
index 774c4eaf4d976..9935fe6a199da 100644
--- a/cross-project-tests/lit.cfg.py
+++ b/cross-project-tests/lit.cfg.py
@@ -4,9 +4,6 @@
 import subprocess
 import sys
 
-# TODO: LooseVersion is undocumented; use something else.
-from distutils.version import LooseVersion
-
 import lit.formats
 import lit.util
 
@@ -279,7 +276,11 @@ def get_clang_default_dwarf_version_string(triple):
 gdb_version_string = get_gdb_version_string()
 if dwarf_version_string and gdb_version_string:
     if int(dwarf_version_string) >= 5:
-        if LooseVersion(gdb_version_string) < LooseVersion("10.1"):
+        try:
+            from packaging import version
+        except:
+            lit_config.fatal("Running gdb tests requires the packaging package")
+        if version.parse(gdb_version_string) < version.parse("10.1"):
             # Example for llgdb-tests, which use lldb on darwin but gdb elsewhere:
             # XFAIL: !system-darwin && gdb-clang-incompatibility
             config.available_features.add("gdb-clang-incompatibility")

From 0eb719fef57e560179cdeff02130638537970c24 Mon Sep 17 00:00:00 2001
From: Jessica Del <50999226+OutOfCache@users.noreply.github.com>
Date: Mon, 22 Jul 2024 20:28:20 +0200
Subject: [PATCH 462/486] [AMDGPU] Fix build failure in raw.atomic.buffer.load
 tests (#99912)

This fixes the failing tests after rebasing
over the attributor move.
---
 .../llvm.amdgcn.raw.atomic.buffer.load.ll     | 31 ++++++++++++-------
 .../llvm.amdgcn.raw.ptr.atomic.buffer.load.ll | 31 ++++++++++++-------
 2 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
index 03f94d6e853f0..419e19083f85e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
@@ -5,7 +5,8 @@
 define amdgpu_kernel void @raw_atomic_buffer_load_i32(<4 x i32> %addr) {
 ; CHECK-LABEL: raw_atomic_buffer_load_i32:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB0_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -33,7 +34,8 @@ bb2:
 define amdgpu_kernel void @raw_atomic_buffer_load_i32_off(<4 x i32> %addr) {
 ; CHECK-LABEL: raw_atomic_buffer_load_i32_off:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB1_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -60,7 +62,8 @@ bb2:
 define amdgpu_kernel void @raw_atomic_buffer_load_i32_soff(<4 x i32> %addr) {
 ; CHECK-LABEL: raw_atomic_buffer_load_i32_soff:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB2_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -87,7 +90,8 @@ bb2:
 define amdgpu_kernel void @raw_atomic_buffer_load_i32_dlc(<4 x i32> %addr) {
 ; CHECK-LABEL: raw_atomic_buffer_load_i32_dlc:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB3_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -115,7 +119,8 @@ bb2:
 define amdgpu_kernel void @raw_nonatomic_buffer_load_i32(<4 x i32> %addr) {
 ; CHECK-LABEL: raw_nonatomic_buffer_load_i32:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
 ; CHECK-NEXT:    s_mov_b32 s0, 0
@@ -144,8 +149,8 @@ bb2:
 define amdgpu_kernel void @raw_atomic_buffer_load_i64(<4 x i32> %addr) {
 ; CHECK-LABEL: raw_atomic_buffer_load_i64:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB5_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -174,7 +179,8 @@ bb2:
 define amdgpu_kernel void @raw_atomic_buffer_load_v2i16(<4 x i32> %addr) {
 ; CHECK-LABEL: raw_atomic_buffer_load_v2i16:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB6_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -203,7 +209,8 @@ bb2:
 define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
 ; CHECK-LABEL: raw_atomic_buffer_load_v4i16:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB7_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -236,7 +243,8 @@ bb2:
 define amdgpu_kernel void @raw_atomic_buffer_load_v4i32(<4 x i32> %addr) {
 ; CHECK-LABEL: raw_atomic_buffer_load_v4i32:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB8_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -265,7 +273,8 @@ bb2:
 define amdgpu_kernel void @raw_atomic_buffer_load_ptr(<4 x i32> %addr) {
 ; CHECK-LABEL: raw_atomic_buffer_load_ptr:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB9_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
index 3228335073d07..6541ac9553231 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
@@ -5,7 +5,8 @@
 define amdgpu_kernel void @raw_ptr_atomic_buffer_ptr_load_i32(ptr addrspace(8) %ptr) {
 ; CHECK-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB0_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -33,7 +34,8 @@ bb2:
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_off(ptr addrspace(8) %ptr) {
 ; CHECK-LABEL: raw_ptr_atomic_buffer_load_i32_off:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB1_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -60,7 +62,8 @@ bb2:
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_soff(ptr addrspace(8) %ptr) {
 ; CHECK-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB2_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -87,7 +90,8 @@ bb2:
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8) %ptr) {
 ; CHECK-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB3_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -115,7 +119,8 @@ bb2:
 define amdgpu_kernel void @raw_nonptr_atomic_buffer_load_i32(ptr addrspace(8) %ptr) {
 ; CHECK-LABEL: raw_nonptr_atomic_buffer_load_i32:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
 ; CHECK-NEXT:    s_mov_b32 s0, 0
@@ -144,8 +149,8 @@ bb2:
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr) {
 ; CHECK-LABEL: raw_ptr_atomic_buffer_load_i64:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB5_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -174,7 +179,8 @@ bb2:
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v2i16(ptr addrspace(8) %ptr) {
 ; CHECK-LABEL: raw_ptr_atomic_buffer_load_v2i16:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB6_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -203,7 +209,8 @@ bb2:
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %ptr) {
 ; CHECK-LABEL: raw_ptr_atomic_buffer_load_v4i16:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB7_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -236,7 +243,8 @@ bb2:
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i32(ptr addrspace(8) %ptr) {
 ; CHECK-LABEL: raw_ptr_atomic_buffer_load_v4i32:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB8_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -265,7 +273,8 @@ bb2:
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %ptr) {
 ; CHECK-LABEL: raw_ptr_atomic_buffer_load_ptr:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; CHECK-NEXT:    s_load_b128 s[0:3], s[2:3], 0x24
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:  .LBB9_1: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1

From 735974e550dae84b40bad06cbe8cf5f22d318896 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Mon, 22 Jul 2024 21:29:56 +0300
Subject: [PATCH 463/486] [builtins] Use __builtin_clzll for 64-bit types
 (#99874)

This addresses the issue with `__LP64__` not being defined for targets
with 32-bit pointers but 64-bit longs, resulting in worse codegen.
---
 compiler-rt/lib/builtins/fp_extend.h | 11 +----------
 compiler-rt/lib/builtins/fp_lib.h    | 11 +----------
 2 files changed, 2 insertions(+), 20 deletions(-)

diff --git a/compiler-rt/lib/builtins/fp_extend.h b/compiler-rt/lib/builtins/fp_extend.h
index 7637417649e62..22bf2b2514e57 100644
--- a/compiler-rt/lib/builtins/fp_extend.h
+++ b/compiler-rt/lib/builtins/fp_extend.h
@@ -37,16 +37,7 @@ static const int srcSigFracBits = 52;
 // srcBits - srcSigFracBits - 1
 static const int srcExpBits = 11;
 
-static inline int src_rep_t_clz_impl(src_rep_t a) {
-#if defined __LP64__
-  return __builtin_clzl(a);
-#else
-  if (a & REP_C(0xffffffff00000000))
-    return clzsi(a >> 32);
-  else
-    return 32 + clzsi(a & REP_C(0xffffffff));
-#endif
-}
+static inline int src_rep_t_clz_impl(src_rep_t a) { return __builtin_clzll(a); }
 #define src_rep_t_clz src_rep_t_clz_impl
 
 #elif defined SRC_80
diff --git a/compiler-rt/lib/builtins/fp_lib.h b/compiler-rt/lib/builtins/fp_lib.h
index 8404d98c93508..b2a89506135be 100644
--- a/compiler-rt/lib/builtins/fp_lib.h
+++ b/compiler-rt/lib/builtins/fp_lib.h
@@ -58,16 +58,7 @@ typedef double fp_t;
 #define REP_C UINT64_C
 #define significandBits 52
 
-static __inline int rep_clz(rep_t a) {
-#if defined __LP64__
-  return __builtin_clzl(a);
-#else
-  if (a & REP_C(0xffffffff00000000))
-    return clzsi(a >> 32);
-  else
-    return 32 + clzsi(a & REP_C(0xffffffff));
-#endif
-}
+static inline int rep_clz(rep_t a) { return __builtin_clzll(a); }
 
 #define loWord(a) (a & 0xffffffffU)
 #define hiWord(a) (a >> 32)

From 2ef12b55b2a655262b03204581076522261d5eb3 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Mon, 22 Jul 2024 20:30:16 +0200
Subject: [PATCH 464/486] [clang] Fix the broken DeductionGuide ToolingTests
 after c7bfc41860a6abe5c92dc5afb47348b0c9e69963

---
 .../Tooling/RecursiveASTVisitorTests/DeductionGuide.cpp       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/DeductionGuide.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/DeductionGuide.cpp
index cd4bf0eb7bd5a..df878bfc113e5 100644
--- a/clang/unittests/Tooling/RecursiveASTVisitorTests/DeductionGuide.cpp
+++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/DeductionGuide.cpp
@@ -37,7 +37,7 @@ TEST(RecursiveASTVisitor, DeductionGuideNonImplicitMode) {
   // Verify that the synthezied deduction guide for alias is not visited in
   // RAV's implicit mode.
   Visitor.ExpectMatch("Foo(T) -> Foo<int>", 11, 1);
-  Visitor.DisallowMatch("Bar(type-parameter-0-0) -> Foo<int>", 14, 1);
+  Visitor.DisallowMatch("Bar(T) -> Foo<int>", 14, 1);
   EXPECT_TRUE(Visitor.runOver(
       R"cpp(
 template <typename T>
@@ -61,7 +61,7 @@ Bar s(1);
 TEST(RecursiveASTVisitor, DeductionGuideImplicitMode) {
   DeductionGuideVisitor Visitor(/*ShouldVisitImplicitCode*/ true);
   Visitor.ExpectMatch("Foo(T) -> Foo<int>", 11, 1);
-  Visitor.ExpectMatch("Bar(type-parameter-0-0) -> Foo<int>", 14, 1);
+  Visitor.ExpectMatch("Bar(T) -> Foo<int>", 14, 1);
   EXPECT_TRUE(Visitor.runOver(
       R"cpp(
 template <typename T>

From bb8a74075b164ea0d9b3155f64d0590fc6072cdd Mon Sep 17 00:00:00 2001
From: Med Ismail Bennani <ismail@bennani.ma>
Date: Mon, 22 Jul 2024 11:43:32 -0700
Subject: [PATCH 465/486] [lldb] Change GetStartSymbol to GetStartAddress in
 DynamicLoader (#99909)

On linux, the start address doesn't necessarily have a symbol attached
to it.

This is why this patch replaces `DynamicLoader::GetStartSymbol` with
`DynamicLoader::GetStartAddress` instead to make it more generic.

Signed-off-by: Med Ismail Bennani <ismail@bennani.ma>
---
 lldb/include/lldb/Target/DynamicLoader.h                 | 9 +++++----
 .../DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp    | 4 ++--
 .../DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h      | 4 ++--
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/lldb/include/lldb/Target/DynamicLoader.h b/lldb/include/lldb/Target/DynamicLoader.h
index 15384245194b0..0629e2faae7e9 100644
--- a/lldb/include/lldb/Target/DynamicLoader.h
+++ b/lldb/include/lldb/Target/DynamicLoader.h
@@ -9,8 +9,8 @@
 #ifndef LLDB_TARGET_DYNAMICLOADER_H
 #define LLDB_TARGET_DYNAMICLOADER_H
 
+#include "lldb/Core/Address.h"
 #include "lldb/Core/PluginInterface.h"
-#include "lldb/Symbol/Symbol.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/UUID.h"
@@ -25,6 +25,7 @@ namespace lldb_private {
 class ModuleList;
 class Process;
 class SectionList;
+class Symbol;
 class SymbolContext;
 class SymbolContextList;
 class Thread;
@@ -329,10 +330,10 @@ class DynamicLoader : public PluginInterface {
   /// safe to call certain APIs or SPIs.
   virtual bool IsFullyInitialized() { return true; }
 
-  /// Return the `start` function \b symbol in the dynamic loader module.
-  /// This is the symbol the process will begin executing with
+  /// Return the `start` \b address in the dynamic loader module.
+  /// This is the address the process will begin executing with
   /// `process launch --stop-at-entry`.
-  virtual std::optional<lldb_private::Symbol> GetStartSymbol() {
+  virtual std::optional<lldb_private::Address> GetStartAddress() {
     return std::nullopt;
   }
 
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
index 5c6331735bde8..3863b6b3520db 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
@@ -609,7 +609,7 @@ void DynamicLoaderDarwin::UpdateDYLDImageInfoFromNewImageInfo(
   }
 }
 
-std::optional<lldb_private::Symbol> DynamicLoaderDarwin::GetStartSymbol() {
+std::optional<lldb_private::Address> DynamicLoaderDarwin::GetStartAddress() {
   Log *log = GetLog(LLDBLog::DynamicLoader);
 
   auto log_err = [log](llvm::StringLiteral err_msg) -> std::nullopt_t {
@@ -626,7 +626,7 @@ std::optional<lldb_private::Symbol> DynamicLoaderDarwin::GetStartSymbol() {
   if (!symbol)
     return log_err("Cannot find `start` symbol in DYLD module.");
 
-  return *symbol;
+  return symbol->GetAddress();
 }
 
 void DynamicLoaderDarwin::SetDYLDModule(lldb::ModuleSP &dyld_module_sp) {
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h
index 4ac55fdf6f3af..3613c4c29b178 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h
@@ -56,6 +56,8 @@ class DynamicLoaderDarwin : public lldb_private::DynamicLoader {
 
   virtual bool NeedToDoInitialImageFetch() = 0;
 
+  std::optional<lldb_private::Address> GetStartAddress() override;
+
 protected:
   void PrivateInitialize(lldb_private::Process *process);
 
@@ -67,8 +69,6 @@ class DynamicLoaderDarwin : public lldb_private::DynamicLoader {
   // Clear method for classes derived from this one
   virtual void DoClear() = 0;
 
-  std::optional<lldb_private::Symbol> GetStartSymbol() override;
-
   void SetDYLDModule(lldb::ModuleSP &dyld_module_sp);
 
   lldb::ModuleSP GetDYLDModule();

From 52dd4dbb1ad30ed057c532148cc95f34efe34aa0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Hana=20Dus=C3=ADkov=C3=A1?= <hanicka@hanicka.net>
Date: Mon, 22 Jul 2024 20:49:25 +0200
Subject: [PATCH 466/486] [clang-tidy] `bugprone-exception-escape` didn't
 detech catching of an exception with pointer type by `void *` exception
 handler (#99773)

As in title, code which checks eligibility of exceptions with pointer
types to be handled by exception handler of type `void *` disallowed
this case. It was working like this:

```c++
if (isStandardPointerConvertible(ExceptionCanTy, HandlerCanTy) &&
          isUnambiguousPublicBaseClass(
              ExceptionCanTy->getTypePtr()->getPointeeType().getTypePtr(),
              HandlerCanTy->getTypePtr()->getPointeeType().getTypePtr())) {
```

but in `isUnambiguousPublicBaseClass` there was code which looked for
definitions:

```c++
bool isUnambiguousPublicBaseClass(const Type *DerivedType,
                                  const Type *BaseType) {
  const auto *DerivedClass =
      DerivedType->getCanonicalTypeUnqualified()->getAsCXXRecordDecl();
  const auto *BaseClass =
      BaseType->getCanonicalTypeUnqualified()->getAsCXXRecordDecl();
  if (!DerivedClass || !BaseClass)
    return false;
```

This code disallowed usage of `void *` type which was already correctly
detected in `isStandardPointerConvertible`.

AFAIK this seems like misinterpretation of specification:

> 14.4 Handling an exception
> a standard [pointer conversion](https://eel.is/c++draft/conv.ptr) not
involving conversions to pointers to private or protected or ambiguous
classes
(https://eel.is/c++draft/except.handle#3.3.1)

and

> 7.3.12 Pointer conversions
> ... If B is an inaccessible
([[class.access]](https://eel.is/c++draft/class.access)) or ambiguous
([[class.member.lookup]](https://eel.is/c++draft/class.member.lookup))
base class of D, a program that necessitates this conversion is
ill-formed[.](https://eel.is/c++draft/conv.ptr#3.sentence-2) ...
(https://eel.is/c++draft/conv.ptr#3)

14.4 is carving out private, protected, and ambiguous base classes, but
they are already carved out in 7.3.12 and implemented in
`isStandardPointerConvertible`

---------

Co-authored-by: Piotr Zegar <me@piotrzegar.pl>
---
 .../clang-tidy/utils/ExceptionAnalyzer.cpp     | 10 +++++-----
 clang-tools-extra/docs/ReleaseNotes.rst        |  4 ++++
 .../checkers/bugprone/exception-escape.cpp     | 18 ++++++++++++++++++
 3 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/utils/ExceptionAnalyzer.cpp b/clang-tools-extra/clang-tidy/utils/ExceptionAnalyzer.cpp
index 6ae46e2b1262a..9bfb7e2677533 100644
--- a/clang-tools-extra/clang-tidy/utils/ExceptionAnalyzer.cpp
+++ b/clang-tools-extra/clang-tidy/utils/ExceptionAnalyzer.cpp
@@ -141,7 +141,10 @@ bool isStandardPointerConvertible(QualType From, QualType To) {
     if (RD->isCompleteDefinition() &&
         isBaseOf(From->getPointeeType().getTypePtr(),
                  To->getPointeeType().getTypePtr())) {
-      return true;
+      // If B is an inaccessible or ambiguous base class of D, a program
+      // that necessitates this conversion is ill-formed
+      return isUnambiguousPublicBaseClass(From->getPointeeType().getTypePtr(),
+                                          To->getPointeeType().getTypePtr());
     }
   }
 
@@ -375,10 +378,7 @@ bool ExceptionAnalyzer::ExceptionInfo::filterByCatch(
         isPointerOrPointerToMember(ExceptionCanTy->getTypePtr())) {
       // A standard pointer conversion not involving conversions to pointers to
       // private or protected or ambiguous classes ...
-      if (isStandardPointerConvertible(ExceptionCanTy, HandlerCanTy) &&
-          isUnambiguousPublicBaseClass(
-              ExceptionCanTy->getTypePtr()->getPointeeType().getTypePtr(),
-              HandlerCanTy->getTypePtr()->getPointeeType().getTypePtr())) {
+      if (isStandardPointerConvertible(ExceptionCanTy, HandlerCanTy)) {
         TypesToDelete.push_back(ExceptionTy);
       }
       // A function pointer conversion ...
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 70fc91fc0f6d1..0b2c04c23761c 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -247,6 +247,10 @@ Changes in existing checks
   where source is already a ``void``` pointer, making middle ``void`` pointer
   casts bug-free.
 
+- Improved :doc:`bugprone-exception-escape
+  <clang-tidy/checks/bugprone/exception-escape>`  check to correctly detect exception 
+  handler of type ``CV void *`` as catching all  ``CV`` compatible pointer types.
+
 - Improved :doc:`bugprone-forwarding-reference-overload
   <clang-tidy/checks/bugprone/forwarding-reference-overload>`
   check to ignore deleted constructors which won't hide other overloads.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp
index f5e74df1621ce..26c443b139629 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp
@@ -756,3 +756,21 @@ struct test_implicit_throw {
 };
 
 }}
+
+void pointer_exception_can_not_escape_with_const_void_handler() noexcept {
+  // CHECK-MESSAGES-NOT: :[[@LINE-1]]:6: warning: an exception may be thrown in function 'pointer_exception_can_not_escape_with_const_void_handler' which should not throw exceptions
+  const int value = 42;
+  try {
+    throw &value;
+  } catch (const void *) {
+  }
+}
+
+void pointer_exception_can_not_escape_with_void_handler() noexcept {
+  // CHECK-MESSAGES-NOT: :[[@LINE-1]]:6: warning: an exception may be thrown in function 'pointer_exception_can_not_escape_with_void_handler' which should not throw exceptions
+  int value = 42;
+  try {
+    throw &value;
+  } catch (void *) {
+  }
+}

From 50c4e0392a42d1203fc17339c86216d7c378b21f Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Mon, 22 Jul 2024 11:49:37 -0700
Subject: [PATCH 467/486] [clang][test] Add missing test file to cmake (#99907)

Seems like this test was never running with cmake, but is running with
bazel and broke at head.
---
 clang/unittests/Tooling/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/unittests/Tooling/CMakeLists.txt b/clang/unittests/Tooling/CMakeLists.txt
index 0eb612f8d9498..401978c31863c 100644
--- a/clang/unittests/Tooling/CMakeLists.txt
+++ b/clang/unittests/Tooling/CMakeLists.txt
@@ -40,6 +40,7 @@ add_clang_unittest(ToolingTests
   RecursiveASTVisitorTests/CXXMethodDecl.cpp
   RecursiveASTVisitorTests/CXXOperatorCallExprTraverser.cpp
   RecursiveASTVisitorTests/DeclRefExpr.cpp
+  RecursiveASTVisitorTests/DeductionGuide.cpp
   RecursiveASTVisitorTests/ImplicitCtor.cpp
   RecursiveASTVisitorTests/ImplicitCtorInitializer.cpp
   RecursiveASTVisitorTests/InitListExprPostOrder.cpp

From d6e17d70969722a3a2ecf6690a7fddcaca2ea14f Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 22 Jul 2024 18:49:56 +0000
Subject: [PATCH 468/486] [gn build] Port 50c4e0392a42

---
 llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
index 258fb61038df5..8e1d09c9c2ee9 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
@@ -59,6 +59,7 @@ unittest("ToolingTests") {
     "RecursiveASTVisitorTests/Concept.cpp",
     "RecursiveASTVisitorTests/ConstructExpr.cpp",
     "RecursiveASTVisitorTests/DeclRefExpr.cpp",
+    "RecursiveASTVisitorTests/DeductionGuide.cpp",
     "RecursiveASTVisitorTests/ImplicitCtor.cpp",
     "RecursiveASTVisitorTests/ImplicitCtorInitializer.cpp",
     "RecursiveASTVisitorTests/InitListExprPostOrder.cpp",

From 3b24e5d450c01080610e3cbf79727b454cad0d32 Mon Sep 17 00:00:00 2001
From: Edd Dawson <edd.dawson@sony.com>
Date: Mon, 22 Jul 2024 19:56:11 +0100
Subject: [PATCH 469/486] Omit .debug_aranges if it is empty (#99897)

SIE tracker: https://jira.sie.sony.com/browse/TOOLCHAIN-16575
---
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp | 6 ++++--
 llvm/test/DebugInfo/omit-empty.ll          | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 80cd5ec501f25..f88653146cc6f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -2990,6 +2990,9 @@ struct ArangeSpan {
 // Emit a debug aranges section, containing a CU lookup for any
 // address we can tie back to a CU.
 void DwarfDebug::emitDebugARanges() {
+  if (ArangeLabels.empty())
+    return;
+
   // Provides a unique id per text section.
   MapVector<MCSection *, SmallVector<SymbolCU, 8>> SectionMap;
 
@@ -3012,8 +3015,7 @@ void DwarfDebug::emitDebugARanges() {
   for (auto &I : SectionMap) {
     MCSection *Section = I.first;
     SmallVector<SymbolCU, 8> &List = I.second;
-    if (List.size() < 1)
-      continue;
+    assert(!List.empty());
 
     // If we have no section (e.g. common), just write out
     // individual spans for each symbol.
diff --git a/llvm/test/DebugInfo/omit-empty.ll b/llvm/test/DebugInfo/omit-empty.ll
index 0267ec5556f11..351d3055e039b 100644
--- a/llvm/test/DebugInfo/omit-empty.ll
+++ b/llvm/test/DebugInfo/omit-empty.ll
@@ -1,4 +1,4 @@
-; RUN: %llc_dwarf %s -filetype=obj -o - | llvm-objdump -h - | FileCheck %s
+; RUN: %llc_dwarf %s -filetype=obj -generate-arange-section -o - | llvm-objdump -h - | FileCheck %s
 ; REQUIRES: object-emission
 
 ; CHECK-NOT: .debug_

From a6341718967a708149159e8583549fa15198f902 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Mon, 22 Jul 2024 11:57:36 -0700
Subject: [PATCH 470/486] [InstrPGO][TypeProf]Annotate vtable types when they
 are present in the profile (#99402)

Before this change, when `file.profdata` have vtable profiles but `--enable-vtable-value-profiling` is not on for optimized build, warnings from this line [1] will show up. They are benign for performance but confusing.

It's better to automatically annotate vtable profiles if `file.profdata` has them. This PR implements it in profile use pass.
* If `-icp-max-num-vtables` is zero (default value is 6), vtable profiles won't be annotated.

[1] https://github.com/llvm/llvm-project/blob/464d321ee8dde1eaf14b5537eaf030e6df513849/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp#L1762-L1768
---
 .../Linux/instrprof-vtable-value-prof.cpp     | 27 ++++++++++++++++---
 .../Instrumentation/PGOInstrumentation.cpp    | 21 +++++++++++++--
 2 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp
index 24a96a227393a..411640fc5176d 100644
--- a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp
+++ b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp
@@ -109,6 +109,25 @@
 // ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E:750
 // ICTEXT: _ZTV8Derived1:250
 
+// When vtable value profiles exist, pgo-instr-use pass should annotate them
+// even if `-enable-vtable-value-profiling` is not explicitly on.
+// RUN: %clangxx -m64 -fprofile-use=test.profdata -fuse-ld=lld -O2 \
+// RUN:   -mllvm -print-after=pgo-instr-use -mllvm -filter-print-funcs=main \
+// RUN:   -mllvm -print-module-scope %s 2>&1 | FileCheck %s --check-prefix=ANNOTATE
+
+// ANNOTATE-NOT: Inconsistent number of value sites
+// ANNOTATE: !{!"VP", i32 2
+
+// When vtable value profiles exist, pgo-instr-use pass will not annotate them
+// if `-icp-max-num-vtables` is set to zero.
+// RUN: %clangxx -m64 -fprofile-use=test.profdata -fuse-ld=lld -O2 \
+// RUN:   -mllvm -icp-max-num-vtables=0 -mllvm -print-after=pgo-instr-use \
+// RUN:   -mllvm -filter-print-funcs=main -mllvm -print-module-scope %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=OMIT
+
+// OMIT: Inconsistent number of value sites
+// OMIT-NOT: !{!"VP", i32 2
+
 // Test indirect call promotion transformation using vtable profiles.
 // - Build with `-g` to enable debug information.
 // - In real world settings, ICP pass is disabled in prelink pipeline. In
@@ -128,12 +147,12 @@
 // RUN:    | FileCheck %s --check-prefixes=REMARK,IR --implicit-check-not="!VP"
 
 // For the indirect call site `ptr->func`
-// REMARK: instrprof-vtable-value-prof.cpp:205:19: Promote indirect call to _ZN12_GLOBAL__N_18Derived24funcEii with count 150 out of 200, sink 1 instruction(s) and compare 1 vtable(s): {_ZTVN12_GLOBAL__N_18Derived2E}
-// REMARK: instrprof-vtable-value-prof.cpp:205:19: Promote indirect call to _ZN8Derived14funcEii with count 50 out of 50, sink 1 instruction(s) and compare 1 vtable(s): {_ZTV8Derived1}
+// REMARK: instrprof-vtable-value-prof.cpp:226:19: Promote indirect call to _ZN12_GLOBAL__N_18Derived24funcEii with count 150 out of 200, sink 1 instruction(s) and compare 1 vtable(s): {_ZTVN12_GLOBAL__N_18Derived2E}
+// REMARK: instrprof-vtable-value-prof.cpp:226:19: Promote indirect call to _ZN8Derived14funcEii with count 50 out of 50, sink 1 instruction(s) and compare 1 vtable(s): {_ZTV8Derived1}
 //
 // For the indirect call site `delete ptr`
-// REMARK: instrprof-vtable-value-prof.cpp:207:5: Promote indirect call to _ZN12_GLOBAL__N_18Derived2D0Ev with count 750 out of 1000, sink 2 instruction(s) and compare 1 vtable(s): {_ZTVN12_GLOBAL__N_18Derived2E}
-// REMARK: instrprof-vtable-value-prof.cpp:207:5: Promote indirect call to _ZN8Derived1D0Ev with count 250 out of 250, sink 2 instruction(s) and compare 1 vtable(s): {_ZTV8Derived1}
+// REMARK: instrprof-vtable-value-prof.cpp:228:5: Promote indirect call to _ZN12_GLOBAL__N_18Derived2D0Ev with count 750 out of 1000, sink 2 instruction(s) and compare 1 vtable(s): {_ZTVN12_GLOBAL__N_18Derived2E}
+// REMARK: instrprof-vtable-value-prof.cpp:228:5: Promote indirect call to _ZN8Derived1D0Ev with count 250 out of 250, sink 2 instruction(s) and compare 1 vtable(s): {_ZTV8Derived1}
 
 // The IR matchers for indirect callsite `ptr->func`.
 // IR-LABEL: @main
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 4f0ccc8f962db..1ce8f58c1aa14 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1096,7 +1096,7 @@ class PGOUseFunc {
       : F(Func), M(Modu), BFI(BFIin), PSI(PSI),
         FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, IsCS,
                  InstrumentFuncEntry, HasSingleByteCoverage),
-        FreqAttr(FFA_Normal), IsCS(IsCS) {}
+        FreqAttr(FFA_Normal), IsCS(IsCS), VPC(Func, TLI) {}
 
   void handleInstrProfError(Error Err, uint64_t MismatchedFuncSum);
 
@@ -1178,6 +1178,8 @@ class PGOUseFunc {
   // Is to use the context sensitive profile.
   bool IsCS;
 
+  ValueProfileCollector VPC;
+
   // Find the Instrumented BB and set the value. Return false on error.
   bool setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
 
@@ -1755,8 +1757,23 @@ void PGOUseFunc::annotateValueSites() {
 void PGOUseFunc::annotateValueSites(uint32_t Kind) {
   assert(Kind <= IPVK_Last);
   unsigned ValueSiteIndex = 0;
-  auto &ValueSites = FuncInfo.ValueSites[Kind];
+
   unsigned NumValueSites = ProfileRecord.getNumValueSites(Kind);
+
+  // Since there isn't a reliable or fast way for profile reader to tell if a
+  // profile is generated with `-enable-vtable-value-profiling` on, we run the
+  // value profile collector over the function IR to find the instrumented sites
+  // iff function profile records shows the number of instrumented vtable sites
+  // is not zero. Function cfg already takes the number of instrumented
+  // indirect call sites into account so it doesn't hash the number of
+  // instrumented vtables; as a side effect it makes it easier to enable
+  // profiling and profile use in two steps if needed.
+  // TODO: Remove this if/when -enable-vtable-value-profiling is on by default.
+  if (NumValueSites > 0 && Kind == IPVK_VTableTarget &&
+      NumValueSites != FuncInfo.ValueSites[IPVK_VTableTarget].size() &&
+      MaxNumVTableAnnotations != 0)
+    FuncInfo.ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);
+  auto &ValueSites = FuncInfo.ValueSites[Kind];
   if (NumValueSites != ValueSites.size()) {
     auto &Ctx = M->getContext();
     Ctx.diagnose(DiagnosticInfoPGOProfile(

From da2f7201f3a24c2eb3120960eeae0ab6bcd5cfa2 Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Mon, 22 Jul 2024 12:10:51 -0700
Subject: [PATCH 471/486] [libc] Include cbrt in baremetal targets (#99916)

This is a follow up to #99262.
---
 libc/config/baremetal/arm/entrypoints.txt   | 1 +
 libc/config/baremetal/riscv/entrypoints.txt | 1 +
 2 files changed, 2 insertions(+)

diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt
index 4f2462abc7452..8025ac09b9f82 100644
--- a/libc/config/baremetal/arm/entrypoints.txt
+++ b/libc/config/baremetal/arm/entrypoints.txt
@@ -236,6 +236,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.canonicalize
     libc.src.math.canonicalizef
     libc.src.math.canonicalizel
+    libc.src.math.cbrt
     libc.src.math.cbrtf
     libc.src.math.ceil
     libc.src.math.ceilf
diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt
index c2ab9d7f19921..fb0308c953746 100644
--- a/libc/config/baremetal/riscv/entrypoints.txt
+++ b/libc/config/baremetal/riscv/entrypoints.txt
@@ -232,6 +232,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.canonicalize
     libc.src.math.canonicalizef
     libc.src.math.canonicalizel
+    libc.src.math.cbrt
     libc.src.math.cbrtf
     libc.src.math.ceil
     libc.src.math.ceilf

From 04d5003f59c1ef10a036d2158f1aa61de506e707 Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Mon, 22 Jul 2024 19:29:19 +0000
Subject: [PATCH 472/486] [clang-tidy][DOC] Update check documentation

Fix issues in list.rst, addapt add_new_check.py to new
format of that file, and run gen-static-analyzer-docs.py
to generate missing documentation for clang-analyzer.
---
 clang-tools-extra/clang-tidy/add_new_check.py      |  4 ++--
 .../checks/clang-analyzer/cplusplus.Move.rst       |  8 ++++++--
 .../clang-analyzer/optin.taint.TaintedAlloc.rst    | 14 ++++++++++++++
 .../clang-analyzer/security.PutenvStackArray.rst   | 10 ++++++++++
 .../clang-analyzer/unix.BlockInCriticalSection.rst | 13 +++++++++++++
 clang-tools-extra/docs/clang-tidy/checks/list.rst  | 10 +++++++---
 6 files changed, 52 insertions(+), 7 deletions(-)
 create mode 100644 clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/optin.taint.TaintedAlloc.rst
 create mode 100644 clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/security.PutenvStackArray.rst
 create mode 100644 clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/unix.BlockInCriticalSection.rst

diff --git a/clang-tools-extra/clang-tidy/add_new_check.py b/clang-tools-extra/clang-tidy/add_new_check.py
index 3b14d5d158d2d..3a62df1f510ba 100755
--- a/clang-tools-extra/clang-tidy/add_new_check.py
+++ b/clang-tools-extra/clang-tidy/add_new_check.py
@@ -552,8 +552,8 @@ def format_link_alias(doc_file):
                 f.write('   :header: "Name", "Offers fixes"\n\n')
                 f.writelines(checks)
                 # and the aliases
-                f.write("\n\n")
-                f.write(".. csv-table:: Aliases..\n")
+                f.write("\nCheck aliases\n-------------\n\n")
+                f.write(".. csv-table::\n")
                 f.write('   :header: "Name", "Redirect", "Offers fixes"\n\n')
                 f.writelines(checks_alias)
                 break
diff --git a/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/cplusplus.Move.rst b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/cplusplus.Move.rst
index e723f21f6bc60..f478598bb4e6c 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/cplusplus.Move.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/cplusplus.Move.rst
@@ -1,9 +1,13 @@
 .. title:: clang-tidy - clang-analyzer-cplusplus.Move
+.. meta::
+   :http-equiv=refresh: 5;URL=https://clang.llvm.org/docs/analyzer/checkers.html#cplusplus-move
 
 clang-analyzer-cplusplus.Move
 =============================
 
 Find use-after-move bugs in C++.
 
-The clang-analyzer-cplusplus.Move check is an alias of
-Clang Static Analyzer cplusplus.Move.
+The `clang-analyzer-cplusplus.Move` check is an alias, please see
+`Clang Static Analyzer Available Checkers
+<https://clang.llvm.org/docs/analyzer/checkers.html#cplusplus-move>`_
+for more information.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/optin.taint.TaintedAlloc.rst b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/optin.taint.TaintedAlloc.rst
new file mode 100644
index 0000000000000..9732333c61aa7
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/optin.taint.TaintedAlloc.rst
@@ -0,0 +1,14 @@
+.. title:: clang-tidy - clang-analyzer-optin.taint.TaintedAlloc
+.. meta::
+   :http-equiv=refresh: 5;URL=https://clang.llvm.org/docs/analyzer/checkers.html#optin-taint-taintedalloc
+
+clang-analyzer-optin.taint.TaintedAlloc
+=======================================
+
+Check for memory allocations, where the size parameter might be a tainted
+(attacker controlled) value.
+
+The `clang-analyzer-optin.taint.TaintedAlloc` check is an alias, please see
+`Clang Static Analyzer Available Checkers
+<https://clang.llvm.org/docs/analyzer/checkers.html#optin-taint-taintedalloc>`_
+for more information.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/security.PutenvStackArray.rst b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/security.PutenvStackArray.rst
new file mode 100644
index 0000000000000..0a5feff8d3ca8
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/security.PutenvStackArray.rst
@@ -0,0 +1,10 @@
+.. title:: clang-tidy - clang-analyzer-security.PutenvStackArray
+
+clang-analyzer-security.PutenvStackArray
+========================================
+
+Finds calls to the function 'putenv' which pass a pointer to an automatic
+(stack-allocated) array as the argument.
+
+The clang-analyzer-security.PutenvStackArray check is an alias of
+Clang Static Analyzer security.PutenvStackArray.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/unix.BlockInCriticalSection.rst b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/unix.BlockInCriticalSection.rst
new file mode 100644
index 0000000000000..40d2d3d8b8aac
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/unix.BlockInCriticalSection.rst
@@ -0,0 +1,13 @@
+.. title:: clang-tidy - clang-analyzer-unix.BlockInCriticalSection
+.. meta::
+   :http-equiv=refresh: 5;URL=https://clang.llvm.org/docs/analyzer/checkers.html#unix-blockincriticalsection
+
+clang-analyzer-unix.BlockInCriticalSection
+==========================================
+
+Check for calls to blocking functions inside a critical section.
+
+The `clang-analyzer-unix.BlockInCriticalSection` check is an alias, please see
+`Clang Static Analyzer Available Checkers
+<https://clang.llvm.org/docs/analyzer/checkers.html#unix-blockincriticalsection>`_
+for more information.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index dd2887edb0f8d..a931ebf025a10 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -118,6 +118,7 @@ Clang-Tidy Checks
    :doc:`bugprone-not-null-terminated-result <bugprone/not-null-terminated-result>`, "Yes"
    :doc:`bugprone-optional-value-conversion <bugprone/optional-value-conversion>`, "Yes"
    :doc:`bugprone-parent-virtual-call <bugprone/parent-virtual-call>`, "Yes"
+   :doc:`bugprone-pointer-arithmetic-on-polymorphic-object <bugprone/pointer-arithmetic-on-polymorphic-object>`,
    :doc:`bugprone-posix-return <bugprone/posix-return>`, "Yes"
    :doc:`bugprone-redundant-branch-condition <bugprone/redundant-branch-condition>`, "Yes"
    :doc:`bugprone-reserved-identifier <bugprone/reserved-identifier>`, "Yes"
@@ -158,7 +159,6 @@ Clang-Tidy Checks
    :doc:`bugprone-unused-raii <bugprone/unused-raii>`, "Yes"
    :doc:`bugprone-unused-return-value <bugprone/unused-return-value>`,
    :doc:`bugprone-use-after-move <bugprone/use-after-move>`,
-   :doc:`bugprone-pointer-arithmetic-on-polymorphic-object <bugprone/pointer-arithmetic-on-polymorphic-object>`,
    :doc:`bugprone-virtual-near-miss <bugprone/virtual-near-miss>`, "Yes"
    :doc:`cert-dcl50-cpp <cert/dcl50-cpp>`,
    :doc:`cert-dcl58-cpp <cert/dcl58-cpp>`,
@@ -269,7 +269,7 @@ Clang-Tidy Checks
    :doc:`misc-unused-parameters <misc/unused-parameters>`, "Yes"
    :doc:`misc-unused-using-decls <misc/unused-using-decls>`, "Yes"
    :doc:`misc-use-anonymous-namespace <misc/use-anonymous-namespace>`,
-   :doc:`misc-use-internal-linkage <misc/use-internal-linkage>`,
+   :doc:`misc-use-internal-linkage <misc/use-internal-linkage>`, "Yes"
    :doc:`modernize-avoid-bind <modernize/avoid-bind>`, "Yes"
    :doc:`modernize-avoid-c-arrays <modernize/avoid-c-arrays>`,
    :doc:`modernize-concat-nested-namespaces <modernize/concat-nested-namespaces>`, "Yes"
@@ -409,6 +409,7 @@ Check aliases
    :doc:`bugprone-narrowing-conversions <bugprone/narrowing-conversions>`, :doc:`cppcoreguidelines-narrowing-conversions <cppcoreguidelines/narrowing-conversions>`,
    :doc:`cert-con36-c <cert/con36-c>`, :doc:`bugprone-spuriously-wake-up-functions <bugprone/spuriously-wake-up-functions>`,
    :doc:`cert-con54-cpp <cert/con54-cpp>`, :doc:`bugprone-spuriously-wake-up-functions <bugprone/spuriously-wake-up-functions>`,
+   :doc:`cert-ctr56-cpp <cert/ctr56-cpp>`, :doc:`bugprone-pointer-arithmetic-on-polymorphic-object <bugprone/pointer-arithmetic-on-polymorphic-object>`,
    :doc:`cert-dcl03-c <cert/dcl03-c>`, :doc:`misc-static-assert <misc/static-assert>`, "Yes"
    :doc:`cert-dcl16-c <cert/dcl16-c>`, :doc:`readability-uppercase-literal-suffix <readability/uppercase-literal-suffix>`, "Yes"
    :doc:`cert-dcl37-c <cert/dcl37-c>`, :doc:`bugprone-reserved-identifier <bugprone/reserved-identifier>`, "Yes"
@@ -448,7 +449,7 @@ Check aliases
    :doc:`clang-analyzer-core.uninitialized.UndefReturn <clang-analyzer/core.uninitialized.UndefReturn>`, `Clang Static Analyzer core.uninitialized.UndefReturn <https://clang.llvm.org/docs/analyzer/checkers.html#core-uninitialized-undefreturn>`_,
    :doc:`clang-analyzer-cplusplus.ArrayDelete <clang-analyzer/cplusplus.ArrayDelete>`, `Clang Static Analyzer cplusplus.ArrayDelete <https://clang.llvm.org/docs/analyzer/checkers.html#cplusplus-arraydelete>`_,
    :doc:`clang-analyzer-cplusplus.InnerPointer <clang-analyzer/cplusplus.InnerPointer>`, `Clang Static Analyzer cplusplus.InnerPointer <https://clang.llvm.org/docs/analyzer/checkers.html#cplusplus-innerpointer>`_,
-   :doc:`clang-analyzer-cplusplus.Move <clang-analyzer/cplusplus.Move>`, Clang Static Analyzer cplusplus.Move,
+   :doc:`clang-analyzer-cplusplus.Move <clang-analyzer/cplusplus.Move>`, `Clang Static Analyzer cplusplus.Move <https://clang.llvm.org/docs/analyzer/checkers.html#cplusplus-move>`_,
    :doc:`clang-analyzer-cplusplus.NewDelete <clang-analyzer/cplusplus.NewDelete>`, `Clang Static Analyzer cplusplus.NewDelete <https://clang.llvm.org/docs/analyzer/checkers.html#cplusplus-newdelete>`_,
    :doc:`clang-analyzer-cplusplus.NewDeleteLeaks <clang-analyzer/cplusplus.NewDeleteLeaks>`, `Clang Static Analyzer cplusplus.NewDeleteLeaks <https://clang.llvm.org/docs/analyzer/checkers.html#cplusplus-newdeleteleaks>`_,
    :doc:`clang-analyzer-cplusplus.PlacementNew <clang-analyzer/cplusplus.PlacementNew>`, `Clang Static Analyzer cplusplus.PlacementNew <https://clang.llvm.org/docs/analyzer/checkers.html#cplusplus-placementnew>`_,
@@ -471,6 +472,7 @@ Check aliases
    :doc:`clang-analyzer-optin.performance.GCDAntipattern <clang-analyzer/optin.performance.GCDAntipattern>`, `Clang Static Analyzer optin.performance.GCDAntipattern <https://clang.llvm.org/docs/analyzer/checkers.html#optin-performance-gcdantipattern>`_,
    :doc:`clang-analyzer-optin.performance.Padding <clang-analyzer/optin.performance.Padding>`, `Clang Static Analyzer optin.performance.Padding <https://clang.llvm.org/docs/analyzer/checkers.html#optin-performance-padding>`_,
    :doc:`clang-analyzer-optin.portability.UnixAPI <clang-analyzer/optin.portability.UnixAPI>`, `Clang Static Analyzer optin.portability.UnixAPI <https://clang.llvm.org/docs/analyzer/checkers.html#optin-portability-unixapi>`_,
+   :doc:`clang-analyzer-optin.taint.TaintedAlloc <clang-analyzer/optin.taint.TaintedAlloc>`, `Clang Static Analyzer optin.taint.TaintedAlloc <https://clang.llvm.org/docs/analyzer/checkers.html#optin-taint-taintedalloc>`_,
    :doc:`clang-analyzer-osx.API <clang-analyzer/osx.API>`, `Clang Static Analyzer osx.API <https://clang.llvm.org/docs/analyzer/checkers.html#osx-api>`_,
    :doc:`clang-analyzer-osx.MIG <clang-analyzer/osx.MIG>`, Clang Static Analyzer osx.MIG,
    :doc:`clang-analyzer-osx.NumberObjectConversion <clang-analyzer/osx.NumberObjectConversion>`, `Clang Static Analyzer osx.NumberObjectConversion <https://clang.llvm.org/docs/analyzer/checkers.html#osx-numberobjectconversion>`_,
@@ -501,6 +503,7 @@ Check aliases
    :doc:`clang-analyzer-osx.coreFoundation.containers.OutOfBounds <clang-analyzer/osx.coreFoundation.containers.OutOfBounds>`, `Clang Static Analyzer osx.coreFoundation.containers.OutOfBounds <https://clang.llvm.org/docs/analyzer/checkers.html#osx-corefoundation-containers-outofbounds>`_,
    :doc:`clang-analyzer-osx.coreFoundation.containers.PointerSizedValues <clang-analyzer/osx.coreFoundation.containers.PointerSizedValues>`, `Clang Static Analyzer osx.coreFoundation.containers.PointerSizedValues <https://clang.llvm.org/docs/analyzer/checkers.html#osx-corefoundation-containers-pointersizedvalues>`_,
    :doc:`clang-analyzer-security.FloatLoopCounter <clang-analyzer/security.FloatLoopCounter>`, `Clang Static Analyzer security.FloatLoopCounter <https://clang.llvm.org/docs/analyzer/checkers.html#security-floatloopcounter>`_,
+   :doc:`clang-analyzer-security.PutenvStackArray <clang-analyzer/security.PutenvStackArray>`, Clang Static Analyzer security.PutenvStackArray,
    :doc:`clang-analyzer-security.SetgidSetuidOrder <clang-analyzer/security.SetgidSetuidOrder>`, Clang Static Analyzer security.SetgidSetuidOrder,
    :doc:`clang-analyzer-security.cert.env.InvalidPtr <clang-analyzer/security.cert.env.InvalidPtr>`, `Clang Static Analyzer security.cert.env.InvalidPtr <https://clang.llvm.org/docs/analyzer/checkers.html#security-cert-env-invalidptr>`_,
    :doc:`clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling <clang-analyzer/security.insecureAPI.DeprecatedOrUnsafeBufferHandling>`, `Clang Static Analyzer security.insecureAPI.DeprecatedOrUnsafeBufferHandling <https://clang.llvm.org/docs/analyzer/checkers.html#security-insecureapi-deprecatedorunsafebufferhandling>`_,
@@ -517,6 +520,7 @@ Check aliases
    :doc:`clang-analyzer-security.insecureAPI.strcpy <clang-analyzer/security.insecureAPI.strcpy>`, `Clang Static Analyzer security.insecureAPI.strcpy <https://clang.llvm.org/docs/analyzer/checkers.html#security-insecureapi-strcpy>`_,
    :doc:`clang-analyzer-security.insecureAPI.vfork <clang-analyzer/security.insecureAPI.vfork>`, `Clang Static Analyzer security.insecureAPI.vfork <https://clang.llvm.org/docs/analyzer/checkers.html#security-insecureapi-vfork>`_,
    :doc:`clang-analyzer-unix.API <clang-analyzer/unix.API>`, `Clang Static Analyzer unix.API <https://clang.llvm.org/docs/analyzer/checkers.html#unix-api>`_,
+   :doc:`clang-analyzer-unix.BlockInCriticalSection <clang-analyzer/unix.BlockInCriticalSection>`, `Clang Static Analyzer unix.BlockInCriticalSection <https://clang.llvm.org/docs/analyzer/checkers.html#unix-blockincriticalsection>`_,
    :doc:`clang-analyzer-unix.Errno <clang-analyzer/unix.Errno>`, `Clang Static Analyzer unix.Errno <https://clang.llvm.org/docs/analyzer/checkers.html#unix-errno>`_,
    :doc:`clang-analyzer-unix.Malloc <clang-analyzer/unix.Malloc>`, `Clang Static Analyzer unix.Malloc <https://clang.llvm.org/docs/analyzer/checkers.html#unix-malloc>`_,
    :doc:`clang-analyzer-unix.MallocSizeof <clang-analyzer/unix.MallocSizeof>`, `Clang Static Analyzer unix.MallocSizeof <https://clang.llvm.org/docs/analyzer/checkers.html#unix-mallocsizeof>`_,

From bdee9b05ded9c5b90556f3a8b22f75e204e54a46 Mon Sep 17 00:00:00 2001
From: Sayhaan Siddiqui <49014204+sayhaan@users.noreply.github.com>
Date: Mon, 22 Jul 2024 12:31:51 -0700
Subject: [PATCH 473/486] Revert "[BOLT][DWARF][NFC] Split processUnitDIE into
 two lambdas" (#99904)

Reverts llvm/llvm-project#99225
---
 bolt/lib/Rewrite/DWARFRewriter.cpp | 144 ++++++++++++++---------------
 1 file changed, 67 insertions(+), 77 deletions(-)

diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp
index ccb45f40c5c7a..1ec216b39e95c 100644
--- a/bolt/lib/Rewrite/DWARFRewriter.cpp
+++ b/bolt/lib/Rewrite/DWARFRewriter.cpp
@@ -620,10 +620,9 @@ void DWARFRewriter::updateDebugInfo() {
   uint32_t CUIndex = 0;
   std::mutex AccessMutex;
   // Needs to be invoked in the same order as CUs are processed.
-  llvm::DenseMap<uint64_t, uint64_t> LocListWritersIndexByCU;
-  auto createRangeLocListAddressWriters = [&](DWARFUnit &CU) {
+  auto createRangeLocListAddressWriters =
+      [&](DWARFUnit &CU) -> DebugLocWriter * {
     std::lock_guard<std::mutex> Lock(AccessMutex);
-
     const uint16_t DwarfVersion = CU.getVersion();
     if (DwarfVersion >= 5) {
       auto AddrW = std::make_unique<DebugAddrWriterDwarf5>(
@@ -642,6 +641,7 @@ void DWARFRewriter::updateDebugInfo() {
         RangeListsWritersByCU[*DWOId] = std::move(DWORangeListsSectionWriter);
       }
       AddressWritersByCU[CU.getOffset()] = std::move(AddrW);
+
     } else {
       auto AddrW =
           std::make_unique<DebugAddrWriter>(&BC, CU.getAddressByteSize());
@@ -657,7 +657,7 @@ void DWARFRewriter::updateDebugInfo() {
             std::move(LegacyRangesSectionWriterByCU);
       }
     }
-    LocListWritersIndexByCU[CU.getOffset()] = CUIndex++;
+    return LocListWritersByCU[CUIndex++].get();
   };
 
   DWARF5AcceleratorTable DebugNamesTable(opts::CreateDebugNames, BC,
@@ -666,68 +666,74 @@ void DWARFRewriter::updateDebugInfo() {
   DWPState State;
   if (opts::WriteDWP)
     initDWPState(State);
-  auto processSplitCU = [&](DWARFUnit &Unit, DWARFUnit &SplitCU,
-                            DIEBuilder &DIEBlder,
-                            DebugRangesSectionWriter &TempRangesSectionWriter,
-                            DebugAddrWriter &AddressWriter) {
-    DIEBuilder DWODIEBuilder(BC, &(SplitCU).getContext(), DebugNamesTable,
-                             &Unit);
-    DWODIEBuilder.buildDWOUnit(SplitCU);
-    std::string DWOName = "";
-    std::optional<std::string> DwarfOutputPath =
-        opts::DwarfOutputPath.empty()
-            ? std::nullopt
-            : std::optional<std::string>(opts::DwarfOutputPath.c_str());
-    {
-      std::lock_guard<std::mutex> Lock(AccessMutex);
-      DWOName = DIEBlder.updateDWONameCompDir(
-          *StrOffstsWriter, *StrWriter, Unit, DwarfOutputPath, std::nullopt);
-    }
-    DebugStrOffsetsWriter DWOStrOffstsWriter(BC);
-    DebugStrWriter DWOStrWriter((SplitCU).getContext(), true);
-    DWODIEBuilder.updateDWONameCompDirForTypes(
-        DWOStrOffstsWriter, DWOStrWriter, SplitCU, DwarfOutputPath, DWOName);
-    DebugLoclistWriter DebugLocDWoWriter(Unit, Unit.getVersion(), true,
-                                         AddressWriter);
-
-    updateUnitDebugInfo(SplitCU, DWODIEBuilder, DebugLocDWoWriter,
-                        TempRangesSectionWriter, AddressWriter);
-    DebugLocDWoWriter.finalize(DWODIEBuilder,
-                               *DWODIEBuilder.getUnitDIEbyUnit(SplitCU));
-    if (Unit.getVersion() >= 5)
-      TempRangesSectionWriter.finalizeSection();
-
-    emitDWOBuilder(DWOName, DWODIEBuilder, *this, SplitCU, Unit, State,
-                   DebugLocDWoWriter, DWOStrOffstsWriter, DWOStrWriter,
-                   GDBIndexSection);
-  };
-  auto processMainBinaryCU = [&](DWARFUnit &Unit, DIEBuilder &DIEBlder) {
-    DebugAddrWriter &AddressWriter =
-        *AddressWritersByCU[Unit.getOffset()].get();
-    DebugRangesSectionWriter &RangesSectionWriter =
-        Unit.getVersion() >= 5 ? *RangeListsSectionWriter.get()
-                               : *LegacyRangesSectionWriter.get();
-    DebugLocWriter &DebugLocWriter =
-        *LocListWritersByCU[LocListWritersIndexByCU[Unit.getOffset()]].get();
-    std::optional<uint64_t> RangesBase;
+  auto processUnitDIE = [&](DWARFUnit *Unit, DIEBuilder *DIEBlder) {
+    // Check if the unit is a skeleton and we need special updates for it and
+    // its matching split/DWO CU.
     std::optional<DWARFUnit *> SplitCU;
-    std::optional<uint64_t> DWOId = Unit.getDWOId();
+    std::optional<uint64_t> RangesBase;
+    std::optional<uint64_t> DWOId = Unit->getDWOId();
     if (DWOId)
       SplitCU = BC.getDWOCU(*DWOId);
-    if (Unit.getVersion() >= 5) {
-      RangesBase = RangesSectionWriter.getSectionOffset() +
+    DebugLocWriter *DebugLocWriter = createRangeLocListAddressWriters(*Unit);
+    DebugRangesSectionWriter *RangesSectionWriter =
+        Unit->getVersion() >= 5 ? RangeListsSectionWriter.get()
+                                : LegacyRangesSectionWriter.get();
+    DebugAddrWriter *AddressWriter =
+        AddressWritersByCU[Unit->getOffset()].get();
+    // Skipping CUs that failed to load.
+    if (SplitCU) {
+      DIEBuilder DWODIEBuilder(BC, &(*SplitCU)->getContext(), DebugNamesTable,
+                               Unit);
+      DWODIEBuilder.buildDWOUnit(**SplitCU);
+      std::string DWOName = "";
+      std::optional<std::string> DwarfOutputPath =
+          opts::DwarfOutputPath.empty()
+              ? std::nullopt
+              : std::optional<std::string>(opts::DwarfOutputPath.c_str());
+      {
+        std::lock_guard<std::mutex> Lock(AccessMutex);
+        DWOName = DIEBlder->updateDWONameCompDir(
+            *StrOffstsWriter, *StrWriter, *Unit, DwarfOutputPath, std::nullopt);
+      }
+      DebugStrOffsetsWriter DWOStrOffstsWriter(BC);
+      DebugStrWriter DWOStrWriter((*SplitCU)->getContext(), true);
+      DWODIEBuilder.updateDWONameCompDirForTypes(DWOStrOffstsWriter,
+                                                 DWOStrWriter, **SplitCU,
+                                                 DwarfOutputPath, DWOName);
+      DebugLoclistWriter DebugLocDWoWriter(*Unit, Unit->getVersion(), true,
+                                           *AddressWriter);
+      DebugRangesSectionWriter *TempRangesSectionWriter = RangesSectionWriter;
+      if (Unit->getVersion() >= 5) {
+        TempRangesSectionWriter = RangeListsWritersByCU[*DWOId].get();
+      } else {
+        TempRangesSectionWriter = LegacyRangesWritersByCU[*DWOId].get();
+        RangesBase = RangesSectionWriter->getSectionOffset();
+      }
+
+      updateUnitDebugInfo(*(*SplitCU), DWODIEBuilder, DebugLocDWoWriter,
+                          *TempRangesSectionWriter, *AddressWriter);
+      DebugLocDWoWriter.finalize(DWODIEBuilder,
+                                 *DWODIEBuilder.getUnitDIEbyUnit(**SplitCU));
+      if (Unit->getVersion() >= 5)
+        TempRangesSectionWriter->finalizeSection();
+
+      emitDWOBuilder(DWOName, DWODIEBuilder, *this, **SplitCU, *Unit, State,
+                     DebugLocDWoWriter, DWOStrOffstsWriter, DWOStrWriter,
+                     GDBIndexSection);
+    }
+
+    if (Unit->getVersion() >= 5) {
+      RangesBase = RangesSectionWriter->getSectionOffset() +
                    getDWARF5RngListLocListHeaderSize();
-      RangesSectionWriter.initSection(Unit);
-      StrOffstsWriter->finalizeSection(Unit, DIEBlder);
-    } else if (SplitCU) {
-      RangesBase = LegacyRangesSectionWriter.get()->getSectionOffset();
+      RangesSectionWriter->initSection(*Unit);
+      StrOffstsWriter->finalizeSection(*Unit, *DIEBlder);
     }
 
-    updateUnitDebugInfo(Unit, DIEBlder, DebugLocWriter, RangesSectionWriter,
-                        AddressWriter, RangesBase);
-    DebugLocWriter.finalize(DIEBlder, *DIEBlder.getUnitDIEbyUnit(Unit));
-    if (Unit.getVersion() >= 5)
-      RangesSectionWriter.finalizeSection();
+    updateUnitDebugInfo(*Unit, *DIEBlder, *DebugLocWriter, *RangesSectionWriter,
+                        *AddressWriter, RangesBase);
+    DebugLocWriter->finalize(*DIEBlder, *DIEBlder->getUnitDIEbyUnit(*Unit));
+    if (Unit->getVersion() >= 5)
+      RangesSectionWriter->finalizeSection();
   };
 
   DIEBuilder DIEBlder(BC, BC.DwCtx.get(), DebugNamesTable);
@@ -745,24 +751,8 @@ void DWARFRewriter::updateDebugInfo() {
   CUPartitionVector PartVec = partitionCUs(*BC.DwCtx);
   for (std::vector<DWARFUnit *> &Vec : PartVec) {
     DIEBlder.buildCompileUnits(Vec);
-    for (DWARFUnit *CU : DIEBlder.getProcessedCUs()) {
-      createRangeLocListAddressWriters(*CU);
-      std::optional<DWARFUnit *> SplitCU;
-      std::optional<uint64_t> DWOId = CU->getDWOId();
-      if (DWOId)
-        SplitCU = BC.getDWOCU(*DWOId);
-      if (!SplitCU)
-        continue;
-      DebugAddrWriter &AddressWriter =
-          *AddressWritersByCU[CU->getOffset()].get();
-      DebugRangesSectionWriter *TempRangesSectionWriter =
-          CU->getVersion() >= 5 ? RangeListsWritersByCU[*DWOId].get()
-                                : LegacyRangesWritersByCU[*DWOId].get();
-      processSplitCU(*CU, **SplitCU, DIEBlder, *TempRangesSectionWriter,
-                     AddressWriter);
-    }
     for (DWARFUnit *CU : DIEBlder.getProcessedCUs())
-      processMainBinaryCU(*CU, DIEBlder);
+      processUnitDIE(CU, &DIEBlder);
     finalizeCompileUnits(DIEBlder, *Streamer, OffsetMap,
                          DIEBlder.getProcessedCUs(), *FinalAddrWriter);
   }

From e6fdecd29059c0951a4970e9e6cf478972844099 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 22 Jul 2024 20:32:58 +0100
Subject: [PATCH 474/486] [VPlan] Drop references to Ingredient from
 VPWidenRecipe comments (NFC)

VPWidenRecipe has been updated to use Opcode + operands instead of an
Instruction 'ingredient'. Reword the comments.
---
 llvm/lib/Transforms/Vectorize/VPlan.h | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 805d9d91fc186..0b596e7e4f633 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1399,9 +1399,10 @@ class VPInstruction : public VPRecipeWithIRFlags {
   bool isSingleScalar() const;
 };
 
-/// VPWidenRecipe is a recipe for producing a copy of vector type its
-/// ingredient. This recipe covers most of the traditional vectorization cases
-/// where each ingredient transforms into a vectorized version of itself.
+/// VPWidenRecipe is a recipe for producing a widened instruction using the
+/// opcode and operands of the recipe. This recipe covers most of the
+/// traditional vectorization cases where each recipe transforms into a
+/// vectorized version of itself.
 class VPWidenRecipe : public VPRecipeWithIRFlags {
   unsigned Opcode;
 
@@ -1421,7 +1422,8 @@ class VPWidenRecipe : public VPRecipeWithIRFlags {
 
   VP_CLASSOF_IMPL(VPDef::VPWidenSC)
 
-  /// Produce widened copies of all Ingredients.
+  /// Produce a widened instruction using the opcode and operands of the recipe,
+  /// processing State.VF elements.
   void execute(VPTransformState &State) override;
 
   unsigned getOpcode() const { return Opcode; }

From e09032f7a36ffb5eb8638a3933aeca7015a9579a Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 22 Jul 2024 20:45:18 +0100
Subject: [PATCH 475/486] [StackFrameLayoutAnalysis] Add basic Scalable stack
 slot output (#99883)

The existing StackFrameLayoutAnalysis details do not do well with
Scalable vector stack slots, which are not marked as scalable and
intertwined with the other fixed-size slots. This patch adds some very
basic support, marking them as scalable and sorting them to the end of
the list. The slot addresses are not really correct (for fixed as well
as scalable), but this prints something a little better with the limited
information curently available.
---
 .../lib/CodeGen/StackFrameLayoutAnalysisPass.cpp | 16 +++++++++++-----
 .../CodeGen/AArch64/sve-stack-frame-layout.ll    |  8 ++++----
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp b/llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp
index 5d3903ed84ce8..940aecd1cb363 100644
--- a/llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp
+++ b/llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp
@@ -62,11 +62,14 @@ struct StackFrameLayoutAnalysisPass : public MachineFunctionPass {
     int Align;
     int Offset;
     SlotType SlotTy;
+    bool Scalable;
 
     SlotData(const MachineFrameInfo &MFI, const int ValOffset, const int Idx)
         : Slot(Idx), Size(MFI.getObjectSize(Idx)),
           Align(MFI.getObjectAlign(Idx).value()),
-          Offset(MFI.getObjectOffset(Idx) - ValOffset), SlotTy(Invalid) {
+          Offset(MFI.getObjectOffset(Idx) - ValOffset), SlotTy(Invalid),
+          Scalable(false) {
+      Scalable = MFI.getStackID(Idx) == TargetStackID::ScalableVector;
       if (MFI.isSpillSlotObjectIndex(Idx))
         SlotTy = SlotType::Spill;
       else if (Idx == MFI.getStackProtectorIndex())
@@ -75,9 +78,12 @@ struct StackFrameLayoutAnalysisPass : public MachineFunctionPass {
         SlotTy = SlotType::Variable;
     }
 
-    // we use this to sort in reverse order, so that the layout is displayed
-    // correctly
-    bool operator<(const SlotData &Rhs) const { return Offset > Rhs.Offset; }
+    // We use this to sort in reverse order, so that the layout is displayed
+    // correctly. Scalable slots are sorted to the end of the list.
+    bool operator<(const SlotData &Rhs) const {
+      return std::make_tuple(!Scalable, Offset) >
+             std::make_tuple(!Rhs.Scalable, Rhs.Offset);
+    }
   };
 
   StackFrameLayoutAnalysisPass() : MachineFunctionPass(ID) {}
@@ -153,7 +159,7 @@ struct StackFrameLayoutAnalysisPass : public MachineFunctionPass {
     Rem << Prefix << ore::NV("Offset", D.Offset)
         << "], Type: " << ore::NV("Type", getTypeString(D.SlotTy))
         << ", Align: " << ore::NV("Align", D.Align)
-        << ", Size: " << ore::NV("Size", D.Size);
+        << ", Size: " << ore::NV("Size", ElementCount::get(D.Size, D.Scalable));
   }
 
   void emitSourceLocRemark(const MachineFunction &MF, const DILocalVariable *N,
diff --git a/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll b/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll
index 67c1486b7a8c0..34d85d1f76086 100644
--- a/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll
+++ b/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll
@@ -4,10 +4,10 @@
 
 ; CHECK-FRAMELAYOUT-LABEL: Function: csr_d8_allocnxv4i32i32f64
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-8], Type: Spill, Align: 8, Size: 8
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Variable, Align: 16, Size: 16
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 8, Size: 8
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-20], Type: Variable, Align: 4, Size: 4
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Variable, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Variable, Align: 16, Size: vscale x 16
 
 define i32 @csr_d8_allocnxv4i32i32f64(double %d) "aarch64_pstate_sm_compatible" {
 ; CHECK-LABEL: csr_d8_allocnxv4i32i32f64:
@@ -46,11 +46,11 @@ entry:
 
 ; CHECK-FRAMELAYOUT-LABEL: Function: csr_d8_allocnxv4i32i32f64_fp
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-8], Type: Spill, Align: 8, Size: 8
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Variable, Align: 16, Size: 16
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 8, Size: 8
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-20], Type: Variable, Align: 4, Size: 4
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Spill, Align: 16, Size: 8
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-40], Type: Variable, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Variable, Align: 16, Size: vscale x 16
 
 define i32 @csr_d8_allocnxv4i32i32f64_fp(double %d) "aarch64_pstate_sm_compatible" "frame-pointer"="all" {
 ; CHECK-LABEL: csr_d8_allocnxv4i32i32f64_fp:
@@ -92,11 +92,11 @@ entry:
 
 ; CHECK-FRAMELAYOUT-LABEL: Function: svecc_z8_allocnxv4i32i32f64_fp
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-8], Type: Spill, Align: 8, Size: 8
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 16, Size: 16
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 8, Size: 8
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-20], Type: Variable, Align: 4, Size: 4
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Variable, Align: 16, Size: 16
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Variable, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Variable, Align: 16, Size: vscale x 16
 
 define i32 @svecc_z8_allocnxv4i32i32f64_fp(double %d, <vscale x 4 x i32> %v) "aarch64_pstate_sm_compatible" "frame-pointer"="all" {
 ; CHECK-LABEL: svecc_z8_allocnxv4i32i32f64_fp:

From 9a8b0407fc16af4ca6f79a2583297318a645d88a Mon Sep 17 00:00:00 2001
From: Daniel Bertalan <dani@danielbertalan.dev>
Date: Mon, 22 Jul 2024 22:03:32 +0200
Subject: [PATCH 476/486] Reapply "[lld] enable fixup chains by default
 (#79894)" (#99255)

This reverts commit f55b79f59a77b4be586d649e9ced9f8667265011.

The known issues with chained fixups have been addressed by #98913,
#98305, #97156 and #95171.

Compared to the original commit, support for xrOS (which postdates
chained fixups' introduction) was added and an unnecessary test change
was removed.

----------
Original commit message:

Enable chained fixups in lld when all platform and version criteria are
met. This is an attempt at simplifying the logic used in ld 907:


https://github.com/apple-oss-distributions/ld64/blob/93d74eafc37c0558b4ffb88a8bc15c17bed44a20/src/ld/Options.cpp#L5458-L5549

Some changes were made to simplify the logic:
- only enable chained fixups for macOS from 13.0 to avoid the arch check
- only enable chained fixups for iphonesimulator from 16.0 to avoid the
arch check
- don't enable chained fixups for not specifically listed platforms
- don't enable chained fixups for arm64_32
---
 lld/MachO/Driver.cpp                          | 64 +++++++++++--------
 lld/docs/ReleaseNotes.rst                     |  5 ++
 lld/test/MachO/arm64-32-stubs.s               |  2 +-
 lld/test/MachO/arm64-stubs.s                  |  2 +-
 lld/test/MachO/dyld-stub-binder.s             | 14 ++--
 .../invalid/chained-fixups-incompatible.s     |  4 --
 lld/test/MachO/objc-selrefs.s                 |  7 +-
 7 files changed, 57 insertions(+), 41 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index dc9d635b48ec4..1883acc781ece 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -49,6 +49,7 @@
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/TimeProfiler.h"
 #include "llvm/TargetParser/Host.h"
+#include "llvm/TextAPI/Architecture.h"
 #include "llvm/TextAPI/PackedVersion.h"
 
 #include <algorithm>
@@ -1080,42 +1081,55 @@ static bool shouldEmitChainedFixups(const InputArgList &args) {
   if (arg && arg->getOption().matches(OPT_no_fixup_chains))
     return false;
 
-  bool isRequested = arg != nullptr;
-
-  // Version numbers taken from the Xcode 13.3 release notes.
-  static const std::array<std::pair<PlatformType, VersionTuple>, 5> minVersion =
-      {{{PLATFORM_MACOS, VersionTuple(11, 0)},
-        {PLATFORM_IOS, VersionTuple(13, 4)},
-        {PLATFORM_TVOS, VersionTuple(14, 0)},
-        {PLATFORM_WATCHOS, VersionTuple(7, 0)},
-        {PLATFORM_XROS, VersionTuple(1, 0)}}};
-  PlatformType platform = removeSimulator(config->platformInfo.target.Platform);
-  auto it = llvm::find_if(minVersion,
-                          [&](const auto &p) { return p.first == platform; });
-  if (it != minVersion.end() &&
-      it->second > config->platformInfo.target.MinDeployment) {
-    if (!isRequested)
-      return false;
+  bool requested = arg && arg->getOption().matches(OPT_fixup_chains);
+  if (!config->isPic) {
+    if (requested)
+      error("-fixup_chains is incompatible with -no_pie");
 
-    warn("-fixup_chains requires " + getPlatformName(config->platform()) + " " +
-         it->second.getAsString() + ", which is newer than target minimum of " +
-         config->platformInfo.target.MinDeployment.getAsString());
+    return false;
   }
 
   if (!is_contained({AK_x86_64, AK_x86_64h, AK_arm64}, config->arch())) {
-    if (isRequested)
+    if (requested)
       error("-fixup_chains is only supported on x86_64 and arm64 targets");
+
     return false;
   }
 
-  if (!config->isPic) {
-    if (isRequested)
-      error("-fixup_chains is incompatible with -no_pie");
+  if (args.hasArg(OPT_preload)) {
+    if (requested)
+      error("-fixup_chains is incompatible with -preload");
+
     return false;
   }
 
-  // TODO: Enable by default once stable.
-  return isRequested;
+  if (requested)
+    return true;
+
+  static const std::array<std::pair<PlatformType, VersionTuple>, 9> minVersion =
+      {{
+          {PLATFORM_IOS, VersionTuple(13, 4)},
+          {PLATFORM_IOSSIMULATOR, VersionTuple(16, 0)},
+          {PLATFORM_MACOS, VersionTuple(13, 0)},
+          {PLATFORM_TVOS, VersionTuple(14, 0)},
+          {PLATFORM_TVOSSIMULATOR, VersionTuple(15, 0)},
+          {PLATFORM_WATCHOS, VersionTuple(7, 0)},
+          {PLATFORM_WATCHOSSIMULATOR, VersionTuple(8, 0)},
+          {PLATFORM_XROS, VersionTuple(1, 0)},
+          {PLATFORM_XROS_SIMULATOR, VersionTuple(1, 0)},
+      }};
+  PlatformType platform = config->platformInfo.target.Platform;
+  auto it = llvm::find_if(minVersion,
+                          [&](const auto &p) { return p.first == platform; });
+
+  // We don't know the versions for other platforms, so default to disabled.
+  if (it == minVersion.end())
+    return false;
+
+  if (it->second > config->platformInfo.target.MinDeployment)
+    return false;
+
+  return true;
 }
 
 static bool shouldEmitRelativeMethodLists(const InputArgList &args) {
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index ba8543732bb8e..09081e421e905 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -102,6 +102,11 @@ MinGW Improvements
 MachO Improvements
 ------------------
 
+* Chained fixups are now enabled by default when targeting macOS 13.0,
+  iOS 13.4, tvOS 14.0, watchOS 7.0, and visionOS 1.0 or later.
+  They can be disabled with the `-no_fixup_chains` flag.
+  (`#99255 <https://github.com/llvm/llvm-project/pull/99255>`_)
+
 WebAssembly Improvements
 ------------------------
 
diff --git a/lld/test/MachO/arm64-32-stubs.s b/lld/test/MachO/arm64-32-stubs.s
index 743d5c419bf1a..a5d48ff5baef3 100644
--- a/lld/test/MachO/arm64-32-stubs.s
+++ b/lld/test/MachO/arm64-32-stubs.s
@@ -10,7 +10,7 @@
 # RUN: llvm-mc -filetype=obj -triple=arm64_32-apple-watchos %t/test.s -o %t/test.o
 # RUN: %lld-watchos -dylib -install_name @executable_path/libfoo.dylib %t/foo.o -o %t/libfoo.dylib
 # RUN: %lld-watchos -dylib -install_name @executable_path/libbar.dylib %t/bar.o -o %t/libbar.dylib
-# RUN: %lld-watchos -lSystem %t/libfoo.dylib %t/libbar.dylib %t/test.o -o %t/test
+# RUN: %lld-watchos -lSystem %t/libfoo.dylib %t/libbar.dylib %t/test.o -o %t/test -no_fixup_chains
 
 # RUN: llvm-objdump --no-print-imm-hex --macho -d --no-show-raw-insn --section="__TEXT,__stubs" --section="__TEXT,__stub_helper" %t/test | FileCheck %s
 
diff --git a/lld/test/MachO/arm64-stubs.s b/lld/test/MachO/arm64-stubs.s
index f714e20084895..6fd94661d32e2 100644
--- a/lld/test/MachO/arm64-stubs.s
+++ b/lld/test/MachO/arm64-stubs.s
@@ -5,7 +5,7 @@
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/test.s -o %t/test.o
 # RUN: %lld -arch arm64 -dylib -install_name @executable_path/libfoo.dylib %t/foo.o -o %t/libfoo.dylib
 # RUN: %lld -arch arm64 -dylib -install_name @executable_path/libbar.dylib %t/bar.o -o %t/libbar.dylib
-# RUN: %lld -arch arm64 -lSystem %t/libfoo.dylib %t/libbar.dylib %t/test.o -o %t/test
+# RUN: %lld -arch arm64 -lSystem %t/libfoo.dylib %t/libbar.dylib %t/test.o -o %t/test -no_fixup_chains
 
 # RUN: llvm-objdump --no-print-imm-hex --macho -d --no-show-raw-insn --section="__TEXT,__stubs" --section="__TEXT,__stub_helper" %t/test | FileCheck %s
 
diff --git a/lld/test/MachO/dyld-stub-binder.s b/lld/test/MachO/dyld-stub-binder.s
index 1d2e556607c79..170fe8abd89bd 100644
--- a/lld/test/MachO/dyld-stub-binder.s
+++ b/lld/test/MachO/dyld-stub-binder.s
@@ -14,27 +14,27 @@
 # RUN: %lld -arch arm64 -lSystem -dylib %t/foo.o -o %t/libfoo.dylib
 # RUN: llvm-nm -m %t/libfoo.dylib | FileCheck --check-prefix=STUB %s
 
-
 ## Dylibs that do lazy dynamic calls do need dyld_stub_binder.
 # RUN: not %no-lsystem-lld -arch arm64 -dylib %t/bar.o %t/libfoo.dylib \
-# RUN:     -o %t/libbar.dylib 2>&1 | FileCheck --check-prefix=MISSINGSTUB %s
+# RUN:     -o %t/libbar.dylib -no_fixup_chains 2>&1 | \
+# RUN:     FileCheck --check-prefix=MISSINGSTUB %s
 # RUN: %lld -arch arm64 -lSystem -dylib %t/bar.o  %t/libfoo.dylib \
-# RUN:     -o %t/libbar.dylib
+# RUN:     -o %t/libbar.dylib -no_fixup_chains
 # RUN: llvm-nm -m %t/libbar.dylib | FileCheck --check-prefix=STUB %s
 
 ## As do executables.
 # RUN: not %no-lsystem-lld -arch arm64 %t/libfoo.dylib %t/libbar.dylib %t/test.o \
-# RUN:     -o %t/test 2>&1 | FileCheck --check-prefix=MISSINGSTUB %s
+# RUN:     -o %t/test -no_fixup_chains 2>&1 | FileCheck --check-prefix=MISSINGSTUB %s
 # RUN: %lld -arch arm64 -lSystem %t/libfoo.dylib %t/libbar.dylib %t/test.o \
-# RUN:     -o %t/test
+# RUN:     -o %t/test -no_fixup_chains
 # RUN: llvm-nm -m %t/test | FileCheck --check-prefix=STUB %s
 
 ## Test dynamic lookup of dyld_stub_binder.
 # RUN: %no-lsystem-lld -arch arm64 %t/libfoo.dylib %t/libbar.dylib %t/test.o \
-# RUN:     -o %t/test -undefined dynamic_lookup
+# RUN:     -o %t/test -undefined dynamic_lookup -no_fixup_chains
 # RUN: llvm-nm -m %t/test | FileCheck --check-prefix=DYNSTUB %s
 # RUN: %no-lsystem-lld -arch arm64 %t/libfoo.dylib %t/libbar.dylib %t/test.o \
-# RUN:     -o %t/test -U dyld_stub_binder
+# RUN:     -o %t/test -U dyld_stub_binder -no_fixup_chains
 # RUN: llvm-nm -m %t/test | FileCheck --check-prefix=DYNSTUB %s
 
 # MISSINGSTUB:      error: undefined symbol: dyld_stub_binder
diff --git a/lld/test/MachO/invalid/chained-fixups-incompatible.s b/lld/test/MachO/invalid/chained-fixups-incompatible.s
index 83411472beb9f..0589204968fe3 100644
--- a/lld/test/MachO/invalid/chained-fixups-incompatible.s
+++ b/lld/test/MachO/invalid/chained-fixups-incompatible.s
@@ -2,9 +2,6 @@
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
 # RUN: not %lld -lSystem -no_pie -fixup_chains %t.o -o /dev/null 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=NO-PIE
-# RUN: %no-fatal-warnings-lld -fixup_chains %t.o -o /dev/null \
-# RUN:   -lSystem -platform_version macos 10.15 10.15 2>&1 | \
-# RUN:   FileCheck %s --check-prefix=VERSION
 # RUN: llvm-mc -filetype=obj -triple=arm64_32-apple-darwin %s -o %t-arm64_32.o
 # RUN: not %lld-watchos -lSystem -fixup_chains %t-arm64_32.o -o /dev/null 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=ARCH
@@ -12,7 +9,6 @@
 ## Check that we emit diagnostics when -fixup_chains is explicitly specified,
 ## but we don't support creating chained fixups for said configuration.
 # NO-PIE:  error: -fixup_chains is incompatible with -no_pie
-# VERSION: warning: -fixup_chains requires macOS 11.0, which is newer than target minimum of 10.15
 # ARCH:    error: -fixup_chains is only supported on x86_64 and arm64 targets
 
 .globl _main
diff --git a/lld/test/MachO/objc-selrefs.s b/lld/test/MachO/objc-selrefs.s
index 6d144f4938b45..eebe7c6476cd5 100644
--- a/lld/test/MachO/objc-selrefs.s
+++ b/lld/test/MachO/objc-selrefs.s
@@ -6,14 +6,14 @@
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/implicit-selrefs.s -o %t/implicit-selrefs.o
 
 # RUN: %lld -dylib -arch arm64 -lSystem -o %t/explicit-only-no-icf \
-# RUN:   %t/explicit-selrefs-1.o %t/explicit-selrefs-2.o
+# RUN:   %t/explicit-selrefs-1.o %t/explicit-selrefs-2.o -no_fixup_chains
 # RUN: llvm-otool -vs __DATA __objc_selrefs %t/explicit-only-no-icf | \
 # RUN:   FileCheck %s --check-prefix=EXPLICIT-NO-ICF
 
 ## NOTE: ld64 always dedups the selrefs unconditionally, but we only do it when
 ## ICF is enabled.
 # RUN: %lld -dylib -arch arm64 -lSystem -o %t/explicit-only-with-icf \
-# RUN:   %t/explicit-selrefs-1.o %t/explicit-selrefs-2.o
+# RUN:   %t/explicit-selrefs-1.o %t/explicit-selrefs-2.o -no_fixup_chains
 # RUN: llvm-otool -vs __DATA __objc_selrefs %t/explicit-only-with-icf \
 # RUN:   | FileCheck %s --check-prefix=EXPLICIT-WITH-ICF
 
@@ -25,7 +25,8 @@
 # SELREFS-EMPTY:
 
 # RUN: %lld -dylib -arch arm64 -lSystem --icf=all -o %t/explicit-and-implicit \
-# RUN:   %t/explicit-selrefs-1.o %t/explicit-selrefs-2.o %t/implicit-selrefs.o
+# RUN:   %t/explicit-selrefs-1.o %t/explicit-selrefs-2.o %t/implicit-selrefs.o \
+# RUN:   -no_fixup_chains
 # RUN: llvm-otool -vs __DATA __objc_selrefs %t/explicit-and-implicit \
 # RUN:   | FileCheck %s --check-prefix=EXPLICIT-AND-IMPLICIT
 

From 3cb82f49dc990dc20a765856c0e126193992fe44 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Mon, 22 Jul 2024 12:45:28 -0700
Subject: [PATCH 477/486] [SLP]Fix PR99899: Use canonical type instead of
 original vector of ptr.

Use adjusted canonical integer type instead of the original ptr type to
fix the crash in the TTI.
Fixes https://github.com/llvm/llvm-project/issues/99899
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  3 +-
 .../SLPVectorizer/SystemZ/cmp-ptr-minmax.ll   | 39 +++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/SystemZ/cmp-ptr-minmax.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 667c4eb311c22..cca9eeebaa53f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9699,7 +9699,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
           CanonicalType = CanonicalType->getWithNewType(IntegerType::get(
               CanonicalType->getContext(),
               DL->getTypeSizeInBits(CanonicalType->getScalarType())));
-        IntrinsicCostAttributes CostAttrs(MinMaxID, VecTy, {VecTy, VecTy});
+        IntrinsicCostAttributes CostAttrs(MinMaxID, CanonicalType,
+                                          {CanonicalType, CanonicalType});
         InstructionCost IntrinsicCost =
             TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
         // If the selects are the only uses of the compares, they will be
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/cmp-ptr-minmax.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/cmp-ptr-minmax.ll
new file mode 100644
index 0000000000000..81837bdf99eaf
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/cmp-ptr-minmax.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=s390x-unknown-linux-gnu -mcpu=z16 -slp-threshold=-10 < %s | FileCheck %s
+
+define i1 @test(i64 %0, i64 %1, ptr %2) {
+; CHECK-LABEL: define i1 @test(
+; CHECK-SAME: i64 [[TMP0:%.*]], i64 [[TMP1:%.*]], ptr [[TMP2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[GEP44:%.*]] = getelementptr i8, ptr null, i64 [[TMP0]]
+; CHECK-NEXT:    [[GEP45:%.*]] = getelementptr i8, ptr null, i64 [[TMP1]]
+; CHECK-NEXT:    [[GEP48:%.*]] = getelementptr i8, ptr null, i64 [[TMP0]]
+; CHECK-NEXT:    [[GEP49:%.*]] = getelementptr i8, ptr null, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x ptr> poison, ptr [[GEP44]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x ptr> [[TMP3]], ptr [[GEP48]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x ptr> poison, ptr [[GEP45]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x ptr> [[TMP5]], ptr [[GEP49]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult <2 x ptr> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP7]], <2 x ptr> [[TMP4]], <2 x ptr> [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x ptr> [[TMP9]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult <2 x ptr> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i1> [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP11]], i32 1
+; CHECK-NEXT:    [[RES:%.*]] = and i1 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+entry:
+  %gep44 = getelementptr i8, ptr null, i64 %0
+  %gep45 = getelementptr i8, ptr null, i64 %1
+  %4 = icmp ult ptr %gep44, %gep45
+  %umin = select i1 %4, ptr %gep44, ptr %gep45
+  %gep48 = getelementptr i8, ptr null, i64 %0
+  %gep49 = getelementptr i8, ptr null, i64 %1
+  %5 = icmp ult ptr %gep48, %gep49
+  %umin50 = select i1 %5, ptr %gep48, ptr %gep49
+  %b095 = icmp ult ptr %umin, %2
+  %b196 = icmp ult ptr %umin50, %2
+  %res = and i1 %b095, %b196
+  ret i1 %res
+}

From 8972979c37b7368735a0b224d5212c29041dcd5c Mon Sep 17 00:00:00 2001
From: RoseZhang03 <rosezhang@google.com>
Date: Mon, 22 Jul 2024 20:15:26 +0000
Subject: [PATCH 478/486] [libc] Updated header_generation.rst (#99712)

Added new headergen documentation.
---
 libc/docs/dev/header_generation.rst | 249 ++++++++++++++++++----------
 1 file changed, 164 insertions(+), 85 deletions(-)

diff --git a/libc/docs/dev/header_generation.rst b/libc/docs/dev/header_generation.rst
index ed6127e403ec7..89036f1277b96 100644
--- a/libc/docs/dev/header_generation.rst
+++ b/libc/docs/dev/header_generation.rst
@@ -1,120 +1,199 @@
 Generating Public and Internal headers
 ======================================
 
-.. warning::
-  This page is severely out of date. Much of the information it contains may be
-  incorrect. Please only remove this warning once the page has been updated.
+This is a new implementation of the previous libc header generator. The old
+header generator (libc-hdrgen aka "Headergen") was based on TableGen, which
+created an awkward dependency on the rest of LLVM for our build system. By
+creating a new standalone Headergen we can eliminate these dependencies for
+easier cross compatibility.
 
-Other libc implementations make use of preprocessor macro tricks to make header
-files platform agnostic. When macros aren't suitable, they rely on build
-system tricks to pick the right set of files to compile and export. While these
-approaches have served them well, parts of their systems have become extremely
-complicated making it hard to modify, extend or maintain. To avoid these
-problems in llvm-libc, we use a header generation mechanism. The mechanism is
-driven by a *header configuration language*.
+There are 3 main components of the new Headergen. The first component are the
+YAML files that contain all the function header information and are separated by
+header specification and standard. The second component are the classes that are
+created for each component of the function header: macros, enumerations, types,
+function, arguments, and objects. The third component is the Python script that
+uses the class representation to deserialize YAML files into its specific
+components and then reserializes the components into the function header. The
+Python script also combines the generated header content with header definitions
+and extra macro and type inclusions from the .h.def file.
 
-Header Configuration Language
------------------------------
 
-Header configuration language consists of few special *commands*. The header
-generation mechanism takes an input file, which has an extension of
-``.h.def``, and produces a header file with ``.h`` extension. The header
-configuration language commands are listed in the input ``.h.def`` file. While
-reading a ``.h.def`` file, the header generation tool does two things:
+Instructions
+------------
 
-1. Copy the lines not containing commands as is into the output ``.h`` file.
-2. Replace the line on which a command occurs with some other text as directed
-   by the command. The replacement text can span multiple lines.
+Required Versions:
+  - Python Version: 3.11.8
 
-Command syntax
-~~~~~~~~~~~~~~
+1. Keep full-build mode on when building, otherwise headers will not be
+   generated.
+2. Once the build is complete, enter in the command line within the build
+   directory ``ninja check-newhdrgen`` to ensure that the integration tests are
+   passing.
+3. Then enter in the command line ``ninja libc`` to generate headers. Headers
+   will be in ``build/projects/libc/include`` or ``build/libc/include`` in a
+   runtime build. Sys spec headers will be located in
+   ``build/projects/libc/include/sys``.
 
-A command should be listed on a line by itself, and should not span more than
-one line. The first token to appear on the line is the command name prefixed
-with ``%%``. For example, a line with the ``include_file`` command should start
-with ``%%include_file``. There can be indentation spaces before the ``%%``
-prefix.
 
-Most commands typically take arguments. They are listed as a comma separated
-list of named identifiers within parenthesis, similar to the C function call
-syntax. Before performing the action corresponding to the command, the header
-generator replaces the arguments with concrete values.
+New Headergen is turned on by default, but if you want to use old Headergen,
+you can include this statement when building: ``-DLIBC_USE_NEW_HEADER_GEN=OFF``
 
-Argument Syntax
-~~~~~~~~~~~~~~~
+To add a function to the YAML files, you can either manually enter it in the
+YAML file corresponding to the header it belongs to or add it through the
+command line.
 
-Arguments are named indentifiers but prefixed with ``$`` and enclosed in ``{``
-and ``}``. For example, ``${path_to_constants}``.
+To add through the command line:
 
-Comments
-~~~~~~~~
+1. Make sure you are in the llvm-project directory.
 
-There can be cases wherein one wants to add comments in the .h.def file but
-does not want them to be copied into the generated header file. Such comments
-can be added by beginning the comment lines with the ``<!>`` prefix. Currently,
-comments have to be on lines of their own. That is, they cannot be suffixes like
-this:
+2. Enter in the command line:
 
-```
-%%include_file(a/b/c) <!> Path to c in b of a.  !!! WRONG SYNTAX
-```
+   .. code-block:: none
 
-Available Commands
-------------------
+     python3 libc/newhdrgen/yaml_to_classes.py
+     libc/newhdrgen/yaml/[yaml_file.yaml] --add_function "<return_type>" <function_name> "<function_arg1, function_arg2>" <standard> <guard> <attribute>
 
-Sub-sections below describe the commands currently available. Under each command
-is the description of the arguments to the command, and the action taken by the
-header generation tool when processing a command.
+   Example:
 
-``include_file``
-~~~~~~~~~~~~~~~~
+   .. code-block:: none
 
-This is a replacement command which should be listed in an input ``.h.def``
-file.
+      python3 libc/newhdrgen/yaml_to_classes.py
+      libc/newhdrgen/yaml/ctype.yaml --add_function "char" example_function
+      "int, void, const void" stdc example_float example_attribute
 
-Arguments
+   Keep in mind only the return_type and arguments have quotes around them. If
+   you do not have any guards or attributes you may enter "null" for both.
 
-  * **path argument** - An argument representing a path to a file. The file
-    should have an extension of ``.h.inc``.
+3. Check the YAML file that the added function is present. You will also get a
+   generated header file with the new addition in the newhdrgen directory to
+   examine.
 
-Action
 
-  This command instructs that the line on which the command appears should be
-  replaced by the contents of the file whose path is passed as argument to the
-  command.
+Testing
+-------
 
-``begin``
-~~~~~~~~~
+New Headergen has an integration test that you may run once you have configured
+your CMake within the build directory. In the command line, enter the following:
+``ninja check-newhdrgen``. The integration test is one test that ensures the
+process of YAML to classes to generate headers works properly. If there are any
+new additions on formatting headers, make sure the test is updated with the
+specific addition.
 
-This is not a replacement command. It is an error to list it in the input
-``.h.def`` file. It is normally listed in the files included by the
-``include_file`` command (the ``.h.inc`` files). A common use of this command it
-mark the beginning of what is to be included. This prevents copying items like
-license headers into the generated header file.
+Integration Test can be found in: ``libc/newhdrgen/tests/test_integration.py``
 
-Arguments
+File to modify if adding something to formatting:
+``libc/newhdrgen/tests/expected_output/test_header.h``
 
-  None.
 
-Action
+Common Errors
+-------------
+1. Missing function specific component
 
-  The header generator will only include content starting from the line after the
-  line on which this command is listed.
+   Example:
 
-``public_api``
-~~~~~~~~~~~~~~
+   .. code-block:: none
 
-This is a replacement command which should be listed in an input ``.h.def``
-file. The header file generator will replace this command with the public API of
-the target platform. See the build system document for more information on the
-relevant build rules. Also, see "Mechanics of public_api" to learn the mechanics
-of how the header generator replaces this command with the public API.
+      "/llvm-project/libc/newhdrgen/yaml_to_classes.py", line 67, in yaml_to_classes function_data["return_type"]
 
-Arguments
+   If you receive this error or any error pertaining to
+   ``function_data[function_specific_component]`` while building the headers
+   that means the function specific component is missing within the YAML files.
+   Through the call stack, you will be able to find the header file which has
+   the issue. Ensure there is no missing function specific component for that
+   YAML header file.
 
-  None.
+2. CMake Error: require argument to be specified
 
-Action
+   Example:
 
-  The header generator will replace this command with the public API to be exposed
-  from the generated header file.
+   .. code-block:: none
+
+     CMake Error at:
+     /llvm-project/libc/cmake/modules/LLVMLibCHeaderRules.cmake:86 (message):
+     'add_gen_hdr2' rule requires GEN_HDR to be specified.
+     Call Stack (most recent call first):
+     /llvm-project/libc/include/CMakeLists.txt:22 (add_gen_header2)
+     /llvm-project/libc/include/CMakeLists.txt:62 (add_header_macro)
+
+   If you receive this error, there is a missing YAML file, h_def file, or
+   header name within the ``libc/include/CMakeLists.txt``. The last line in the
+   error call stack will point to the header where there is a specific component
+   missing. Ensure the correct style and required files are present:
+
+   | ``[header_name]``
+   | ``[../libc/newhdrgen/yaml/[yaml_file.yaml]``
+   | ``[header_name.h.def]``
+   | ``[header_name.h]``
+   | ``DEPENDS``
+   |   ``{Necessary Depend Files}``
+
+3. Command line: expected arguments
+
+   Example:
+
+   .. code-block:: none
+
+     usage: yaml_to_classes.py [-h] [--output_dir OUTPUT_DIR] [--h_def_file H_DEF_FILE]
+     [--add_function RETURN_TYPE NAME ARGUMENTS STANDARDS GUARD ATTRIBUTES]
+     [--e ENTRY_POINTS] [--export-decls]
+     yaml_file
+     yaml_to_classes.py:
+     error: argument --add_function: expected 6 arguments
+
+   In the process of adding a function, you may run into an issue where the
+   command line is requiring more arguments than what you currently have. Ensure
+   that all components of the new function are filled. Even if you do not have a
+   guard or attribute, make sure to put null in those two areas.
+
+4. Object has no attribute
+
+   Example:
+
+   .. code-block:: none
+
+     File "/llvm-project/libc/newhdrgen/header.py", line 60, in __str__ for
+     function in self.functions: AttributeError: 'HeaderFile' object has no
+     attribute 'functions'
+
+   When running ``ninja libc`` in the build directory to generate headers you
+   may receive the error above. Essentially this means that in
+   ``libc/newhdrgen/header.py`` there is a missing attribute named functions.
+   Make sure all function components are defined within this file and there are
+   no missing functions to add these components.
+
+5. Unknown type name
+
+   Example:
+
+   .. code-block:: none
+
+     /llvm-project/build/projects/libc/include/sched.h:20:25: error: unknown type
+     name 'size_t'; did you mean 'time_t'?
+     20 | int_sched_getcpucount(size_t, const cpu_set_t*) __NOEXCEPT
+      |           ^
+     /llvm-project/build/projects/libc/include/llvm-libc-types/time_t.h:15:24:
+     note: 'time_t' declared here
+     15 | typedef __INT64_TYPE__ time_t;
+     |                    ^
+
+   During the header generation process errors like the one above may occur
+   because there are missing types for a specific header file. Check the YAML
+   file corresponding to the header file and make sure all the necessary types
+   that are being used are input into the types as well. Delete the specific
+   header file from the build folder and re-run ``ninja libc`` to ensure the
+   types are being recognized.
+
+6. Test Integration Errors
+
+   Sometimes the integration test will fail but that
+   still means the process is working unless the comparison between the output
+   and expected_output is not showing. If that is the case make sure in
+   ``libc/newhdrgen/tests/test_integration.py`` there are no missing arguments
+   that run through the script.
+
+   If the integration tests are failing due to mismatching of lines or small
+   errors in spacing that is nothing to worry about. If this is happening while
+   you are making a new change to the formatting of the headers, then
+   ensure the expected output file
+   ``libc/newhdrgen/tests/expected_output/test_header.h`` has the changes you
+   are applying.

From d2f42c737234662d1bdc2f2e393e1b59536f8a93 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Mon, 22 Jul 2024 13:16:22 -0700
Subject: [PATCH 479/486] [mlir] Add unit test for RankedTensorType wrapper
 example. (#99789)

Add example as unit test for creating a wrapper type/view for
RankedTensorType with encoding. This view provides a more restricted &
typed API while it allows one to avoid repeated casting queries and
accessing the encoding directly.

For users with more advance encodings, the expectation would be a
separate attribute type, but here just StringAttr is used.
---
 mlir/unittests/IR/ShapedTypeTest.cpp | 58 ++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/mlir/unittests/IR/ShapedTypeTest.cpp b/mlir/unittests/IR/ShapedTypeTest.cpp
index 61264bc523648..7a5b0722a03ba 100644
--- a/mlir/unittests/IR/ShapedTypeTest.cpp
+++ b/mlir/unittests/IR/ShapedTypeTest.cpp
@@ -11,6 +11,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/DialectInterface.h"
+#include "mlir/Support/LLVM.h"
 #include "llvm/ADT/SmallVector.h"
 #include "gtest/gtest.h"
 #include <cstdint>
@@ -226,4 +227,61 @@ TEST(ShapedTypeTest, RankedTensorTypeBuilder) {
   }
 }
 
+/// Simple wrapper class to enable "isa querying" and simple accessing of
+/// encoding.
+class TensorWithString : public RankedTensorType {
+public:
+  using RankedTensorType::RankedTensorType;
+
+  static TensorWithString get(ArrayRef<int64_t> shape, Type elementType,
+                              StringRef name) {
+    return mlir::cast<TensorWithString>(RankedTensorType::get(
+        shape, elementType, StringAttr::get(elementType.getContext(), name)));
+  }
+
+  StringRef getName() const {
+    if (Attribute enc = getEncoding())
+      return mlir::cast<StringAttr>(enc).getValue();
+    return {};
+  }
+
+  static bool classof(Type type) {
+    if (auto rt = mlir::dyn_cast_or_null<RankedTensorType>(type))
+      return mlir::isa_and_present<StringAttr>(rt.getEncoding());
+    return false;
+  }
+};
+
+TEST(ShapedTypeTest, RankedTensorTypeView) {
+  MLIRContext context;
+  Type f32 = FloatType::getF32(&context);
+
+  Type noEncodingRankedTensorType = RankedTensorType::get({10, 20}, f32);
+
+  UnitAttr unitAttr = UnitAttr::get(&context);
+  Type unitEncodingRankedTensorType =
+      RankedTensorType::get({10, 20}, f32, unitAttr);
+
+  StringAttr stringAttr = StringAttr::get(&context, "app");
+  Type stringEncodingRankedTensorType =
+      RankedTensorType::get({10, 20}, f32, stringAttr);
+
+  EXPECT_FALSE(mlir::isa<TensorWithString>(noEncodingRankedTensorType));
+  EXPECT_FALSE(mlir::isa<TensorWithString>(unitEncodingRankedTensorType));
+  ASSERT_TRUE(mlir::isa<TensorWithString>(stringEncodingRankedTensorType));
+
+  // Cast to TensorWithString view.
+  auto view = mlir::cast<TensorWithString>(stringEncodingRankedTensorType);
+  ASSERT_TRUE(mlir::isa<TensorWithString>(view));
+  EXPECT_EQ(view.getName(), "app");
+  // Verify one could cast view type back to base type.
+  ASSERT_TRUE(mlir::isa<RankedTensorType>(view));
+
+  Type viewCreated = TensorWithString::get({10, 20}, f32, "bob");
+  ASSERT_TRUE(mlir::isa<TensorWithString>(viewCreated));
+  ASSERT_TRUE(mlir::isa<RankedTensorType>(viewCreated));
+  view = mlir::cast<TensorWithString>(viewCreated);
+  EXPECT_EQ(view.getName(), "bob");
+}
+
 } // namespace

From 6777c3400708abbd607e7c167f73922eb0d807dc Mon Sep 17 00:00:00 2001
From: "S. Bharadwaj Yadavalli" <Bharadwaj.Yadavalli@microsoft.com>
Date: Mon, 22 Jul 2024 16:23:27 -0400
Subject: [PATCH 480/486] [DXIL][Doc] Update specification of TableGen DXIL
 properties (#99055)

Update the specification of various DXIL Operation properties that
are predicated by DXIL version.
---
 llvm/docs/DirectX/DXILOpTableGenDesign.rst | 403 +++++++++++++++------
 1 file changed, 284 insertions(+), 119 deletions(-)

diff --git a/llvm/docs/DirectX/DXILOpTableGenDesign.rst b/llvm/docs/DirectX/DXILOpTableGenDesign.rst
index 5f3ac75e82494..50d801bd05efd 100644
--- a/llvm/docs/DirectX/DXILOpTableGenDesign.rst
+++ b/llvm/docs/DirectX/DXILOpTableGenDesign.rst
@@ -24,12 +24,12 @@ DXIL Operations are represented in one of the following `two ways
 
    These are  collectively referred to as `LLVM Intrinsics` in this note.
 
-Following is the complete list of attributes of DXIL Ops with the corresponding field name
-as used in ``hctdb.py``. A DXIL Op is represented by a set of associated attributes. These
-are consumed in DXIL backend passes as well as in other usage scenarios such as validation, 
+Following is the complete list of properties of DXIL Ops with the corresponding field name
+as used in ``hctdb.py``. A DXIL Op is represented by a set of associated properties. These
+are consumed in DXIL backend passes as well as in other usage scenarios such as validation,
 DXIL reader, etc.
 
-A. Attributes consumed in DXIL backend passes
+A. Properties consumed in DXIL backend passes
 
    1. Name of operation (``dxil_op``)
    2. A string that documents the operation (``doc``) - This is not strictly necessary but is included
@@ -38,7 +38,7 @@ A. Attributes consumed in DXIL backend passes
    4. Unique Integer ID (``dxil_opid``)
    5. Operation Class signifying the name and function signature of the operation (``dxil_class``).
       This string is an integral part of the DXIL Op function name and is constructed in
-      the format ``dx.op.<class-name>.<overload-type>``. Each DXIL Op call target function name 
+      the format ``dx.op.<class-name>.<overload-type>``. Each DXIL Op call target function name
       is required to conform to this format per existing contract with the driver.
    6. List of valid overload types for the operation (``oload_types``).
    7. Required DXIL Version with support for the operation.
@@ -58,26 +58,26 @@ A. Attributes consumed in DXIL backend passes
 Motivation
 ==========
 
-DXIL backend passes depend on various attributes of DXIL Operations. For example, ``DXILOpLowering``
+DXIL backend passes depend on various properties of DXIL Operations. For example, ``DXILOpLowering``
 pass will need information such as the DXIL operation an LLVM intrinsic is to be lowered to,
-along with valid overload and parameter types etc. The TableGen file -
+along with valid overload and argument types etc. The TableGen file -
 ``llvm/lib/Target/DirectX/DXIL.td`` - is used to represent DXIL Operations
-by specifying their attributes listed above. ``DXIL.td`` is designed to be the single source
-of reference of DXIL Operations primarily for the implementation of passes in DXIL backend in 
-``llvm-project`` repo - analogous to ``hctdb.py`` for ``DirectXShadeCompiler`` repo. However, 
-the current design does not intend to encapsulate various validation rules, present in ``hctdb.py``, 
-but do not pertain to DXIL Operations. It needs to have a rich representation capabilities that 
-TableGen backends (such as ``DXILEmitter``) can rely on. Additionally, the DXIL Op specification 
+by specifying their properties listed above. ``DXIL.td`` is designed to be the single source
+of reference of DXIL Operations primarily for the implementation of passes in DXIL backend in
+``llvm-project`` repo - analogous to ``hctdb.py`` for ``DirectXShadeCompiler`` repo. However,
+the current design does not intend to encapsulate various validation rules, present in ``hctdb.py``,
+but do not pertain to DXIL Operations. It needs to have a rich representation capabilities that
+TableGen backends (such as ``DXILEmitter``) can rely on. Additionally, the DXIL Op specification
 should be easy to read and comprehend.
 
 This note provides the design of the specification DXIL Ops as TableGen class ``DXILOp``
-by specifying its attributes identified above.
+by specifying its properties identified above.
 
 DXIL Operation Specification
 ============================
 
 The DXIL Operation is represented using the TableGen class ``DXILOp``. The DXIL operation
-attributes are specified as fields of the ``DXILOp`` class as described below.
+properties are specified as fields of the ``DXILOp`` class as described below.
 
 1. Each DXIL Operation is represented as a TableGen record. The name of each of the records
    signifies operation name.
@@ -93,76 +93,81 @@ attributes are specified as fields of the ``DXILOp`` class as described below.
         class DXILOpClass;
 
    Concrete operation records, such as ``unary`` are defined by inheriting from ``DXILOpClass``.
-6. Return and argument types of the operation are represented as ``dag``s using the
-   special markers ``out`` and ``ins``. An overload type, if supported by the operation, is
-   denoted as the positional type ``dxil_overload_ty`` in the argument or in the result, where
-   ``dxil_overload_ty`` is defined to be synonymous to ``llvm_any_ty``.
+6. Return type of the operation is represented as ``LLVMType``.
+7. Operation arguments are represented as a list of ``LLVMType`` with each type
+   corresponding to the argument position. An overload type, if supported by the operation, is
+   denoted as the positional type ``overloadTy`` in the argument or in the result, where
+   ``overloadTy`` is defined to be synonymous to ``llvm_any_ty``.
 
    .. code-block::
 
-      defvar dxil_overload_ty = llvm_any_ty
+      defvar overloadTy = llvm_any_ty
 
+   Empty list, ``[]`` represents an operation with no arguments.
 
-7. Valid overload types and shader stages predicated on Shader Model version are specified
-   as a list of ``Constraint`` records. Representation of ``Constraints`` class is described
-   a later section.
-8. Various attributes of the DXIL Operation that are not predicated on Shader Model version
-   are represented as a ``dag`` using the special marker ``attrs``. Representation of ``Attributes`` 
+8. Valid operation overload types predicated on DXIL version are specified as
+   a list of ``Overloads`` records. Representation of ``Overloads``
    class is described in a later section.
+9.  Valid shader stages predicated on DXIL version are specified as a list of
+    ``Stages`` records. Representation of ``Stages`` class is
+    described in a later section.
+10. Various attributes of the DXIL Operation are represented as a ``list`` of
+    ``Attributes`` class records. Representation of ``Attributes``
+    class is described in a later section.
 
-A DXIL Operation is represented by the following TableGen class by encapsulating the various
-TableGen representations of its attributes described above.
+Types specific to DXIL
+----------------------
+
+Type notation used in this document viz., ``<size>Ty`` corresponds to TableGen records for
+LLVM types ``llvm_<size>_ty``. Apart from ``overloadTy`` described above, ``resRetF32Ty`` is
+used to denote resource return type and ``handleTy`` is used to denote handle type.
+
+Specification of DXIL Operation
+================================
+
+A DXIL Operation is represented by the following TableGen class that encapsulates the various
+TableGen representations of its properties described above.
 
 .. code-block::
 
    // Abstraction DXIL Operation
-   class DXILOp {
+   class DXILOp<int opcode, DXILOpClass opclass> {
      // A short description of the operation
      string Doc = "";
 
      // Opcode of DXIL Operation
-     int OpCode = 0;
+     int OpCode = opcode;
 
      // Class of DXIL Operation.
-     DXILOpClass OpClass = UnknownOpClass;
+     DXILOpClass OpClass = opclass;
 
      // LLVM Intrinsic DXIL Operation maps to
      Intrinsic LLVMIntrinsic = ?;
 
-     // Dag containing the arguments of the op. Default to 0 arguments.
-     dag arguments = (ins);
-
-     // Results of the op. Default to 0 results.
-     dag result = (out);
-
-     // List of constraints predicated on Shader Model version
-     list<SMVersionConstraints> sm_constraints;
+     // Result type of the op.
+     LLVMType result;
 
-     // Non-predicated operation attributes
-     dag attrtibutes = (attrs);
-     Version DXILVersion = ?;
-   }
-
-Constraint Specification
-========================
+     // List of argument types of the op. Default to 0 arguments.
+     list<LLVMType> arguments = [];
 
-DXIL Operation attributes such as valid overload types and valid shader stages are
-predicated on Shader Model version. These are represented as list of constrained
-attributes.
+     // List of valid overload types predicated by DXIL version
+     list<Overloads> overloads;
 
-Following is the definition of a generic constraint and the associated predicate
+     // List of valid shader stages predicated by DXIL version
+    list<Stages> stages;
 
-.. code-block::
+     // List of valid attributes predicated by DXIL version
+     list<Attributes> attributes = [];
+   }
 
-   // Primitive predicate
-   class Pred;
+Version Specification
+=====================
 
-   // Generic constraint
-   class Constraint<Pred pred> {
-     Pred predicate = pred;
-   }
+DXIL version is used to specify various version-dependent operation properties in
+place of Shader Model version.
 
-Shader Model version is represented as follows:
+A ``Version`` class encapsulating ``Major`` and ``Minor`` version number is defined
+as follows:
 
 .. code-block::
 
@@ -172,112 +177,272 @@ Shader Model version is represented as follows:
      int Minor = minor;
    }
 
-   // Valid Shader model version records
 
-   // Definition of Shader Model 6.0 - 6.8 and DXIL Version 1.0 - 1.8
+Concrete representations of valid DXIL versions are defined as follows:
+
+.. code-block::
+
+   // Definition of DXIL Version 1.0 - 1.8
    foreach i = 0...8 in {
-     def SM6_#i : Version<6, i>;
-     def DX1_#i : Version<1, i>;
+     def DXIL1_#i : Version<1, i>;
    }
 
-A shader model version predicate class is defined as
+Shader Stage Specification
+==========================
+
+Various shader stages such as ``compute``, ``pixel``, ``vertex``, etc., are represented
+as follows
 
 .. code-block::
 
-   class SMVersion<Version ver> : Pred {
-     Version SMVersion = ver;
-   }
+   // Shader stages
+   class DXILShaderStage;
 
-A constraint class to represent overload types and shader stages predicated on shader
-model version is defined as
+   def compute : DXILShaderStage;
+   def pixel : DXILShaderStage;
+   def vertex : DXILShaderStage;
+   ...
+
+Shader Attribute Specification
+==============================
+
+Various operation memory access and boolean attributes such as ``ReadNone``,
+``IsWave`` etc., are represented as follows
 
 .. code-block::
 
-   class SMVersionConstraints<SMVersion smver, dag oloads, dag stages> : Constraint<smver> {
-     dag overload_types = oloads;
-     dag stage_kinds = stages;
-   }
+  class DXILAttribute;
 
-The ``dag overload_types`` and ``dag shader_kinds`` use a special markers ``overloads``
-and ``stages``, respectively.
+  def ReadOnly : DXILOpAttributes;
+  def ReadNone : DXILOpAttributes;
+  def IsWave : DXILOpAttributes;
+  ...
 
-Examples of Constraints
------------------------
+Versioned Property Specification
+================================
+
+DXIL Operation properties such as valid overload types, shader stages and
+attributes are predicated on DXIL version. These are represented as list of
+versioned properties.
+
+Overload Type Specification
+---------------------------
 
-Consider a DXIL Operation that is valid in Shader Model 6.2 and later,
+``overloads`` field of ``class DXILOp`` is used to represent valid operation
+overloads predicated on DXIL version as list of records of the following class
+
+.. code-block::
 
-1. with valid overload types ``half``, ``float``, ``i16`` and ``i32``
-2. is valid for stages ``pixel`` and ``compute``
-3. with valid overload types ``double`` and ``i614`` if Shader Model version 6.3 and later
-4. is valid for all stages if Shader Model version 6.3 and later
+   class Overloads<Version minver, list<LLVMType> ols> {
+     Version dxil_version = minver;
+     list<LLVMType> overload_types = ols;
+   }
 
-This is represented as
+Following is an example specification of valid overload types for ``DXIL1_0`` and
+``DXIL1_2``.
 
 .. code-block::
 
-   [SMVersionConstraints<SMVersion<SM6_2>,
-                          (overloads llvm_half_ty, llvm_float_ty, llvm_i16_ty, llvm_i32_ty),
-                          (stages pixel, compute)>,
-    SMVersionConstraints<SMVersion<SM6_3>,
-                          (overloads llvm_half_ty, llvm_float_ty, llvm_double_ty,
-                                 llvm_i16_ty, llvm_i32_ty, llvm_i64_ty),
-                          (stages allKinds)>];
+   overloads = [
+                 Overloads<DXIL1_0, [halfTy, floatTy]>,
+                 Overloads<DXIL1_2, [halfTy, floatTy, doubleTy]>
+               ];
+
+An empty list signifies that the operation supports no overload types.
 
-Consider a DXIL operation that is valid in Shader Model version 6.2 and later,
 
-1. with no overload types, i.e., all argument typess and result type are fixed.
-2. is valid for all stages.
+Stages Specification
+--------------------
 
-This is represented as
+``stages`` field of ``class DXILOp`` is used to represent valid operation
+stages predicated on DXIL version as list of records of the following class
 
 .. code-block::
 
-     [SMVersionConstraints<SMVersion<SM6_2>, (overloads), (stages allKinds)>];
+   class Stages<Version minver, list<DXILShaderStage> sts> {
+     Version dxil_version = minver;
+     list<DXILShaderStage> shader_stages = sts;
+   }
 
+Following is an example specification of valid stages for ``DXIL1_0``,
+``DXIL1_2``, ``DXIL1_4`` and ``DXIL1_6``.
+
+.. code-block::
+
+   stages = [
+             Stages<DXIL1_0, [compute, pixel]>,
+             Stages<DXIL1_2, [compute, pixel, mesh]>,
+             Stages<DXIL1_4, [all_stages]>,
+             Stages<DXIL1_6, [removed]>
+            ];
+
+The following two pseudo stage records in addition to standard shader stages
+are defined.
+
+1. ``all_stages`` signifies that the operation is valid for all stages in the
+   specified DXIL version and later.
+2. ``removed`` signifies removal of support for the operation in the specified
+   DXIL version and later.
+
+A non-empty list of supported stages is required to be specified. If an operation
+is supported in all DXIL versions and all stages it is required to be specified as
+
+.. code-block::
+
+   stages = [Stages<DXIL1_0, [all_stages]>];
 
-Specifying attributes predicated on Shader Model version using the single field 
-``sm_constraints`` not only allows for all of them to be specified together but
-also allows for a single place to specify minimum shader model version that supports
-the operation. Thus, a separate fiels is not needed to specify minimum shader model 
-version.
 
 Attribute Specification
-=======================
+-----------------------
 
-DXIL Operation attributes that are not predicated on any constraint, are represented as
-a ``dag`` of Attribute records of the following abstract ``DXILAttributes`` class.
+``attributes`` field of ``class DXILOp`` is used to represent valid operation
+attributes predicated on DXIL version as list of records of the following class
 
 .. code-block::
 
-  class DXILAttributes;
+  class Attributes<MinVersion minver, list<DXILAttribute> attrs> {
+    MinVersion dxil_version = ver;
+    list<DXILAttribute> attributes = attrs;
+  }
 
-Following example records represent memory attributes 
+Following is an example specification of valid attributes for ``DXIL1_0``.
 
 .. code-block::
 
-  def ReadOnly : DXILOpAttributes;
-  def ReadNone : DXILOpAttributes;
+   attributes = [Attributes<DXIL1_0, [ReadNone]];
+
+A null list of ``attributes`` signifies no operation attributes.
 
-DXIL Operation Specification Example
-====================================
-Following illustrates the specification of the DXIL Op ``Sin``
+Interpretation of Multiple Versioned Properties
+-----------------------------------------------
+
+Each of the versioned properties states that the specified overload type, stage or
+attribute records are valid for the predicated DXIL version. Only
+the properties corresponding to latest minimal DXIL version are applicable.
+Note as in the above example, any overload types, stages or attributes,
+that remain valid in a later DXIL version need to be specified in full.
+For example, consider the following specification of valid overload types:
 
 .. code-block::
 
-  def Sin  : DXILOp {
-    let Doc ="Returns sine(theta) for theta in radians.";
-    let OpCode = 13;
-    let OpClass = unary;
+   overloads = [
+                Overloads<DXIL1_0, [halfTy, floatTy]>,
+                Overloads<DXIL1_2, [halfTy, floatTy, doubleTy]>
+               ];
+
+It specifies that the overload types ``halfTy`` and ``floatTy`` are valid for DXIL
+version 1.0 and later. It also specifies that  ``doubleTy`` is additionally supported
+in DXIL version 1.2 and later.
+
+This provides the flexibility to specify properties independent of other
+versioned specifications in the list.
+
+
+DXIL Operation Specification Examples
+=====================================
+
+Following examples illustrate the specification of some of the DXIL Ops.
+
+``Sin`` operation - an operation valid in all DXIL versions and all stages
+and has valid overload types predicated on DXIL version.
+
+.. code-block::
+
+  def Sin : DXILOp<13, unary> {
+    let Doc = "Returns sine(theta) for theta in radians.";
     let LLVMIntrinsic = int_sin;
-    let arguments = (ins LLVMMatchType<0>);
-    let result = (out dxil_overload_ty);
-    let sm_constraints = [SMVersionConstraints<SMVersion<SM6_0>,
-                          (overloads llvm_half_ty, llvm_float_ty),
-                          (stages allKinds)>];
-    let attributes = (attrs ReadNone);
-    let DXILVersion = DX1_0;
+    let result = overloadTy;
+    let arguments = [overloadTy];
+    let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+    let stages = [Stages<DXIL1_0, [all_stages]>];
+    let attributes = [Attributes<DXIL1_0, [ReadNone]>];
   }
 
+``FlattenedThreadIdInGroup`` - an operation with no arguments, no
+overload types, and valid stages and attributes predicated by DXIL Version.
+
+.. code-block::
+
+   def FlattenedThreadIdInGroup :  DXILOp<96, flattenedThreadIdInGroup> {
+    let Doc = "Provides a flattened index for a given thread within a given "
+              "group (SV_GroupIndex)";
+    let LLVMIntrinsic = int_dx_flattened_thread_id_in_group;
+    let result = i32Ty;
+    let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
+    let attributes = [Attributes<DXIL1_0, [ReadNone]>];
+   }
+
+``RawBufferStore`` - an operation with ``void`` return type, valid overload types
+predicated by DXIL Version and valid in all DXIL versions and stages.
+
+.. code-block::
+
+   def RawBufferStore : DXILOp<140, rawBufferStore> {
+     let Doc = "Writes to a RWByteAddressBuffer or RWStructuredBuffer.";
+     let result = voidTy;
+     let arguments = [dxil_resource_ty, i32Ty, i32Ty, overloadTy,
+                      overloadTy, overloadTy, overloadTy, i8Ty, i32Ty];
+     let overloads = [
+                      Overloads<DXIL1_2, [halfTy, floatTy, i16Ty, i32Ty]>,
+                      Overloads<DXIL1_3>,[halfTy, floatTy, doubleTy,
+                                                   i16Ty, i32Ty, i64Ty]>
+                     ];
+      let stages = [Stages<DXIL1_2, all_stages>];
+      let attributes = [Attributes<DXIL1_0, [ReadOnly]>];
+   }
+
+``DerivCoarseX`` - an operation with no overload types and stages predicated
+by DXIL Version.
+
+.. code-block::
+
+   def DerivCoarseX : DXILOp<83, unary> {
+    let doc = "Computes the rate of change per stamp in x direction.";
+    let LLVMIntrinsic = int_dx_ddx;
+    let result = overloadTy;
+    let arguments = [overloadTy];
+    let stages = [
+                   Stages<DXIL1_0, [library, pixel]>,
+                   Stages<DXIL1_6, [library, pixel, amplification, compute, mesh]>
+                 ];
+    let attributes = [Attributes<DXIL1_0, [ReadNone]>];
+   }
+
+``CreateHandle`` - an operation with no overload types, no associated ``LLVMIntrinsic``
+and stages predicated  by DXIL Version.
+
+.. code-block::
+
+   def CreateHandle : DXILOp<57, createHandle> {
+     let doc = "Creates the handle to a resource";
+     let result = i32Ty;
+     let arguments = [i8Ty, i32Ty, i32Ty, i1Ty];
+     let stages = [
+                   Stages<DXIL1_0, [all_stages]>,
+                   Stages<DXIL1_6, [removed]
+                  ];
+     let attributes = [Attributes<DXIL1_0, [ReadOnly]>];
+   }
+
+``Sample`` - an operation with valid overload types, stages and attributes
+predicated by DXIL version.
+
+.. code-block::
+
+   def Sample : DXILOp<60, sample> {
+     let Doc = "Samples a texture";
+     let LLVMIntrinsic = int_dx_sample;
+     let result = resRetF32Ty;
+     let arguments = [handleTy, handleTy, floatTy, floatTy, floatTy, floatTy,
+                      i32Ty, i32Ty, i32Ty, floatTy];
+     let overloads = [Overloads<DXIL1_0, [halfTy, floatTy, i16Ty, i32Ty]>];
+     let stages = [
+                   Stages<DXIL1_0, [library, pixel]>,
+                   Stages<DXIL1_6, [library, pixel, amplification, compute, mesh]>
+                  ];
+     let attributes = [Attributes<DXIL1_0, [ReadOnly]>];
+   }
+
 Summary
 =======
 

From aac3a2a291598f3aef01833927fa754c53849cdf Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 22 Jul 2024 15:28:48 -0500
Subject: [PATCH 481/486] [libc] Fix callback type in `exit_handlers.cpp` not
 matching (#97642)

Summary:
This file is an object library, but uses the `LIBC_COPT_PUBLIC_PACKAING`
option. This will always be undefined which leads to a type mismatch
when uses actually try to link against it. This patch simply removes
this and turns it into a header only library. This means that the
implementations of the callback lists and the mutexes need to live in
their respective files. The result is that `atexit` needs to be defined
for `at_quick_exit` to be valid.
---
 libc/src/stdlib/CMakeLists.txt    | 15 ++++++-----
 libc/src/stdlib/at_quick_exit.cpp |  2 ++
 libc/src/stdlib/atexit.cpp        |  3 +++
 libc/src/stdlib/exit_handler.cpp  | 43 -------------------------------
 libc/src/stdlib/exit_handler.h    | 28 +++++++++++++++-----
 5 files changed, 36 insertions(+), 55 deletions(-)
 delete mode 100644 libc/src/stdlib/exit_handler.cpp

diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index 513f6ad723d56..b9b10bd2b4f71 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -51,6 +51,8 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.OSUtil.osutil
     .exit_handler
+    .at_quick_exit
+    .atexit
 )
 
 add_entrypoint_object(
@@ -462,15 +464,11 @@ add_entrypoint_object(
 
 # TODO: Move all exit functions to linux specific 
 
-if (TARGET libc.src.__support.threads.mutex)
-add_object_library(
+if(TARGET libc.src.__support.threads.mutex)
+add_header_library(
   exit_handler
-  SRCS
-    exit_handler.cpp
   HDRS
     exit_handler.h
-  CXX_STANDARD
-    20 # For constinit 
   DEPENDS
     libc.src.__support.CPP.mutex
     libc.src.__support.CPP.new
@@ -487,6 +485,8 @@ add_entrypoint_object(
     atexit.cpp
   HDRS
     atexit.h
+  CXX_STANDARD
+    20 # For constinit 
   DEPENDS
     .exit_handler
 )
@@ -497,8 +497,11 @@ add_entrypoint_object(
     at_quick_exit.cpp
   HDRS
     at_quick_exit.h
+  CXX_STANDARD
+    20 # For constinit 
   DEPENDS
     .exit_handler
+    .atexit
 )
 
 list(APPEND exit_deps
diff --git a/libc/src/stdlib/at_quick_exit.cpp b/libc/src/stdlib/at_quick_exit.cpp
index 7acae8c52def3..d2b4c0cec986f 100644
--- a/libc/src/stdlib/at_quick_exit.cpp
+++ b/libc/src/stdlib/at_quick_exit.cpp
@@ -14,6 +14,8 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+constinit ExitCallbackList at_quick_exit_callbacks;
+
 LLVM_LIBC_FUNCTION(int, at_quick_exit, (__atexithandler_t callback)) {
   return add_atexit_unit(
       at_quick_exit_callbacks,
diff --git a/libc/src/stdlib/atexit.cpp b/libc/src/stdlib/atexit.cpp
index 6844fb7aacaf6..c8a15dd3cfef2 100644
--- a/libc/src/stdlib/atexit.cpp
+++ b/libc/src/stdlib/atexit.cpp
@@ -14,6 +14,9 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+constinit ExitCallbackList atexit_callbacks;
+Mutex handler_list_mtx(false, false, false, false);
+
 extern "C" {
 
 int __cxa_atexit(AtExitCallback *callback, void *payload, void *) {
diff --git a/libc/src/stdlib/exit_handler.cpp b/libc/src/stdlib/exit_handler.cpp
deleted file mode 100644
index ed0751a4c889e..0000000000000
--- a/libc/src/stdlib/exit_handler.cpp
+++ /dev/null
@@ -1,43 +0,0 @@
-//===--- Implementation of exit_handler------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/stdlib/exit_handler.h"
-#include "src/__support/CPP/mutex.h" // lock_guard
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-constinit ExitCallbackList at_quick_exit_callbacks;
-constinit ExitCallbackList atexit_callbacks;
-
-Mutex handler_list_mtx(false, false, false, false);
-
-void stdc_at_exit_func(void *payload) {
-  reinterpret_cast<StdCAtExitCallback *>(payload)();
-}
-
-void call_exit_callbacks(ExitCallbackList &callbacks) {
-  handler_list_mtx.lock();
-  while (!callbacks.empty()) {
-    AtExitUnit &unit = callbacks.back();
-    callbacks.pop_back();
-    handler_list_mtx.unlock();
-    unit.callback(unit.payload);
-    handler_list_mtx.lock();
-  }
-  ExitCallbackList::destroy(&callbacks);
-}
-
-int add_atexit_unit(ExitCallbackList &callbacks, const AtExitUnit &unit) {
-  cpp::lock_guard lock(handler_list_mtx);
-  if (callbacks.push_back(unit))
-    return 0;
-  return -1;
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/exit_handler.h b/libc/src/stdlib/exit_handler.h
index 41733b7a0e7e5..9720c5473940e 100644
--- a/libc/src/stdlib/exit_handler.h
+++ b/libc/src/stdlib/exit_handler.h
@@ -38,16 +38,32 @@ using ExitCallbackList = ReverseOrderBlockStore<AtExitUnit, 32>;
 using ExitCallbackList = FixedVector<AtExitUnit, CALLBACK_LIST_SIZE_FOR_TESTS>;
 #endif
 
-extern ExitCallbackList atexit_callbacks;
-extern ExitCallbackList at_quick_exit_callbacks;
-
+// This is handled by the 'atexit' implementation and shared by 'at_quick_exit'.
 extern Mutex handler_list_mtx;
 
-void stdc_at_exit_func(void *payload);
+LIBC_INLINE void stdc_at_exit_func(void *payload) {
+  reinterpret_cast<StdCAtExitCallback *>(payload)();
+}
 
-void call_exit_callbacks(ExitCallbackList &callbacks);
+LIBC_INLINE void call_exit_callbacks(ExitCallbackList &callbacks) {
+  handler_list_mtx.lock();
+  while (!callbacks.empty()) {
+    AtExitUnit &unit = callbacks.back();
+    callbacks.pop_back();
+    handler_list_mtx.unlock();
+    unit.callback(unit.payload);
+    handler_list_mtx.lock();
+  }
+  ExitCallbackList::destroy(&callbacks);
+}
 
-int add_atexit_unit(ExitCallbackList &callbacks, const AtExitUnit &unit);
+LIBC_INLINE int add_atexit_unit(ExitCallbackList &callbacks,
+                                const AtExitUnit &unit) {
+  cpp::lock_guard lock(handler_list_mtx);
+  if (callbacks.push_back(unit))
+    return 0;
+  return -1;
+}
 
 } // namespace LIBC_NAMESPACE_DECL
 

From b177ac4a44ec4024fec6927f4870ee96e9070696 Mon Sep 17 00:00:00 2001
From: davidtrevelyan <davidtrevelyan@users.noreply.github.com>
Date: Mon, 22 Jul 2024 21:35:37 +0100
Subject: [PATCH 482/486] [compiler-rt][rtsan] Use sanitizer internal allocator
 during rtsan init to avoid segfault in dlsym (#98679)

Follows https://github.com/llvm/llvm-project/pull/98268 with a fix for a
segfault during preinit on `ubuntu:20.04` environments. Previously,
`rtsan` was not handling the situation where `dlsym` calls `calloc`
during the interceptors initialization, resulting in a call to a
function at a null address.

@cjappl and I took inspiration from the solution in `nsan`, but we
re-used the sanitizer internal allocator instead of our own static
buffer. This PR also re-enables the existing non-instrumented `rtsan`
tests for `x86_64` and `arm64` architectures.

---------

Co-authored-by: Chris Apple <cja-private@pm.me>
---
 .../cmake/Modules/AllSupportedArchDefs.cmake  |  4 +--
 compiler-rt/lib/rtsan/rtsan.cpp               | 15 ++++++++++-
 compiler-rt/lib/rtsan/rtsan.h                 |  7 ++++++
 compiler-rt/lib/rtsan/rtsan_interceptors.cpp  | 25 +++++++++++++++++++
 4 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index 02ff92f693810..29e5beb6182ba 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -32,9 +32,7 @@ set(ALL_ASAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64}
     ${LOONGARCH64})
 set(ALL_ASAN_ABI_SUPPORTED_ARCH ${X86_64} ${ARM64} ${ARM64_32})
 set(ALL_DFSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${LOONGARCH64})
-#set(ALL_RTSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64}
-#    ${MIPS32} ${MIPS64} ${PPC64} ${S390X} ${SPARC} ${SPARCV9} ${HEXAGON}
-#    ${LOONGARCH64})
+set(ALL_RTSAN_SUPPORTED_ARCH ${X86_64} ${ARM64})
 
 if(ANDROID)
   set(OS_NAME "Android")
diff --git a/compiler-rt/lib/rtsan/rtsan.cpp b/compiler-rt/lib/rtsan/rtsan.cpp
index 43a63b4636f1e..cf7fbddd9eb9c 100644
--- a/compiler-rt/lib/rtsan/rtsan.cpp
+++ b/compiler-rt/lib/rtsan/rtsan.cpp
@@ -12,10 +12,23 @@
 #include <rtsan/rtsan_context.h>
 #include <rtsan/rtsan_interceptors.h>
 
+using namespace __rtsan;
+
+bool __rtsan::rtsan_initialized;
+bool __rtsan::rtsan_init_is_running;
+
 extern "C" {
 
 SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_init() {
-  __rtsan::InitializeInterceptors();
+  CHECK(!rtsan_init_is_running);
+  if (rtsan_initialized)
+    return;
+  rtsan_init_is_running = true;
+
+  InitializeInterceptors();
+
+  rtsan_init_is_running = false;
+  rtsan_initialized = true;
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_realtime_enter() {
diff --git a/compiler-rt/lib/rtsan/rtsan.h b/compiler-rt/lib/rtsan/rtsan.h
index 8b1219c13cbd3..ccddaf2c893ef 100644
--- a/compiler-rt/lib/rtsan/rtsan.h
+++ b/compiler-rt/lib/rtsan/rtsan.h
@@ -14,6 +14,13 @@
 
 extern "C" {
 
+namespace __rtsan {
+
+extern bool rtsan_initialized;
+extern bool rtsan_init_is_running;
+
+} // namespace __rtsan
+
 // Initialise rtsan interceptors.
 // A call to this method is added to the preinit array on Linux systems.
 SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_init();
diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
index 3a65f9d3f779d..4d5423ec629d2 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
@@ -10,10 +10,14 @@
 
 #include "rtsan/rtsan_interceptors.h"
 
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
 #include "sanitizer_common/sanitizer_platform.h"
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
 
 #include "interception/interception.h"
+#include "rtsan/rtsan.h"
 #include "rtsan/rtsan_context.h"
 
 #if SANITIZER_APPLE
@@ -35,6 +39,15 @@
 
 using namespace __sanitizer;
 
+using __rtsan::rtsan_init_is_running;
+using __rtsan::rtsan_initialized;
+
+namespace {
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+  static bool UseImpl() { return !rtsan_initialized; }
+};
+} // namespace
+
 void ExpectNotRealtime(const char *intercepted_function_name) {
   __rtsan::GetContextForThisThread().ExpectNotRealtime(
       intercepted_function_name);
@@ -238,11 +251,17 @@ INTERCEPTOR(int, nanosleep, const struct timespec *rqtp,
 // Memory
 
 INTERCEPTOR(void *, calloc, SIZE_T num, SIZE_T size) {
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Callocate(num, size);
+
   ExpectNotRealtime("calloc");
   return REAL(calloc)(num, size);
 }
 
 INTERCEPTOR(void, free, void *ptr) {
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
+
   if (ptr != NULL) {
     ExpectNotRealtime("free");
   }
@@ -250,11 +269,17 @@ INTERCEPTOR(void, free, void *ptr) {
 }
 
 INTERCEPTOR(void *, malloc, SIZE_T size) {
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Allocate(size);
+
   ExpectNotRealtime("malloc");
   return REAL(malloc)(size);
 }
 
 INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
+  if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Realloc(ptr, size);
+
   ExpectNotRealtime("realloc");
   return REAL(realloc)(ptr, size);
 }

From b8461769507d5b4a8c90d3722830b802d948e8b5 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private@pm.me>
Date: Mon, 22 Jul 2024 13:40:20 -0700
Subject: [PATCH 483/486] [compiler-rt] Moved cmake libatomic check to top
 level config-ix (#99437)

Also add atomic check to rtsan for when the tests are merged (related
review #98679)
---
 compiler-rt/cmake/config-ix.cmake                     | 1 +
 compiler-rt/lib/rtsan/tests/CMakeLists.txt            | 1 +
 compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt | 7 ++-----
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index 74fdbd288a1f0..b69c391751707 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -176,6 +176,7 @@ check_cxx_compiler_flag(-nostdlib++ COMPILER_RT_HAS_NOSTDLIBXX_FLAG)
 check_include_files("sys/auxv.h"    COMPILER_RT_HAS_AUXV)
 
 # Libraries.
+check_library_exists(atomic __atomic_load_8 "" COMPILER_RT_HAS_LIBATOMIC)
 check_library_exists(dl dlopen "" COMPILER_RT_HAS_LIBDL)
 check_library_exists(rt shm_open "" COMPILER_RT_HAS_LIBRT)
 check_library_exists(m pow "" COMPILER_RT_HAS_LIBM)
diff --git a/compiler-rt/lib/rtsan/tests/CMakeLists.txt b/compiler-rt/lib/rtsan/tests/CMakeLists.txt
index 9eda116541ae3..3b783c90c2658 100644
--- a/compiler-rt/lib/rtsan/tests/CMakeLists.txt
+++ b/compiler-rt/lib/rtsan/tests/CMakeLists.txt
@@ -36,6 +36,7 @@ set(RTSAN_UNITTEST_LINK_FLAGS
   ${SANITIZER_TEST_CXX_LIBRARIES}
   -no-pie)
 
+append_list_if(COMPILER_RT_HAS_LIBATOMIC -latomic RTSAN_UNITTEST_LINK_FLAGS)
 append_list_if(COMPILER_RT_HAS_LIBDL -ldl RTSAN_UNITTEST_LINK_FLAGS)
 append_list_if(COMPILER_RT_HAS_LIBRT -lrt RTSAN_UNITTEST_LINK_FLAGS)
 append_list_if(COMPILER_RT_HAS_LIBM -lm RTSAN_UNITTEST_LINK_FLAGS)
diff --git a/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt
index 672cdc0ba655e..a85eb737dba0a 100644
--- a/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt
+++ b/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt
@@ -46,11 +46,8 @@ set(SCUDO_UNITTEST_LINK_FLAGS
   ${COMPILER_RT_UNWINDER_LINK_LIBS}
   ${SANITIZER_TEST_CXX_LIBRARIES})
 list(APPEND SCUDO_UNITTEST_LINK_FLAGS -pthread -no-pie)
-# Linking against libatomic is required with some compilers
-check_library_exists(atomic __atomic_load_8 "" COMPILER_RT_HAS_LIBATOMIC)
-if (COMPILER_RT_HAS_LIBATOMIC)
-  list(APPEND SCUDO_UNITTEST_LINK_FLAGS -latomic)
-endif()
+
+append_list_if(COMPILER_RT_HAS_LIBATOMIC -latomic SCUDO_UNITTEST_LINK_FLAGS)
 
 set(SCUDO_TEST_HEADERS
   scudo_unit_test.h

From d48d4805f792adbdac58d480f890449def4964ea Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Mon, 22 Jul 2024 20:38:16 +0000
Subject: [PATCH 484/486] [clang-tidy][NFC] Fix tiny bug in
 areStatementsIdentical

Function areStatementsIdentical had an early exit
when classes of stmt does not match. That if were
added to speed up checking and do not calculate
hash id of both objects if they are not the same
type. Due to some bug, wrong pointer were used
and this resulted with comparing self to self.

This patch fixes this issue.
---
 clang-tools-extra/clang-tidy/utils/ASTUtils.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clang-tidy/utils/ASTUtils.cpp b/clang-tools-extra/clang-tidy/utils/ASTUtils.cpp
index fd5dadc9b01db..0cdc7d08abc99 100644
--- a/clang-tools-extra/clang-tidy/utils/ASTUtils.cpp
+++ b/clang-tools-extra/clang-tidy/utils/ASTUtils.cpp
@@ -96,7 +96,7 @@ bool areStatementsIdentical(const Stmt *FirstStmt, const Stmt *SecondStmt,
   if (FirstStmt == SecondStmt)
     return true;
 
-  if (FirstStmt->getStmtClass() != FirstStmt->getStmtClass())
+  if (FirstStmt->getStmtClass() != SecondStmt->getStmtClass())
     return false;
 
   if (isa<Expr>(FirstStmt) && isa<Expr>(SecondStmt)) {

From 25897ba420b93449542a60ffc9542e62b7480807 Mon Sep 17 00:00:00 2001
From: xur-llvm <59886942+xur-llvm@users.noreply.github.com>
Date: Mon, 22 Jul 2024 14:05:34 -0700
Subject: [PATCH 485/486] [PGO] Sync InstrProfData.inc from llvm to compiler-rt
 (#99930)

Sync InstrProfData.inc from llvm to compiler-rt. The difference was
introduced from https://github.com/llvm/llvm-project/pull/69535.
---
 compiler-rt/include/profile/InstrProfData.inc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index e9866d94b762c..847e53cfa7432 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -739,6 +739,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
 #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
 #define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
+#define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling
 
 /* The variable that holds the name of the profile data
  * specified via command line. */

From 8ee4b532ad9819d681412b0dd8851f052cc6a323 Mon Sep 17 00:00:00 2001
From: Chris Apple <14171107+cjappl@users.noreply.github.com>
Date: Mon, 3 Jun 2024 18:40:25 -0700
Subject: [PATCH 486/486] Add read and write interceptors

---
 compiler-rt/lib/rtsan/rtsan_interceptors.cpp  |  42 ++++++
 .../rtsan/tests/rtsan_test_interceptors.cpp   | 121 ++++++++++++++----
 2 files changed, 139 insertions(+), 24 deletions(-)

diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
index 4d5423ec629d2..dd84ba582bea6 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
@@ -150,6 +150,40 @@ INTERCEPTOR(int, puts, const char *s) {
   return REAL(puts)(s);
 }
 
+INTERCEPTOR(ssize_t, read, int fd, void *buf, size_t count) {
+  ExpectNotRealtime("read");
+  return REAL(read)(fd, buf, count);
+}
+
+INTERCEPTOR(ssize_t, write, int fd, const void *buf, size_t count) {
+  ExpectNotRealtime("write");
+  return REAL(write)(fd, buf, count);
+}
+
+#if SANITIZER_APPLE
+INTERCEPTOR(ssize_t, pread, int fd, void *buf, size_t count, off_t offset) {
+  ExpectNotRealtime("pread");
+  return REAL(pread)(fd, buf, count, offset);
+}
+
+INTERCEPTOR(ssize_t, readv, int fd, const struct iovec *iov, int iovcnt) {
+  ExpectNotRealtime("readv");
+  return REAL(readv)(fd, iov, iovcnt);
+}
+
+INTERCEPTOR(ssize_t, pwrite, int fd, const void *buf, size_t count,
+            off_t offset) {
+  ExpectNotRealtime("pwrite");
+  return REAL(pwrite)(fd, buf, count, offset);
+}
+
+INTERCEPTOR(ssize_t, writev, int fd, const struct iovec *iov, int iovcnt) {
+  ExpectNotRealtime("writev");
+  return REAL(writev)(fd, iov, iovcnt);
+}
+
+#endif // SANITIZER_APPLE
+
 // Concurrency
 #if SANITIZER_APPLE
 #pragma clang diagnostic push
@@ -389,6 +423,14 @@ void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(close);
   INTERCEPT_FUNCTION(fopen);
   INTERCEPT_FUNCTION(fread);
+  INTERCEPT_FUNCTION(read);
+  INTERCEPT_FUNCTION(write);
+#if SANITIZER_APPLE
+  INTERCEPT_FUNCTION(pread);
+  INTERCEPT_FUNCTION(readv);
+  INTERCEPT_FUNCTION(pwrite);
+  INTERCEPT_FUNCTION(writev);
+#endif
   INTERCEPT_FUNCTION(fwrite);
   INTERCEPT_FUNCTION(fclose);
   INTERCEPT_FUNCTION(fcntl);
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp
index f5b016089087d..89dd368904042 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp
@@ -18,6 +18,9 @@
 #if SANITIZER_APPLE
 #include <libkern/OSAtomic.h>
 #include <os/lock.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <unistd.h>
 #endif
 
 #if SANITIZER_INTERCEPT_MEMALIGN || SANITIZER_INTERCEPT_PVALLOC
@@ -261,38 +264,53 @@ TEST(TestRtsanInterceptors, CloseDiesWhenRealtime) {
 
 TEST_F(RtsanFileTest, FopenDiesWhenRealtime) {
   auto func = [this]() {
-    auto fd = fopen(GetTemporaryFilePath(), "w");
-    EXPECT_THAT(fd, Ne(nullptr));
+    FILE *f = fopen(GetTemporaryFilePath(), "w");
+    EXPECT_THAT(f, Ne(nullptr));
   };
   ExpectRealtimeDeath(func, "fopen");
   ExpectNonRealtimeSurvival(func);
 }
 
-TEST_F(RtsanFileTest, FreadDiesWhenRealtime) {
-  auto fd = fopen(GetTemporaryFilePath(), "w");
-  auto func = [fd]() {
+class RtsanOpenedFileTest : public RtsanFileTest {
+protected:
+  void SetUp() override {
+    RtsanFileTest::SetUp();
+    file_ = fopen(GetTemporaryFilePath(), "w");
+    ASSERT_THAT(file_, Ne(nullptr));
+    fd_ = fileno(file_);
+    ASSERT_THAT(fd_, Ne(-1));
+  }
+
+  void TearDown() override {
+    if (file_ != nullptr)
+      fclose(file_);
+    RtsanFileTest::TearDown();
+  }
+
+  FILE *file_ = nullptr;
+  int fd_ = -1;
+};
+
+TEST_F(RtsanOpenedFileTest, FreadDiesWhenRealtime) {
+  auto Func = [this]() {
     char c{};
-    fread(&c, 1, 1, fd);
+    fread(&c, 1, 1, file_);
   };
   ExpectRealtimeDeath(func, "fread");
   ExpectNonRealtimeSurvival(func);
-  if (fd != nullptr)
-    fclose(fd);
 }
 
-TEST_F(RtsanFileTest, FwriteDiesWhenRealtime) {
-  auto fd = fopen(GetTemporaryFilePath(), "w");
-  ASSERT_NE(nullptr, fd);
-  auto message = "Hello, world!";
-  auto func = [&]() { fwrite(&message, 1, 4, fd); };
+TEST_F(RtsanOpenedFileTest, FwriteDiesWhenRealtime) {
+  const char *message = "Hello, world!";
+  auto func = [&]() { fwrite(&message, 1, 4, file_); };
   ExpectRealtimeDeath(func, "fwrite");
   ExpectNonRealtimeSurvival(func);
 }
 
 TEST_F(RtsanFileTest, FcloseDiesWhenRealtime) {
-  auto fd = fopen(GetTemporaryFilePath(), "w");
-  EXPECT_THAT(fd, Ne(nullptr));
-  auto func = [fd]() { fclose(fd); };
+  FILE *file = fopen(GetTemporaryFilePath(), "w");
+  ASSERT_THAT(file, Ne(nullptr));
+  auto func = [file]() { fclose(file); };
   ExpectRealtimeDeath(func, "fclose");
   ExpectNonRealtimeSurvival(func);
 }
@@ -303,16 +321,71 @@ TEST(TestRtsanInterceptors, PutsDiesWhenRealtime) {
   ExpectNonRealtimeSurvival(func);
 }
 
-TEST_F(RtsanFileTest, FputsDiesWhenRealtime) {
-  auto fd = fopen(GetTemporaryFilePath(), "w");
-  ASSERT_THAT(fd, Ne(nullptr)) << errno;
-  auto func = [fd]() { fputs("Hello, world!\n", fd); };
-  ExpectRealtimeDeath(func);
-  ExpectNonRealtimeSurvival(func);
-  if (fd != nullptr)
-    fclose(fd);
+TEST_F(RtsanOpenedFileTest, FputsDiesWhenRealtime) {
+  auto Func = [this]() { fputs("Hello, world!\n", file_); };
+  ExpectRealtimeDeath(Func);
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST_F(RtsanOpenedFileTest, ReadDiesWhenRealtime) {
+  auto Func = [this]() {
+    char c{};
+    read(fd_, &c, 1);
+  };
+  ExpectRealtimeDeath(Func, "read");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST_F(RtsanOpenedFileTest, WriteDiesWhenRealtime) {
+  auto Func = [this]() {
+    char c{};
+    write(fd_, &c, 1);
+  };
+  ExpectRealtimeDeath(Func, "write");
+  ExpectNonRealtimeSurvival(Func);
 }
 
+#if SANITIZER_APPLE
+TEST_F(RtsanOpenedFileTest, PreadDiesWhenRealtime) {
+  auto Func = [this]() {
+    char c{};
+    pread(fd_, &c, 1, 0);
+  };
+  ExpectRealtimeDeath(Func, "pread");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST_F(RtsanOpenedFileTest, ReadvDiesWhenRealtime) {
+  auto Func = [this]() {
+    char c{};
+    iovec iov{&c, 1};
+    readv(fd_, &iov, 1);
+  };
+  ExpectRealtimeDeath(Func, "readv");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST_F(RtsanOpenedFileTest, PwriteDiesWhenRealtime) {
+  auto Func = [this]() {
+    char c{};
+    pwrite(fd_, &c, 1, 0);
+  };
+  ExpectRealtimeDeath(Func, "pwrite");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST_F(RtsanOpenedFileTest, WritevDiesWhenRealtime) {
+  auto Func = [this]() {
+    char c{};
+    iovec iov{&c, 1};
+    writev(fd_, &iov, 1);
+  };
+  ExpectRealtimeDeath(Func, "writev");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+#endif // SANITIZER_APPLE
+
 /*
     Concurrency
 */